From cec1877e180393eba0f6ddb0cf97bf3a791631c7 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Fri, 7 Jun 2024 07:48:42 +0200 Subject: Merging upstream version 1.75.0+dfsg1. Signed-off-by: Daniel Baumann --- src/bootstrap/CHANGELOG.md | 71 - src/bootstrap/Cargo.lock | 181 +- src/bootstrap/Cargo.toml | 39 +- src/bootstrap/README.md | 14 +- src/bootstrap/bin/_helper.rs | 24 - src/bootstrap/bin/main.rs | 124 - src/bootstrap/bin/rustc.rs | 410 --- src/bootstrap/bin/rustdoc.rs | 86 - src/bootstrap/bin/sccache-plus-cl.rs | 38 - src/bootstrap/bootstrap.py | 29 +- src/bootstrap/bootstrap_test.py | 3 +- src/bootstrap/builder.rs | 2318 -------------- src/bootstrap/builder/tests.rs | 702 ----- src/bootstrap/cache.rs | 272 -- src/bootstrap/cc_detect.rs | 273 -- src/bootstrap/channel.rs | 160 - src/bootstrap/check.rs | 542 ---- src/bootstrap/clean.rs | 237 -- src/bootstrap/compile.rs | 2015 ------------ src/bootstrap/config.rs | 2104 ------------- src/bootstrap/config/tests.rs | 198 -- src/bootstrap/configure.py | 20 +- src/bootstrap/defaults/config.codegen.toml | 2 +- src/bootstrap/dist.rs | 2274 -------------- src/bootstrap/doc.rs | 1085 ------- src/bootstrap/download-ci-llvm-stamp | 2 +- src/bootstrap/download.rs | 686 ---- src/bootstrap/dylib_util.rs | 30 - src/bootstrap/flags.rs | 565 ---- src/bootstrap/format.rs | 322 -- src/bootstrap/install.rs | 320 -- src/bootstrap/job.rs | 143 - src/bootstrap/lib.rs | 1846 ----------- src/bootstrap/llvm.rs | 1357 -------- src/bootstrap/metadata.rs | 101 - src/bootstrap/metrics.rs | 258 -- src/bootstrap/render_tests.rs | 400 --- src/bootstrap/run.rs | 297 -- src/bootstrap/sanity.rs | 267 -- src/bootstrap/setup.rs | 599 ---- src/bootstrap/setup/tests.rs | 14 - src/bootstrap/src/bin/main.rs | 143 + src/bootstrap/src/bin/rustc.rs | 406 +++ src/bootstrap/src/bin/rustdoc.rs | 92 + src/bootstrap/src/bin/sccache-plus-cl.rs | 38 + src/bootstrap/src/core/build_steps/check.rs | 544 ++++ src/bootstrap/src/core/build_steps/clean.rs | 242 ++ src/bootstrap/src/core/build_steps/compile.rs | 2069 ++++++++++++ src/bootstrap/src/core/build_steps/dist.rs | 2366 ++++++++++++++ src/bootstrap/src/core/build_steps/doc.rs | 1094 +++++++ src/bootstrap/src/core/build_steps/format.rs | 322 ++ src/bootstrap/src/core/build_steps/install.rs | 332 ++ src/bootstrap/src/core/build_steps/llvm.rs | 1366 ++++++++ src/bootstrap/src/core/build_steps/mod.rs | 15 + src/bootstrap/src/core/build_steps/run.rs | 301 ++ src/bootstrap/src/core/build_steps/setup.rs | 623 ++++ src/bootstrap/src/core/build_steps/suggest.rs | 78 + .../src/core/build_steps/synthetic_targets.rs | 82 + src/bootstrap/src/core/build_steps/test.rs | 3315 ++++++++++++++++++++ src/bootstrap/src/core/build_steps/tool.rs | 849 +++++ src/bootstrap/src/core/build_steps/toolstate.rs | 478 +++ src/bootstrap/src/core/builder.rs | 2392 ++++++++++++++ src/bootstrap/src/core/config/config.rs | 2208 +++++++++++++ src/bootstrap/src/core/config/flags.rs | 571 ++++ src/bootstrap/src/core/config/mod.rs | 4 + src/bootstrap/src/core/download.rs | 701 +++++ src/bootstrap/src/core/metadata.rs | 101 + src/bootstrap/src/core/mod.rs | 6 + src/bootstrap/src/core/sanity.rs | 267 ++ src/bootstrap/src/lib.rs | 1874 +++++++++++ src/bootstrap/src/tests/builder.rs | 701 +++++ src/bootstrap/src/tests/config.rs | 219 ++ src/bootstrap/src/tests/setup.rs | 14 + src/bootstrap/src/utils/bin_helpers.rs | 28 + src/bootstrap/src/utils/cache.rs | 272 ++ src/bootstrap/src/utils/cc_detect.rs | 288 ++ src/bootstrap/src/utils/channel.rs | 159 + src/bootstrap/src/utils/dylib.rs | 27 + src/bootstrap/src/utils/exec.rs | 60 + src/bootstrap/src/utils/helpers.rs | 472 +++ src/bootstrap/src/utils/job.rs | 159 + src/bootstrap/src/utils/metrics.rs | 258 ++ src/bootstrap/src/utils/mod.rs | 15 + src/bootstrap/src/utils/render_tests.rs | 400 +++ src/bootstrap/src/utils/tarball.rs | 381 +++ src/bootstrap/suggest.rs | 74 - src/bootstrap/synthetic_targets.rs | 82 - src/bootstrap/tarball.rs | 373 --- src/bootstrap/test.rs | 3093 ------------------ src/bootstrap/tool.rs | 848 ----- src/bootstrap/toolstate.rs | 478 --- src/bootstrap/util.rs | 497 --- src/ci/docker/host-x86_64/arm-android/Dockerfile | 2 +- src/ci/docker/host-x86_64/dist-android/Dockerfile | 7 +- .../docker/host-x86_64/dist-various-1/Dockerfile | 47 +- .../dist-various-1/install-mips-musl.sh | 15 - .../dist-various-1/install-mipsel-musl.sh | 15 - .../host-x86_64/dist-x86_64-linux/Dockerfile | 7 +- .../host-x86_64/dist-x86_64-linux/build-clang.sh | 2 +- .../host-x86_64/dist-x86_64-linux/build-gcc.sh | 15 +- .../test-various/uefi_qemu_test/Cargo.lock | 16 + src/ci/docker/host-x86_64/wasm32/Dockerfile | 63 - .../host-x86_64/x86_64-gnu-llvm-15/Dockerfile | 5 + .../host-x86_64/x86_64-gnu-llvm-15/script.sh | 68 +- .../host-x86_64/x86_64-gnu-llvm-16/Dockerfile | 4 + .../host-x86_64/x86_64-gnu-llvm-17/Dockerfile | 50 + .../docker/host-x86_64/x86_64-gnu-tools/Dockerfile | 11 +- .../host-x86_64/x86_64-gnu-tools/checktools.sh | 29 +- src/ci/docker/run.sh | 23 +- src/ci/docker/scripts/fuchsia-test-runner.py | 48 +- src/ci/github-actions/ci.yml | 83 +- src/ci/run.sh | 20 +- src/ci/scripts/install-awscli.sh | 29 + src/ci/scripts/install-tidy.sh | 24 + src/ci/scripts/verify-channel.sh | 2 +- src/doc/book/redirects/compiler-plugins.md | 11 +- src/doc/book/src/ch02-00-guessing-game-tutorial.md | 4 +- src/doc/embedded-book/src/start/hardware.md | 3 + src/doc/guide-plugins.md | 3 +- src/doc/nomicon/src/exception-safety.md | 2 +- src/doc/reference/src/attributes.md | 4 +- src/doc/reference/src/attributes/codegen.md | 60 + .../reference/src/behavior-considered-undefined.md | 46 +- src/doc/reference/src/destructors.md | 6 +- src/doc/reference/src/expressions/operator-expr.md | 10 + src/doc/reference/src/inline-assembly.md | 4 +- src/doc/reference/src/items/traits.md | 2 +- src/doc/reference/src/types/impl-trait.md | 10 +- src/doc/reference/src/types/textual.md | 4 +- src/doc/rust-by-example/src/SUMMARY.md | 4 +- src/doc/rust-by-example/src/attribute.md | 32 +- .../rust-by-example/src/custom_types/constants.md | 2 +- .../src/error/option_unwrap/question_mark.md | 3 +- .../rust-by-example/src/flow_control/while_let.md | 34 +- src/doc/rust-by-example/src/fn/closures.md | 4 +- src/doc/rust-by-example/src/fn/hof.md | 2 +- src/doc/rust-by-example/src/meta/doc.md | 9 +- src/doc/rust-by-example/src/meta/playground.md | 15 +- src/doc/rust-by-example/src/primitives/array.md | 2 +- src/doc/rust-by-example/src/scope/lifetime.md | 2 +- src/doc/rust-by-example/src/scope/move.md | 2 +- .../rust-by-example/src/std/hash/alt_key_types.md | 2 +- .../rust-by-example/src/std_misc/arg/matching.md | 3 + .../src/std_misc/file/read_lines.md | 8 +- .../src/std_misc/threads/testcase_mapreduce.md | 2 +- .../examples/rustc-driver-example.rs | 2 + .../examples/rustc-driver-getting-diagnostics.rs | 2 + .../rustc-driver-interacting-with-the-ast.rs | 2 + src/doc/rustc-dev-guide/src/SUMMARY.md | 2 + .../rustc-dev-guide/src/appendix/bibliography.md | 2 +- src/doc/rustc-dev-guide/src/appendix/glossary.md | 2 + src/doc/rustc-dev-guide/src/building/suggested.md | 1 - src/doc/rustc-dev-guide/src/effects.md | 66 + src/doc/rustc-dev-guide/src/feature-gates.md | 12 +- src/doc/rustc-dev-guide/src/hir-debugging.md | 9 +- .../src/implementing_new_features.md | 6 +- .../src/llvm-coverage-instrumentation.md | 123 +- .../src/return-position-impl-trait-in-trait.md | 18 + .../src/rustc-driver-getting-diagnostics.md | 2 +- .../src/rustc-driver-interacting-with-the-ast.md | 2 +- src/doc/rustc-dev-guide/src/solve/invariants.md | 154 + src/doc/rustc-dev-guide/src/solve/the-solver.md | 73 +- src/doc/rustc-dev-guide/src/solve/trait-solving.md | 71 - src/doc/rustc-dev-guide/src/stabilization_guide.md | 2 +- src/doc/rustc-dev-guide/src/tests/headers.md | 2 +- src/doc/rustc-dev-guide/src/traits/unsize.md | 84 + src/doc/rustc/src/SUMMARY.md | 3 +- src/doc/rustc/src/codegen-options/index.md | 8 +- src/doc/rustc/src/exploit-mitigations.md | 404 ++- src/doc/rustc/src/images/image1.png | Bin 15293 -> 164896 bytes src/doc/rustc/src/images/image2.png | Bin 28772 -> 155307 bytes src/doc/rustc/src/images/image3.png | Bin 19069 -> 19936 bytes src/doc/rustc/src/platform-support.md | 73 +- .../src/platform-support/aarch64-unknown-teeos.md | 2 +- src/doc/rustc/src/platform-support/aix.md | 26 + src/doc/rustc/src/platform-support/android.md | 16 + src/doc/rustc/src/platform-support/apple-tvos.md | 4 +- .../armv7-sony-vita-newlibeabihf.md | 137 +- .../csky-unknown-linux-gnuabiv2.md | 24 +- src/doc/rustc/src/platform-support/fuchsia.md | 4 +- .../rustc/src/platform-support/mips-release-6.md | 2 +- src/doc/rustc/src/platform-support/nto-qnx.md | 2 +- src/doc/rustc/src/platform-support/openharmony.md | 2 +- src/doc/rustc/src/platform-support/unknown-uefi.md | 10 +- src/doc/rustc/src/profile-guided-optimization.md | 23 + src/doc/rustdoc/src/advanced-features.md | 20 + src/doc/rustdoc/src/unstable-features.md | 19 +- .../src/write-documentation/what-to-include.md | 4 +- .../unstable-book/src/compiler-flags/check-cfg.md | 216 +- .../src/compiler-flags/no-jump-tables.md | 19 + .../src/compiler-flags/remap-path-scope.md | 24 + .../unstable-book/src/compiler-flags/sanitizer.md | 161 +- .../src/language-features/closure-track-caller.md | 4 +- .../src/language-features/coroutines.md | 246 ++ .../src/language-features/diagnostic-namespace.md | 84 + .../src/language-features/generators.md | 246 -- .../unstable-book/src/language-features/plugin.md | 114 - .../src/language-features/string-deref-patterns.md | 45 + src/doc/unstable-book/src/the-unstable-book.md | 18 +- src/etc/completions/x.py.fish | 120 +- src/etc/completions/x.py.ps1 | 38 +- src/etc/completions/x.py.sh | 32 +- src/etc/completions/x.py.zsh | 766 +++++ src/etc/gdb_lookup.py | 133 +- src/etc/gdb_providers.py | 307 +- src/etc/test-float-parse/Cargo.lock | 75 + src/etc/test-float-parse/Cargo.toml | 2 +- src/librustdoc/Cargo.toml | 3 +- src/librustdoc/clean/auto_trait.rs | 7 +- src/librustdoc/clean/inline.rs | 37 +- src/librustdoc/clean/mod.rs | 209 +- src/librustdoc/clean/simplify.rs | 21 +- src/librustdoc/clean/types.rs | 98 +- src/librustdoc/clean/types/tests.rs | 3 +- src/librustdoc/clean/utils.rs | 164 +- src/librustdoc/core.rs | 17 +- src/librustdoc/doctest.rs | 13 +- src/librustdoc/formats/cache.rs | 86 +- src/librustdoc/formats/item_type.rs | 5 +- src/librustdoc/html/format.rs | 80 +- src/librustdoc/html/layout.rs | 37 +- src/librustdoc/html/markdown.rs | 5 +- src/librustdoc/html/render/context.rs | 75 +- src/librustdoc/html/render/mod.rs | 140 +- src/librustdoc/html/render/print_item.rs | 274 +- src/librustdoc/html/render/search_index.rs | 33 +- src/librustdoc/html/render/sidebar.rs | 105 +- src/librustdoc/html/render/write_shared.rs | 289 +- src/librustdoc/html/sources.rs | 7 +- src/librustdoc/html/static/css/rustdoc.css | 87 +- src/librustdoc/html/static/js/main.js | 281 +- src/librustdoc/html/static/js/search.js | 21 +- src/librustdoc/html/templates/page.html | 52 +- src/librustdoc/html/templates/sidebar.html | 7 +- src/librustdoc/json/conversions.rs | 28 +- src/librustdoc/lib.rs | 15 +- src/librustdoc/markdown.rs | 2 +- src/librustdoc/passes/collect_intra_doc_links.rs | 6 +- src/librustdoc/passes/collect_trait_impls.rs | 20 +- src/librustdoc/passes/lint/html_tags.rs | 150 +- .../passes/lint/redundant_explicit_links.rs | 2 +- src/stage0.json | 553 ++-- src/tools/build-manifest/src/main.rs | 40 +- src/tools/build-manifest/src/versions.rs | 3 + src/tools/build_helper/src/ci.rs | 6 +- src/tools/build_helper/src/git.rs | 41 +- src/tools/cargo/.github/renovate.json5 | 40 +- src/tools/cargo/.github/workflows/audit.yml | 2 +- src/tools/cargo/.github/workflows/contrib.yml | 26 +- src/tools/cargo/.github/workflows/main.yml | 34 +- src/tools/cargo/CHANGELOG.md | 219 +- src/tools/cargo/CONTRIBUTING.md | 10 +- src/tools/cargo/Cargo.lock | 470 +-- src/tools/cargo/Cargo.toml | 87 +- src/tools/cargo/ci/generate.py | 49 + src/tools/cargo/crates/cargo-platform/Cargo.toml | 3 +- .../cargo/crates/cargo-test-support/src/compare.rs | 2 + .../cargo/crates/cargo-test-support/src/diff.rs | 2 +- .../cargo/crates/cargo-test-support/src/lib.rs | 48 + src/tools/cargo/crates/cargo-util/Cargo.toml | 4 +- src/tools/cargo/crates/cargo-util/src/paths.rs | 39 +- src/tools/cargo/crates/crates-io/Cargo.toml | 2 +- src/tools/cargo/crates/crates-io/lib.rs | 4 + src/tools/cargo/crates/home/Cargo.toml | 3 +- src/tools/cargo/crates/home/src/lib.rs | 1 - src/tools/cargo/crates/resolver-tests/src/lib.rs | 21 +- src/tools/cargo/crates/xtask-bump-check/Cargo.toml | 3 +- .../cargo/crates/xtask-bump-check/src/xtask.rs | 24 +- .../cargo-credential-1password/Cargo.toml | 3 +- .../cargo-credential-1password/LICENSE-APACHE | 1 + .../cargo-credential-1password/LICENSE-MIT | 1 + .../cargo-credential-1password/README.md | 38 +- .../cargo-credential-1password/src/main.rs | 4 + .../cargo-credential-libsecret/Cargo.toml | 3 +- .../cargo-credential-libsecret/LICENSE-APACHE | 1 + .../cargo-credential-libsecret/LICENSE-MIT | 1 + .../cargo-credential-macos-keychain/Cargo.toml | 3 +- .../cargo-credential-macos-keychain/LICENSE-APACHE | 1 + .../cargo-credential-macos-keychain/LICENSE-MIT | 1 + .../credential/cargo-credential-wincred/Cargo.toml | 3 +- .../cargo-credential-wincred/LICENSE-APACHE | 1 + .../cargo-credential-wincred/LICENSE-MIT | 1 + .../cargo/credential/cargo-credential/Cargo.toml | 4 +- .../credential/cargo-credential/LICENSE-APACHE | 1 + .../cargo/credential/cargo-credential/LICENSE-MIT | 1 + src/tools/cargo/src/bin/cargo/cli.rs | 32 +- src/tools/cargo/src/bin/cargo/commands/add.rs | 2 +- src/tools/cargo/src/bin/cargo/commands/bench.rs | 5 +- src/tools/cargo/src/bin/cargo/commands/build.rs | 13 +- src/tools/cargo/src/bin/cargo/commands/check.rs | 4 +- src/tools/cargo/src/bin/cargo/commands/fix.rs | 4 +- src/tools/cargo/src/bin/cargo/commands/init.rs | 7 +- src/tools/cargo/src/bin/cargo/commands/install.rs | 17 +- src/tools/cargo/src/bin/cargo/commands/login.rs | 2 +- src/tools/cargo/src/bin/cargo/commands/new.rs | 7 +- src/tools/cargo/src/bin/cargo/commands/owner.rs | 2 +- src/tools/cargo/src/bin/cargo/commands/pkgid.rs | 2 +- src/tools/cargo/src/bin/cargo/commands/remove.rs | 13 +- src/tools/cargo/src/bin/cargo/commands/run.rs | 1 + src/tools/cargo/src/bin/cargo/commands/rustc.rs | 5 +- src/tools/cargo/src/bin/cargo/commands/rustdoc.rs | 5 +- src/tools/cargo/src/bin/cargo/commands/search.rs | 2 +- src/tools/cargo/src/bin/cargo/commands/test.rs | 5 +- .../cargo/src/bin/cargo/commands/uninstall.rs | 2 +- src/tools/cargo/src/bin/cargo/commands/yank.rs | 2 +- src/tools/cargo/src/bin/cargo/main.rs | 9 +- .../cargo/src/cargo/core/compiler/context/mod.rs | 8 + .../cargo/src/cargo/core/compiler/custom_build.rs | 20 +- .../src/cargo/core/compiler/future_incompat.rs | 10 +- .../cargo/src/cargo/core/compiler/job_queue/mod.rs | 5 +- src/tools/cargo/src/cargo/core/compiler/layout.rs | 2 +- src/tools/cargo/src/cargo/core/compiler/mod.rs | 129 +- src/tools/cargo/src/cargo/core/compiler/timings.rs | 32 +- src/tools/cargo/src/cargo/core/dependency.rs | 4 +- src/tools/cargo/src/cargo/core/features.rs | 82 +- src/tools/cargo/src/cargo/core/manifest.rs | 2 +- src/tools/cargo/src/cargo/core/mod.rs | 2 +- src/tools/cargo/src/cargo/core/package.rs | 11 +- src/tools/cargo/src/cargo/core/package_id.rs | 85 +- src/tools/cargo/src/cargo/core/package_id_spec.rs | 162 +- src/tools/cargo/src/cargo/core/profiles.rs | 70 +- src/tools/cargo/src/cargo/core/registry.rs | 10 +- src/tools/cargo/src/cargo/core/resolver/context.rs | 4 - .../cargo/src/cargo/core/resolver/dep_cache.rs | 77 +- src/tools/cargo/src/cargo/core/resolver/encode.rs | 2 +- src/tools/cargo/src/cargo/core/resolver/errors.rs | 3 +- src/tools/cargo/src/cargo/core/resolver/mod.rs | 54 +- src/tools/cargo/src/cargo/core/resolver/resolve.rs | 11 + .../cargo/src/cargo/core/resolver/version_prefs.rs | 126 +- src/tools/cargo/src/cargo/core/shell.rs | 137 +- src/tools/cargo/src/cargo/core/source_id.rs | 244 +- src/tools/cargo/src/cargo/core/summary.rs | 6 +- src/tools/cargo/src/cargo/core/workspace.rs | 6 +- src/tools/cargo/src/cargo/lib.rs | 3 +- src/tools/cargo/src/cargo/ops/cargo_add/mod.rs | 47 +- src/tools/cargo/src/cargo/ops/cargo_compile/mod.rs | 2 +- src/tools/cargo/src/cargo/ops/cargo_doc.rs | 24 +- .../cargo/src/cargo/ops/cargo_generate_lockfile.rs | 22 +- src/tools/cargo/src/cargo/ops/cargo_install.rs | 42 +- src/tools/cargo/src/cargo/ops/cargo_new.rs | 113 +- src/tools/cargo/src/cargo/ops/cargo_package.rs | 20 +- src/tools/cargo/src/cargo/ops/cargo_uninstall.rs | 1 + .../cargo/ops/common_for_install_and_uninstall.rs | 70 +- src/tools/cargo/src/cargo/ops/fix.rs | 5 + src/tools/cargo/src/cargo/ops/lockfile.rs | 23 +- src/tools/cargo/src/cargo/ops/registry/mod.rs | 3 +- src/tools/cargo/src/cargo/ops/registry/publish.rs | 5 +- src/tools/cargo/src/cargo/ops/registry/search.rs | 39 +- src/tools/cargo/src/cargo/ops/resolve.rs | 24 +- src/tools/cargo/src/cargo/ops/vendor.rs | 2 +- src/tools/cargo/src/cargo/sources/config.rs | 6 +- src/tools/cargo/src/cargo/sources/git/source.rs | 17 +- .../cargo/src/cargo/sources/registry/download.rs | 7 +- .../src/cargo/sources/registry/http_remote.rs | 5 +- .../cargo/src/cargo/sources/registry/index.rs | 87 +- src/tools/cargo/src/cargo/sources/registry/mod.rs | 71 +- .../cargo/src/cargo/sources/registry/remote.rs | 15 +- src/tools/cargo/src/cargo/util/auth/mod.rs | 6 + src/tools/cargo/src/cargo/util/cache_lock.rs | 549 ++++ src/tools/cargo/src/cargo/util/command_prelude.rs | 70 +- src/tools/cargo/src/cargo/util/config/mod.rs | 135 +- src/tools/cargo/src/cargo/util/config/target.rs | 8 +- src/tools/cargo/src/cargo/util/flock.rs | 287 +- src/tools/cargo/src/cargo/util/hostname.rs | 77 + src/tools/cargo/src/cargo/util/mod.rs | 7 +- src/tools/cargo/src/cargo/util/restricted_names.rs | 79 + src/tools/cargo/src/cargo/util/rustc.rs | 14 + src/tools/cargo/src/cargo/util/semver_ext.rs | 293 +- src/tools/cargo/src/cargo/util/to_semver.rs | 36 - src/tools/cargo/src/cargo/util/toml/embedded.rs | 16 +- src/tools/cargo/src/cargo/util/toml/mod.rs | 2709 +++++----------- src/tools/cargo/src/cargo/util/toml/schema.rs | 1189 +++++++ src/tools/cargo/src/cargo/util/toml/targets.rs | 120 +- .../cargo/src/cargo/util/toml_mut/dependency.rs | 45 +- .../cargo/src/cargo/util/toml_mut/manifest.rs | 100 +- src/tools/cargo/src/cargo/util/toml_mut/mod.rs | 16 + src/tools/cargo/src/cargo/util/workspace.rs | 4 +- src/tools/cargo/src/cargo/util_semver.rs | 195 ++ src/tools/cargo/src/doc/contrib/book.toml | 2 + src/tools/cargo/src/doc/contrib/src/SUMMARY.md | 2 + .../doc/contrib/src/implementation/formatting.md | 17 + .../src/doc/contrib/src/implementation/packages.md | 52 + .../cargo/src/doc/contrib/src/process/index.md | 9 +- .../doc/contrib/src/process/working-on-cargo.md | 9 +- src/tools/cargo/src/doc/man/cargo-add.md | 2 +- src/tools/cargo/src/doc/man/cargo-bench.md | 2 +- src/tools/cargo/src/doc/man/cargo-install.md | 2 +- src/tools/cargo/src/doc/man/cargo-login.md | 4 +- src/tools/cargo/src/doc/man/cargo-rustc.md | 2 +- src/tools/cargo/src/doc/man/cargo-vendor.md | 10 +- .../cargo/src/doc/man/generated_txt/cargo-add.txt | 4 +- .../src/doc/man/generated_txt/cargo-bench.txt | 5 +- .../src/doc/man/generated_txt/cargo-build.txt | 2 +- .../src/doc/man/generated_txt/cargo-check.txt | 2 +- .../cargo/src/doc/man/generated_txt/cargo-doc.txt | 2 +- .../cargo/src/doc/man/generated_txt/cargo-fix.txt | 2 +- .../cargo/src/doc/man/generated_txt/cargo-init.txt | 2 +- .../src/doc/man/generated_txt/cargo-install.txt | 4 +- .../src/doc/man/generated_txt/cargo-login.txt | 5 +- .../cargo/src/doc/man/generated_txt/cargo-new.txt | 2 +- .../cargo/src/doc/man/generated_txt/cargo-run.txt | 2 +- .../src/doc/man/generated_txt/cargo-rustc.txt | 2 +- .../src/doc/man/generated_txt/cargo-rustdoc.txt | 2 +- .../cargo/src/doc/man/generated_txt/cargo-test.txt | 2 +- .../src/doc/man/generated_txt/cargo-vendor.txt | 11 +- .../cargo/src/doc/man/includes/options-new.md | 2 +- .../man/includes/options-profile-legacy-check.md | 2 +- .../cargo/src/doc/man/includes/options-profile.md | 2 +- src/tools/cargo/src/doc/src/commands/cargo-add.md | 2 +- .../cargo/src/doc/src/commands/cargo-bench.md | 4 +- .../cargo/src/doc/src/commands/cargo-build.md | 2 +- .../cargo/src/doc/src/commands/cargo-check.md | 2 +- src/tools/cargo/src/doc/src/commands/cargo-doc.md | 2 +- src/tools/cargo/src/doc/src/commands/cargo-fix.md | 2 +- src/tools/cargo/src/doc/src/commands/cargo-init.md | 2 +- .../cargo/src/doc/src/commands/cargo-install.md | 4 +- .../cargo/src/doc/src/commands/cargo-login.md | 4 +- src/tools/cargo/src/doc/src/commands/cargo-new.md | 2 +- src/tools/cargo/src/doc/src/commands/cargo-run.md | 2 +- .../cargo/src/doc/src/commands/cargo-rustc.md | 2 +- .../cargo/src/doc/src/commands/cargo-rustdoc.md | 2 +- src/tools/cargo/src/doc/src/commands/cargo-test.md | 2 +- .../cargo/src/doc/src/commands/cargo-vendor.md | 10 +- src/tools/cargo/src/doc/src/reference/config.md | 23 +- .../src/doc/src/reference/environment-variables.md | 2 +- src/tools/cargo/src/doc/src/reference/features.md | 7 + src/tools/cargo/src/doc/src/reference/manifest.md | 35 +- src/tools/cargo/src/doc/src/reference/profiles.md | 4 +- .../cargo/src/doc/src/reference/publishing.md | 2 +- .../doc/src/reference/registry-authentication.md | 2 +- src/tools/cargo/src/doc/src/reference/resolver.md | 47 +- .../doc/src/reference/specifying-dependencies.md | 4 +- src/tools/cargo/src/doc/src/reference/unstable.md | 172 +- src/tools/cargo/src/etc/man/cargo-add.1 | 2 +- src/tools/cargo/src/etc/man/cargo-bench.1 | 4 +- src/tools/cargo/src/etc/man/cargo-build.1 | 2 +- src/tools/cargo/src/etc/man/cargo-check.1 | 2 +- src/tools/cargo/src/etc/man/cargo-doc.1 | 2 +- src/tools/cargo/src/etc/man/cargo-fix.1 | 2 +- src/tools/cargo/src/etc/man/cargo-init.1 | 2 +- src/tools/cargo/src/etc/man/cargo-install.1 | 4 +- src/tools/cargo/src/etc/man/cargo-login.1 | 4 +- src/tools/cargo/src/etc/man/cargo-new.1 | 2 +- src/tools/cargo/src/etc/man/cargo-run.1 | 2 +- src/tools/cargo/src/etc/man/cargo-rustc.1 | 2 +- src/tools/cargo/src/etc/man/cargo-rustdoc.1 | 2 +- src/tools/cargo/src/etc/man/cargo-test.1 | 2 +- src/tools/cargo/src/etc/man/cargo-vendor.1 | 16 +- src/tools/cargo/tests/testsuite/artifact_dep.rs | 2 + src/tools/cargo/tests/testsuite/bad_config.rs | 36 +- src/tools/cargo/tests/testsuite/build.rs | 137 +- src/tools/cargo/tests/testsuite/build_script.rs | 11 +- .../cargo/tests/testsuite/build_script_env.rs | 22 +- .../tests/testsuite/build_script_extra_link_arg.rs | 2 +- src/tools/cargo/tests/testsuite/cache_lock.rs | 304 ++ src/tools/cargo/tests/testsuite/cargo_add/mod.rs | 1 + .../cargo_add/preserve_dep_std_table/in/Cargo.toml | 14 + .../cargo_add/preserve_dep_std_table/in/src/lib.rs | 0 .../cargo_add/preserve_dep_std_table/mod.rs | 31 + .../preserve_dep_std_table/out/Cargo.toml | 12 + .../cargo_add/preserve_dep_std_table/stderr.log | 7 + .../cargo_add/preserve_dep_std_table/stdout.log | 0 .../tests/testsuite/cargo_bench/help/stdout.log | 8 +- .../testsuite/cargo_bench/no_keep_going/stderr.log | 2 +- .../tests/testsuite/cargo_build/help/stdout.log | 4 +- .../tests/testsuite/cargo_check/help/stdout.log | 4 +- src/tools/cargo/tests/testsuite/cargo_command.rs | 12 +- src/tools/cargo/tests/testsuite/cargo_features.rs | 8 +- .../tests/testsuite/cargo_fix/help/stdout.log | 4 +- .../tests/testsuite/cargo_init/empty_dir/.keep | 0 .../tests/testsuite/cargo_init/help/stdout.log | 6 +- src/tools/cargo/tests/testsuite/cargo_init/mod.rs | 1 + .../testsuite/cargo_init/unknown_flags/stderr.log | 2 +- .../cargo_init/workspace_add_member/in/Cargo.toml | 2 + .../workspace_add_member/in/crates/foo/.keep | 0 .../cargo_init/workspace_add_member/mod.rs | 21 + .../cargo_init/workspace_add_member/out/Cargo.toml | 3 + .../workspace_add_member/out/crates/foo/Cargo.toml | 8 + .../out/crates/foo/src/main.rs | 3 + .../cargo_init/workspace_add_member/stderr.log | 1 + .../cargo_init/workspace_add_member/stdout.log | 0 .../tests/testsuite/cargo_install/help/stdout.log | 4 +- .../tests/testsuite/cargo_login/help/stdout.log | 4 +- .../in/Cargo.toml | 5 + .../mod.rs | 22 + .../out/Cargo.toml | 6 + .../out/crates/foo/Cargo.toml | 9 + .../out/crates/foo/src/main.rs | 3 + .../stderr.log | 1 + .../stdout.log | 0 .../in/Cargo.toml | 3 + .../in/crates/bar/Cargo.toml | 8 + .../in/crates/bar/src/main.rs | 3 + .../in/crates/qux/Cargo.toml | 8 + .../in/crates/qux/src/main.rs | 3 + .../add_members_to_workspace_format_sorted/mod.rs | 22 + .../out/Cargo.toml | 3 + .../out/crates/foo/Cargo.toml | 8 + .../out/crates/foo/src/main.rs | 3 + .../stderr.log | 1 + .../stdout.log | 0 .../in/Cargo.toml | 3 + .../mod.rs | 23 + .../out/Cargo.toml | 3 + .../out/crates/foo/Cargo.toml | 8 + .../out/crates/foo/src/main.rs | 3 + .../stderr.log | 1 + .../stdout.log | 0 .../in/Cargo.toml | 3 + .../mod.rs | 22 + .../out/Cargo.toml | 3 + .../out/crates/foo/Cargo.toml | 8 + .../out/crates/foo/src/main.rs | 3 + .../stderr.log | 1 + .../stdout.log | 0 .../in/Cargo.toml | 4 + .../in/src/lib.rs | 14 + .../mod.rs | 22 + .../out/Cargo.toml | 4 + .../out/crates/foo/Cargo.toml | 8 + .../out/crates/foo/src/main.rs | 3 + .../stderr.log | 1 + .../stdout.log | 0 .../in/Cargo.toml | 3 + .../mod.rs | 22 + .../out/Cargo.toml | 3 + .../out/crates/foo/Cargo.toml | 8 + .../out/crates/foo/src/main.rs | 3 + .../stderr.log | 1 + .../stdout.log | 0 .../tests/testsuite/cargo_new/help/stdout.log | 6 +- src/tools/cargo/tests/testsuite/cargo_new/mod.rs | 6 + .../out/Cargo.toml | 2 + .../stderr.log | 8 - .../tests/testsuite/cargo_owner/help/stdout.log | 4 +- .../tests/testsuite/cargo_pkgid/help/stdout.log | 4 +- .../tests/testsuite/cargo_remove/help/stdout.log | 6 +- .../testsuite/cargo_remove/invalid_dep/stderr.log | 2 +- .../cargo_remove/invalid_section/stderr.log | 2 +- .../cargo_remove/invalid_section_dep/stderr.log | 2 +- .../cargo_remove/invalid_target/stderr.log | 2 +- .../cargo_remove/invalid_target_dep/stderr.log | 2 +- .../cargo/tests/testsuite/cargo_remove/mod.rs | 1 + .../cargo_remove/multiple_dev/out/Cargo.toml | 2 +- .../optional_dep_feature/out/Cargo.toml | 2 +- .../optional_dep_feature_formatting/in/Cargo.toml | 42 + .../optional_dep_feature_formatting/in/src/lib.rs | 1 + .../optional_dep_feature_formatting/mod.rs | 35 + .../optional_dep_feature_formatting/out/Cargo.toml | 40 + .../optional_dep_feature_formatting/stderr.log | 2 + .../optional_dep_feature_formatting/stdout.log | 0 .../tests/testsuite/cargo_run/help/stdout.log | 4 +- .../tests/testsuite/cargo_rustc/help/stdout.log | 8 +- .../tests/testsuite/cargo_rustdoc/help/stdout.log | 8 +- .../tests/testsuite/cargo_search/help/stdout.log | 4 +- .../tests/testsuite/cargo_test/help/stdout.log | 8 +- .../testsuite/cargo_test/no_keep_going/stderr.log | 2 +- .../testsuite/cargo_uninstall/help/stdout.log | 4 +- .../tests/testsuite/cargo_yank/help/stdout.log | 4 +- src/tools/cargo/tests/testsuite/check.rs | 23 + src/tools/cargo/tests/testsuite/check_cfg.rs | 307 +- src/tools/cargo/tests/testsuite/collisions.rs | 6 + src/tools/cargo/tests/testsuite/config.rs | 109 +- src/tools/cargo/tests/testsuite/cross_compile.rs | 86 - src/tools/cargo/tests/testsuite/custom_target.rs | 2 + src/tools/cargo/tests/testsuite/death.rs | 153 +- src/tools/cargo/tests/testsuite/doc.rs | 46 +- src/tools/cargo/tests/testsuite/docscrape.rs | 33 +- src/tools/cargo/tests/testsuite/features.rs | 88 +- src/tools/cargo/tests/testsuite/features2.rs | 5 +- src/tools/cargo/tests/testsuite/glob_targets.rs | 5 + src/tools/cargo/tests/testsuite/install.rs | 50 +- src/tools/cargo/tests/testsuite/install_upgrade.rs | 2 +- src/tools/cargo/tests/testsuite/list_availables.rs | 8 +- src/tools/cargo/tests/testsuite/main.rs | 3 +- src/tools/cargo/tests/testsuite/metadata.rs | 282 ++ src/tools/cargo/tests/testsuite/multitarget.rs | 28 + src/tools/cargo/tests/testsuite/new.rs | 26 +- src/tools/cargo/tests/testsuite/out_dir.rs | 23 + src/tools/cargo/tests/testsuite/package.rs | 41 +- src/tools/cargo/tests/testsuite/plugins.rs | 421 --- src/tools/cargo/tests/testsuite/proc_macro.rs | 46 - src/tools/cargo/tests/testsuite/profile_config.rs | 2 +- src/tools/cargo/tests/testsuite/profile_targets.rs | 1 + .../cargo/tests/testsuite/profile_trim_paths.rs | 614 ++++ src/tools/cargo/tests/testsuite/pub_priv.rs | 49 + src/tools/cargo/tests/testsuite/publish.rs | 35 +- src/tools/cargo/tests/testsuite/registry.rs | 51 + src/tools/cargo/tests/testsuite/run.rs | 4 +- src/tools/cargo/tests/testsuite/rustdoc.rs | 8 +- src/tools/cargo/tests/testsuite/rustdocflags.rs | 6 +- src/tools/cargo/tests/testsuite/script.rs | 1 + src/tools/cargo/tests/testsuite/search.rs | 5 +- src/tools/cargo/tests/testsuite/update.rs | 98 + src/tools/cargo/tests/testsuite/version.rs | 4 + src/tools/cargo/tests/testsuite/warn_on_failure.rs | 4 +- src/tools/cargo/tests/testsuite/workspaces.rs | 17 +- src/tools/cargo/triagebot.toml | 10 +- src/tools/clippy/.github/workflows/clippy.yml | 2 +- src/tools/clippy/.github/workflows/clippy_bors.yml | 6 +- src/tools/clippy/CHANGELOG.md | 102 +- src/tools/clippy/Cargo.toml | 7 +- .../clippy/book/src/development/adding_lints.md | 21 +- src/tools/clippy/book/src/lint_configuration.md | 152 +- src/tools/clippy/clippy_config/Cargo.toml | 21 + src/tools/clippy/clippy_config/src/conf.rs | 791 +++++ src/tools/clippy/clippy_config/src/lib.rs | 23 + src/tools/clippy/clippy_config/src/metadata.rs | 116 + src/tools/clippy/clippy_config/src/msrvs.rs | 138 + src/tools/clippy/clippy_config/src/types.rs | 142 + src/tools/clippy/clippy_dev/src/main.rs | 1 - src/tools/clippy/clippy_dev/src/new_lint.rs | 67 +- src/tools/clippy/clippy_dev/src/update_lints.rs | 2 +- src/tools/clippy/clippy_lints/Cargo.toml | 10 +- .../clippy/clippy_lints/src/absolute_paths.rs | 4 +- .../clippy/clippy_lints/src/allow_attributes.rs | 2 +- .../clippy_lints/src/almost_complete_range.rs | 31 +- src/tools/clippy/clippy_lints/src/approx_const.rs | 6 +- .../clippy_lints/src/arc_with_non_send_sync.rs | 26 +- .../clippy/clippy_lints/src/as_conversions.rs | 9 +- .../src/assertions_on_result_states.rs | 6 +- .../clippy/clippy_lints/src/async_yields_async.rs | 10 +- src/tools/clippy/clippy_lints/src/attrs.rs | 88 +- .../clippy_lints/src/await_holding_invalid.rs | 38 +- .../clippy_lints/src/blocks_in_if_conditions.rs | 4 +- .../clippy_lints/src/bool_assert_comparison.rs | 4 +- .../clippy/clippy_lints/src/bool_to_int_with_if.rs | 31 +- src/tools/clippy/clippy_lints/src/booleans.rs | 16 +- .../clippy/clippy_lints/src/borrow_deref_ref.rs | 113 +- src/tools/clippy/clippy_lints/src/box_default.rs | 32 +- .../clippy_lints/src/cargo/common_metadata.rs | 2 +- .../clippy/clippy_lints/src/cargo/feature_name.rs | 2 +- .../src/cargo/multiple_crate_versions.rs | 2 +- .../src/cargo/wildcard_dependencies.rs | 2 +- .../clippy_lints/src/casts/as_ptr_cast_mut.rs | 17 +- .../clippy_lints/src/casts/cast_abs_to_unsigned.rs | 2 +- .../clippy/clippy_lints/src/casts/cast_lossless.rs | 2 +- .../clippy_lints/src/casts/cast_ptr_alignment.rs | 22 +- .../src/casts/cast_slice_different_sizes.rs | 2 +- .../src/casts/cast_slice_from_raw_parts.rs | 14 +- src/tools/clippy/clippy_lints/src/casts/mod.rs | 72 +- .../clippy/clippy_lints/src/casts/ptr_as_ptr.rs | 2 +- .../clippy_lints/src/casts/ptr_cast_constness.rs | 4 +- .../clippy_lints/src/casts/unnecessary_cast.rs | 17 +- .../clippy/clippy_lints/src/checked_conversions.rs | 6 +- .../clippy_lints/src/cognitive_complexity.rs | 3 +- .../clippy/clippy_lints/src/collapsible_if.rs | 4 +- .../clippy_lints/src/collection_is_never_read.rs | 4 +- .../clippy/clippy_lints/src/crate_in_macro_def.rs | 4 +- src/tools/clippy/clippy_lints/src/create_dir.rs | 4 +- .../clippy/clippy_lints/src/declared_lints.rs | 19 +- src/tools/clippy/clippy_lints/src/default.rs | 18 +- .../src/default_constructed_unit_structs.rs | 9 +- .../src/default_instead_of_iter_empty.rs | 15 +- .../clippy_lints/src/default_numeric_fallback.rs | 4 +- .../src/default_union_representation.rs | 4 +- src/tools/clippy/clippy_lints/src/dereference.rs | 67 +- .../clippy/clippy_lints/src/derivable_impls.rs | 6 +- src/tools/clippy/clippy_lints/src/derive.rs | 7 +- .../clippy/clippy_lints/src/disallowed_macros.rs | 9 +- .../clippy/clippy_lints/src/disallowed_methods.rs | 8 +- .../clippy/clippy_lints/src/disallowed_names.rs | 2 +- .../clippy_lints/src/disallowed_script_idents.rs | 2 +- .../clippy/clippy_lints/src/disallowed_types.rs | 11 +- src/tools/clippy/clippy_lints/src/doc.rs | 50 +- src/tools/clippy/clippy_lints/src/double_parens.rs | 4 +- .../clippy/clippy_lints/src/drop_forget_ref.rs | 14 +- .../clippy_lints/src/else_if_without_else.rs | 4 +- src/tools/clippy/clippy_lints/src/empty_drop.rs | 4 +- src/tools/clippy/clippy_lints/src/empty_enum.rs | 4 +- .../src/empty_structs_with_brackets.rs | 12 +- src/tools/clippy/clippy_lints/src/entry.rs | 8 +- src/tools/clippy/clippy_lints/src/enum_clike.rs | 6 +- src/tools/clippy/clippy_lints/src/enum_variants.rs | 313 -- .../clippy/clippy_lints/src/equatable_if_let.rs | 7 +- .../clippy/clippy_lints/src/error_impl_error.rs | 30 +- src/tools/clippy/clippy_lints/src/escape.rs | 6 +- src/tools/clippy/clippy_lints/src/eta_reduction.rs | 66 +- .../clippy/clippy_lints/src/excessive_bools.rs | 13 +- .../clippy/clippy_lints/src/exhaustive_items.rs | 8 +- src/tools/clippy/clippy_lints/src/exit.rs | 7 +- .../clippy/clippy_lints/src/explicit_write.rs | 47 +- .../src/extra_unused_type_parameters.rs | 32 +- .../clippy/clippy_lints/src/fallible_impl_from.rs | 4 +- src/tools/clippy/clippy_lints/src/float_literal.rs | 8 +- .../clippy_lints/src/floating_point_arithmetic.rs | 30 +- src/tools/clippy/clippy_lints/src/format.rs | 13 +- src/tools/clippy/clippy_lints/src/format_args.rs | 24 +- src/tools/clippy/clippy_lints/src/format_impl.rs | 8 +- .../clippy/clippy_lints/src/format_push_string.rs | 16 +- src/tools/clippy/clippy_lints/src/formatting.rs | 8 +- .../clippy_lints/src/four_forward_slashes.rs | 6 +- .../clippy/clippy_lints/src/from_over_into.rs | 20 +- .../clippy_lints/src/from_raw_with_void_ptr.rs | 30 +- .../src/functions/impl_trait_in_params.rs | 126 +- src/tools/clippy/clippy_lints/src/functions/mod.rs | 36 +- .../clippy/clippy_lints/src/functions/must_use.rs | 7 +- .../clippy/clippy_lints/src/functions/result.rs | 4 +- .../clippy/clippy_lints/src/future_not_send.rs | 4 +- src/tools/clippy/clippy_lints/src/if_not_else.rs | 16 +- .../clippy_lints/src/if_then_some_else_none.rs | 21 +- .../clippy_lints/src/ignored_unit_patterns.rs | 10 +- .../clippy/clippy_lints/src/implicit_hasher.rs | 11 +- .../clippy/clippy_lints/src/implicit_return.rs | 4 +- .../clippy_lints/src/implicit_saturating_add.rs | 28 +- .../clippy_lints/src/implicit_saturating_sub.rs | 4 +- .../clippy_lints/src/implied_bounds_in_impls.rs | 82 +- .../src/inconsistent_struct_constructor.rs | 6 +- .../clippy_lints/src/index_refutable_slice.rs | 6 +- .../clippy/clippy_lints/src/indexing_slicing.rs | 4 +- src/tools/clippy/clippy_lints/src/infinite_iter.rs | 2 +- src/tools/clippy/clippy_lints/src/inherent_impl.rs | 4 +- .../clippy/clippy_lints/src/inherent_to_string.rs | 8 +- .../clippy_lints/src/init_numbered_fields.rs | 2 +- .../clippy_lints/src/inline_fn_without_body.rs | 2 +- .../clippy/clippy_lints/src/instant_subtraction.rs | 16 +- src/tools/clippy/clippy_lints/src/int_plus_one.rs | 4 +- .../clippy_lints/src/invalid_upcast_comparisons.rs | 2 +- .../clippy_lints/src/item_name_repetitions.rs | 447 +++ .../clippy_lints/src/items_after_statements.rs | 4 +- .../clippy_lints/src/items_after_test_module.rs | 107 +- .../src/iter_not_returning_iterator.rs | 4 +- .../clippy_lints/src/iter_without_into_iter.rs | 295 ++ .../clippy/clippy_lints/src/large_enum_variant.rs | 6 +- src/tools/clippy/clippy_lints/src/large_futures.rs | 6 +- .../clippy/clippy_lints/src/large_stack_arrays.rs | 52 +- .../clippy/clippy_lints/src/large_stack_frames.rs | 4 +- src/tools/clippy/clippy_lints/src/len_zero.rs | 38 +- .../clippy/clippy_lints/src/let_underscore.rs | 29 +- .../clippy_lints/src/let_with_type_underscore.rs | 2 +- src/tools/clippy/clippy_lints/src/lib.rs | 103 +- src/tools/clippy/clippy_lints/src/lifetimes.rs | 18 +- .../clippy/clippy_lints/src/lines_filter_map_ok.rs | 55 +- .../clippy_lints/src/literal_representation.rs | 12 +- .../clippy_lints/src/loops/explicit_iter_loop.rs | 40 +- .../clippy/clippy_lints/src/loops/for_kv_map.rs | 12 +- .../clippy/clippy_lints/src/loops/manual_find.rs | 2 +- .../clippy_lints/src/loops/manual_flatten.rs | 2 +- src/tools/clippy/clippy_lints/src/loops/mod.rs | 85 +- .../clippy_lints/src/loops/mut_range_bound.rs | 2 +- .../clippy/clippy_lints/src/loops/never_loop.rs | 6 +- .../clippy_lints/src/loops/same_item_push.rs | 2 +- .../src/loops/unused_enumerate_index.rs | 62 + src/tools/clippy/clippy_lints/src/manual_assert.rs | 18 +- .../clippy/clippy_lints/src/manual_async_fn.rs | 8 +- src/tools/clippy/clippy_lints/src/manual_bits.rs | 10 +- src/tools/clippy/clippy_lints/src/manual_clamp.rs | 94 +- .../clippy_lints/src/manual_float_methods.rs | 92 +- .../clippy/clippy_lints/src/manual_hash_one.rs | 132 + .../clippy_lints/src/manual_is_ascii_check.rs | 44 +- .../clippy/clippy_lints/src/manual_let_else.rs | 57 +- .../clippy_lints/src/manual_main_separator_str.rs | 48 +- .../clippy_lints/src/manual_non_exhaustive.rs | 30 +- .../clippy_lints/src/manual_range_patterns.rs | 4 +- .../clippy/clippy_lints/src/manual_rem_euclid.rs | 51 +- src/tools/clippy/clippy_lints/src/manual_retain.rs | 52 +- .../src/manual_slice_size_calculation.rs | 4 +- .../clippy/clippy_lints/src/manual_string_new.rs | 33 +- src/tools/clippy/clippy_lints/src/manual_strip.rs | 6 +- src/tools/clippy/clippy_lints/src/map_unit_fn.rs | 11 +- .../clippy_lints/src/matches/collapsible_match.rs | 2 +- .../clippy_lints/src/matches/manual_filter.rs | 20 +- .../clippy_lints/src/matches/manual_utils.rs | 4 +- .../clippy_lints/src/matches/match_wild_err_arm.rs | 2 +- src/tools/clippy/clippy_lints/src/matches/mod.rs | 85 +- .../clippy_lints/src/matches/overlapping_arms.rs | 6 +- .../clippy_lints/src/matches/redundant_guards.rs | 39 +- .../clippy_lints/src/matches/single_match.rs | 5 +- src/tools/clippy/clippy_lints/src/mem_replace.rs | 14 +- .../clippy/clippy_lints/src/methods/bytecount.rs | 8 +- .../clippy/clippy_lints/src/methods/bytes_nth.rs | 11 +- .../clippy_lints/src/methods/clone_on_ref_ptr.rs | 19 +- .../src/methods/cloned_instead_of_copied.rs | 2 +- .../clippy/clippy_lints/src/methods/err_expect.rs | 2 +- .../clippy_lints/src/methods/expect_fun_call.rs | 2 +- .../clippy_lints/src/methods/filetype_is_file.rs | 3 +- .../clippy/clippy_lints/src/methods/filter_map.rs | 32 +- .../src/methods/filter_map_bool_then.rs | 2 +- .../src/methods/filter_map_identity.rs | 7 +- .../clippy_lints/src/methods/filter_map_next.rs | 2 +- .../clippy_lints/src/methods/flat_map_identity.rs | 7 +- .../clippy_lints/src/methods/flat_map_option.rs | 3 +- .../clippy_lints/src/methods/format_collect.rs | 16 +- .../src/methods/from_iter_instead_of_collect.rs | 4 +- .../clippy/clippy_lints/src/methods/get_first.rs | 42 +- .../clippy/clippy_lints/src/methods/get_unwrap.rs | 6 +- .../src/methods/inefficient_to_string.rs | 3 +- .../clippy_lints/src/methods/inspect_for_each.rs | 3 +- .../clippy_lints/src/methods/into_iter_on_ref.rs | 2 +- .../src/methods/is_digit_ascii_radix.rs | 2 +- .../clippy/clippy_lints/src/methods/iter_kv_map.rs | 15 +- .../clippy_lints/src/methods/iter_out_of_bounds.rs | 12 +- .../src/methods/iter_overeager_cloned.rs | 88 +- .../clippy_lints/src/methods/manual_try_fold.rs | 6 +- .../clippy/clippy_lints/src/methods/map_clone.rs | 2 +- .../clippy_lints/src/methods/map_identity.rs | 7 +- .../clippy_lints/src/methods/map_unwrap_or.rs | 4 +- src/tools/clippy/clippy_lints/src/methods/mod.rs | 703 +++-- .../clippy_lints/src/methods/needless_collect.rs | 51 +- .../clippy_lints/src/methods/open_options.rs | 10 +- .../src/methods/option_as_ref_deref.rs | 7 +- .../src/methods/option_map_unwrap_or.rs | 5 +- .../clippy/clippy_lints/src/methods/or_fun_call.rs | 8 +- .../clippy_lints/src/methods/path_ends_with_ext.rs | 5 +- .../src/methods/read_line_without_trim.rs | 9 +- .../src/methods/readonly_write_lock.rs | 21 +- .../clippy_lints/src/methods/search_is_some.rs | 4 +- .../clippy_lints/src/methods/seek_from_current.rs | 23 +- .../src/methods/seek_to_start_instead_of_rewind.rs | 22 +- .../clippy/clippy_lints/src/methods/str_splitn.rs | 20 +- .../src/methods/string_lit_chars_any.rs | 6 +- .../src/methods/suspicious_command_arg_space.rs | 14 +- .../clippy_lints/src/methods/type_id_on_box.rs | 4 +- .../methods/unnecessary_fallible_conversions.rs | 122 + .../clippy_lints/src/methods/unnecessary_fold.rs | 3 +- .../src/methods/unnecessary_lazy_eval.rs | 18 +- .../src/methods/unnecessary_literal_unwrap.rs | 17 +- .../src/methods/unnecessary_to_owned.rs | 114 +- .../clippy_lints/src/methods/waker_clone_wake.rs | 32 + .../src/methods/wrong_self_convention.rs | 2 +- .../clippy/clippy_lints/src/min_ident_chars.rs | 8 +- src/tools/clippy/clippy_lints/src/misc.rs | 10 +- .../clippy/clippy_lints/src/misc_early/mod.rs | 34 +- .../src/misc_early/unneeded_wildcard_pattern.rs | 2 +- .../src/mismatching_type_param_order.rs | 4 +- .../clippy_lints/src/missing_assert_message.rs | 8 +- .../src/missing_asserts_for_indexing.rs | 27 +- .../clippy_lints/src/missing_const_for_fn.rs | 8 +- src/tools/clippy/clippy_lints/src/missing_doc.rs | 5 +- .../src/missing_enforced_import_rename.rs | 9 +- .../clippy_lints/src/missing_fields_in_debug.rs | 16 +- .../clippy/clippy_lints/src/missing_inline.rs | 5 +- .../clippy_lints/src/missing_trait_methods.rs | 4 +- .../src/mixed_read_write_in_expression.rs | 24 +- .../clippy/clippy_lints/src/multi_assignments.rs | 4 +- .../src/multiple_unsafe_ops_per_block.rs | 11 +- src/tools/clippy/clippy_lints/src/mut_key.rs | 4 +- src/tools/clippy/clippy_lints/src/mut_mut.rs | 2 +- src/tools/clippy/clippy_lints/src/mut_reference.rs | 6 +- src/tools/clippy/clippy_lints/src/mutex_atomic.rs | 8 +- .../src/needless_arbitrary_self_type.rs | 4 +- src/tools/clippy/clippy_lints/src/needless_bool.rs | 12 +- .../clippy_lints/src/needless_borrowed_ref.rs | 4 +- .../src/needless_borrows_for_generic_args.rs | 43 +- .../clippy/clippy_lints/src/needless_continue.rs | 14 +- src/tools/clippy/clippy_lints/src/needless_else.rs | 4 +- .../clippy/clippy_lints/src/needless_for_each.rs | 7 +- src/tools/clippy/clippy_lints/src/needless_if.rs | 2 +- .../clippy/clippy_lints/src/needless_late_init.rs | 4 +- .../src/needless_parens_on_range_literals.rs | 10 +- .../clippy_lints/src/needless_pass_by_ref_mut.rs | 121 +- .../clippy_lints/src/needless_pass_by_value.rs | 8 +- .../clippy_lints/src/needless_question_mark.rs | 10 +- .../clippy/clippy_lints/src/needless_update.rs | 2 +- .../clippy_lints/src/neg_cmp_op_on_partial_ord.rs | 4 +- src/tools/clippy/clippy_lints/src/neg_multiply.rs | 2 +- src/tools/clippy/clippy_lints/src/no_effect.rs | 13 +- .../clippy_lints/src/no_mangle_with_rust_abi.rs | 7 +- .../clippy/clippy_lints/src/non_canonical_impls.rs | 38 +- .../clippy/clippy_lints/src/non_copy_const.rs | 323 +- .../clippy_lints/src/non_expressive_names.rs | 5 +- .../clippy_lints/src/non_octal_unix_permissions.rs | 14 +- .../clippy_lints/src/non_send_fields_in_send_ty.rs | 4 +- .../clippy_lints/src/nonstandard_macro_braces.rs | 89 +- src/tools/clippy/clippy_lints/src/octal_escapes.rs | 4 +- .../clippy_lints/src/only_used_in_recursion.rs | 22 +- .../src/operators/arithmetic_side_effects.rs | 45 +- .../clippy/clippy_lints/src/operators/bit_mask.rs | 2 +- .../clippy/clippy_lints/src/operators/cmp_owned.rs | 4 +- .../src/operators/const_comparisons.rs | 3 +- .../src/operators/double_comparison.rs | 2 +- .../clippy/clippy_lints/src/operators/eq_op.rs | 19 +- .../clippy_lints/src/operators/identity_op.rs | 2 +- src/tools/clippy/clippy_lints/src/operators/mod.rs | 76 +- .../src/operators/numeric_arithmetic.rs | 2 +- .../clippy/clippy_lints/src/option_env_unwrap.rs | 23 +- .../clippy/clippy_lints/src/option_if_let_else.rs | 14 +- .../clippy_lints/src/overflow_check_conditional.rs | 2 +- .../clippy/clippy_lints/src/panic_in_result_fn.rs | 4 +- .../clippy/clippy_lints/src/panic_unimplemented.rs | 13 +- .../clippy/clippy_lints/src/partial_pub_fields.rs | 4 +- .../clippy/clippy_lints/src/partialeq_ne_impl.rs | 2 +- .../clippy/clippy_lints/src/partialeq_to_none.rs | 4 +- .../clippy_lints/src/pass_by_ref_or_value.rs | 13 +- .../clippy_lints/src/pattern_type_mismatch.rs | 2 +- .../src/permissions_set_readonly_false.rs | 16 +- src/tools/clippy/clippy_lints/src/ptr.rs | 274 +- .../clippy_lints/src/ptr_offset_with_cast.rs | 4 +- src/tools/clippy/clippy_lints/src/pub_use.rs | 27 +- src/tools/clippy/clippy_lints/src/question_mark.rs | 24 +- src/tools/clippy/clippy_lints/src/ranges.rs | 19 +- src/tools/clippy/clippy_lints/src/raw_strings.rs | 38 +- .../clippy_lints/src/rc_clone_in_vec_init.rs | 13 +- .../clippy/clippy_lints/src/read_zero_byte_vec.rs | 29 +- .../clippy_lints/src/redundant_async_block.rs | 30 +- .../clippy/clippy_lints/src/redundant_clone.rs | 9 +- .../clippy_lints/src/redundant_closure_call.rs | 29 +- .../clippy/clippy_lints/src/redundant_else.rs | 4 +- .../clippy_lints/src/redundant_field_names.rs | 4 +- .../clippy/clippy_lints/src/redundant_locals.rs | 42 +- .../clippy/clippy_lints/src/redundant_pub_crate.rs | 4 +- .../clippy/clippy_lints/src/redundant_slicing.rs | 4 +- .../clippy_lints/src/redundant_static_lifetimes.rs | 2 +- .../clippy_lints/src/redundant_type_annotations.rs | 24 +- src/tools/clippy/clippy_lints/src/ref_patterns.rs | 6 +- src/tools/clippy/clippy_lints/src/reference.rs | 2 +- src/tools/clippy/clippy_lints/src/regex.rs | 2 +- .../src/reserve_after_initialization.rs | 43 +- .../clippy_lints/src/return_self_not_must_use.rs | 4 +- src/tools/clippy/clippy_lints/src/returns.rs | 15 +- .../clippy/clippy_lints/src/same_name_method.rs | 16 +- .../clippy/clippy_lints/src/semicolon_block.rs | 8 +- .../src/semicolon_if_nothing_returned.rs | 4 +- src/tools/clippy/clippy_lints/src/shadow.rs | 12 +- .../src/significant_drop_tightening.rs | 33 +- .../clippy/clippy_lints/src/single_call_fn.rs | 6 +- .../clippy_lints/src/single_char_lifetime_names.rs | 4 +- .../clippy_lints/src/single_range_in_vec_init.rs | 4 +- .../clippy_lints/src/size_of_in_element_count.rs | 22 +- src/tools/clippy/clippy_lints/src/size_of_ref.rs | 4 +- .../clippy_lints/src/slow_vector_initialization.rs | 6 +- .../clippy/clippy_lints/src/std_instead_of_core.rs | 39 +- src/tools/clippy/clippy_lints/src/strings.rs | 26 +- .../clippy_lints/src/suspicious_doc_comments.rs | 4 +- .../src/suspicious_operation_groupings.rs | 6 +- .../clippy_lints/src/suspicious_xor_used_as_pow.rs | 35 +- src/tools/clippy/clippy_lints/src/swap.rs | 10 +- .../clippy/clippy_lints/src/swap_ptr_to_ref.rs | 24 +- .../clippy_lints/src/tabs_in_doc_comments.rs | 6 +- .../clippy_lints/src/temporary_assignment.rs | 2 +- .../clippy_lints/src/tests_outside_test_module.rs | 4 +- .../clippy/clippy_lints/src/to_digit_is_some.rs | 4 +- .../clippy_lints/src/trailing_empty_array.rs | 4 +- src/tools/clippy/clippy_lints/src/trait_bounds.rs | 18 +- src/tools/clippy/clippy_lints/src/transmute/mod.rs | 38 +- .../src/transmute/transmute_ptr_to_ref.rs | 2 +- .../src/transmute/transmute_undefined_repr.rs | 22 +- .../clippy_lints/src/transmute/transmuting_null.rs | 12 +- .../clippy_lints/src/tuple_array_conversions.rs | 2 +- src/tools/clippy/clippy_lints/src/types/mod.rs | 20 +- src/tools/clippy/clippy_lints/src/types/utils.rs | 2 +- .../clippy_lints/src/undocumented_unsafe_blocks.rs | 24 +- src/tools/clippy/clippy_lints/src/unicode.rs | 6 +- src/tools/clippy/clippy_lints/src/uninit_vec.rs | 2 +- .../clippy_lints/src/unit_return_expecting_ord.rs | 2 +- .../clippy_lints/src/unit_types/let_unit_value.rs | 11 +- .../clippy/clippy_lints/src/unit_types/mod.rs | 8 +- .../clippy/clippy_lints/src/unnamed_address.rs | 6 +- .../clippy_lints/src/unnecessary_box_returns.rs | 4 +- .../src/unnecessary_map_on_constructor.rs | 38 +- .../src/unnecessary_owned_empty_strings.rs | 4 +- .../clippy_lints/src/unnecessary_self_imports.rs | 4 +- .../src/unnecessary_struct_initialization.rs | 20 +- .../clippy/clippy_lints/src/unnecessary_wraps.rs | 4 +- .../clippy_lints/src/unnested_or_patterns.rs | 6 +- .../clippy_lints/src/unsafe_removed_from_name.rs | 2 +- src/tools/clippy/clippy_lints/src/unused_async.rs | 6 +- .../clippy/clippy_lints/src/unused_peekable.rs | 24 +- .../clippy/clippy_lints/src/unused_rounding.rs | 23 +- src/tools/clippy/clippy_lints/src/unused_unit.rs | 7 +- src/tools/clippy/clippy_lints/src/unwrap.rs | 9 +- .../clippy/clippy_lints/src/unwrap_in_result.rs | 4 +- .../clippy/clippy_lints/src/upper_case_acronyms.rs | 4 +- src/tools/clippy/clippy_lints/src/use_self.rs | 9 +- .../clippy/clippy_lints/src/useless_conversion.rs | 73 +- src/tools/clippy/clippy_lints/src/utils/author.rs | 39 +- src/tools/clippy/clippy_lints/src/utils/conf.rs | 746 ----- .../clippy/clippy_lints/src/utils/dump_hir.rs | 12 +- .../src/utils/format_args_collector.rs | 15 +- .../clippy_lints/src/utils/internal_lints.rs | 2 +- .../almost_standard_lint_formulation.rs | 2 +- .../utils/internal_lints/clippy_lints_internal.rs | 49 - .../src/utils/internal_lints/if_chain_style.rs | 2 +- .../internal_lints/interning_defined_symbol.rs | 23 +- .../utils/internal_lints/lint_without_lint_pass.rs | 5 +- .../src/utils/internal_lints/metadata_collector.rs | 30 +- .../utils/internal_lints/unnecessary_def_path.rs | 3 +- .../internal_lints/unsorted_clippy_utils_paths.rs | 49 + src/tools/clippy/clippy_lints/src/utils/mod.rs | 143 - src/tools/clippy/clippy_lints/src/vec.rs | 14 +- .../clippy/clippy_lints/src/vec_init_then_push.rs | 10 +- src/tools/clippy/clippy_lints/src/visibility.rs | 4 +- .../clippy/clippy_lints/src/wildcard_imports.rs | 5 +- src/tools/clippy/clippy_lints/src/write.rs | 181 +- src/tools/clippy/clippy_lints/src/zero_div_zero.rs | 4 +- .../clippy_lints/src/zero_sized_map_values.rs | 4 +- src/tools/clippy/clippy_utils/Cargo.toml | 6 +- src/tools/clippy/clippy_utils/src/ast_utils.rs | 2 +- .../clippy/clippy_utils/src/check_proc_macro.rs | 78 +- src/tools/clippy/clippy_utils/src/consts.rs | 95 +- src/tools/clippy/clippy_utils/src/diagnostics.rs | 4 +- src/tools/clippy/clippy_utils/src/higher.rs | 4 +- src/tools/clippy/clippy_utils/src/hir_utils.rs | 8 +- src/tools/clippy/clippy_utils/src/lib.rs | 263 +- src/tools/clippy/clippy_utils/src/macros.rs | 18 +- src/tools/clippy/clippy_utils/src/msrvs.rs | 148 - src/tools/clippy/clippy_utils/src/paths.rs | 70 +- .../clippy_utils/src/qualify_min_const_fn.rs | 29 +- src/tools/clippy/clippy_utils/src/source.rs | 8 +- src/tools/clippy/clippy_utils/src/str_utils.rs | 67 +- src/tools/clippy/clippy_utils/src/sugg.rs | 9 +- src/tools/clippy/clippy_utils/src/ty.rs | 44 +- .../clippy_utils/src/ty/type_certainty/mod.rs | 4 +- src/tools/clippy/clippy_utils/src/visitors.rs | 44 + src/tools/clippy/declare_clippy_lint/Cargo.toml | 2 +- src/tools/clippy/declare_clippy_lint/src/lib.rs | 28 +- src/tools/clippy/lintcheck/src/main.rs | 2 +- src/tools/clippy/rust-toolchain | 2 +- src/tools/clippy/src/driver.rs | 73 +- src/tools/clippy/src/main.rs | 63 +- src/tools/clippy/tests/compile-test.rs | 89 +- src/tools/clippy/tests/dogfood.rs | 12 +- .../tests/ui-internal/invalid_msrv_attr_impl.fixed | 2 +- .../tests/ui-internal/invalid_msrv_attr_impl.rs | 2 +- .../unnecessary_def_path_hardcoded_path.rs | 4 +- .../unnecessary_def_path_hardcoded_path.stderr | 4 +- .../borrow_interior_mutable_const/clippy.toml | 1 + .../borrow_interior_mutable_const/ignore.rs | 37 + .../declare_interior_mutable_const/clippy.toml | 1 + .../declare_interior_mutable_const/ignore.rs | 46 + .../tests/ui-toml/enum_variant_names/clippy.toml | 1 - .../enum_variant_names/enum_variant_names.rs | 16 - .../enum_variant_names/enum_variant_names.stderr | 18 - .../ui-toml/enum_variants_threshold0/clippy.toml | 1 - .../enum_variants_name_threshold.rs | 3 - .../tests/ui-toml/impl_trait_in_params/clippy.toml | 1 + .../impl_trait_in_params/impl_trait_in_params.rs | 16 + .../impl_trait_in_params.stderr | 15 + .../invalid_min_rust_version.rs | 2 +- .../invalid_min_rust_version.stderr | 6 +- .../item_name_repetitions/threshold0/clippy.toml | 2 + .../threshold0/item_name_repetitions.rs | 5 + .../item_name_repetitions/threshold5/clippy.toml | 2 + .../threshold5/item_name_repetitions.rs | 32 + .../threshold5/item_name_repetitions.stderr | 34 + .../pub_crate_missing_doc.rs | 1 + .../pub_crate_missing_doc.stderr | 14 +- .../toml_unknown_key/conf_unknown_key.stderr | 2 + .../too_many_arguments/too_many_arguments.stderr | 2 +- src/tools/clippy/tests/ui/author/blocks.stdout | 4 +- .../clippy/tests/ui/author/macro_in_closure.rs | 5 + .../clippy/tests/ui/author/macro_in_closure.stdout | 39 + src/tools/clippy/tests/ui/author/macro_in_loop.rs | 8 + .../clippy/tests/ui/author/macro_in_loop.stdout | 48 + .../clippy/tests/ui/auxiliary/proc_macro_derive.rs | 28 + src/tools/clippy/tests/ui/auxiliary/proc_macros.rs | 10 +- .../clippy/tests/ui/bool_to_int_with_if.fixed | 8 +- src/tools/clippy/tests/ui/bool_to_int_with_if.rs | 8 +- .../clippy/tests/ui/bool_to_int_with_if.stderr | 2 +- .../clippy/tests/ui/builtin_type_shadow.stderr | 2 +- .../clippy/tests/ui/comparison_to_empty.fixed | 8 +- src/tools/clippy/tests/ui/comparison_to_empty.rs | 8 +- .../clippy/tests/ui/comparison_to_empty.stderr | 8 +- src/tools/clippy/tests/ui/crashes/ice-10645.stderr | 4 +- src/tools/clippy/tests/ui/crashes/ice-11230.rs | 6 + src/tools/clippy/tests/ui/crashes/ice-11755.rs | 5 + src/tools/clippy/tests/ui/crashes/ice-5238.rs | 2 +- src/tools/clippy/tests/ui/crashes/ice-6252.stderr | 2 +- src/tools/clippy/tests/ui/doc/doc-fixable.fixed | 3 + src/tools/clippy/tests/ui/doc/doc-fixable.rs | 3 + src/tools/clippy/tests/ui/doc_unsafe.rs | 2 +- src/tools/clippy/tests/ui/enum_glob_use.fixed | 1 + src/tools/clippy/tests/ui/enum_glob_use.rs | 1 + src/tools/clippy/tests/ui/enum_variants.rs | 17 + src/tools/clippy/tests/ui/enum_variants.stderr | 14 +- .../clippy/tests/ui/floating_point_mul_add.fixed | 3 + .../clippy/tests/ui/floating_point_mul_add.rs | 3 + .../clippy/tests/ui/floating_point_mul_add.stderr | 8 +- src/tools/clippy/tests/ui/functions.stderr | 6 +- src/tools/clippy/tests/ui/future_not_send.stderr | 37 +- src/tools/clippy/tests/ui/get_first.fixed | 18 +- src/tools/clippy/tests/ui/get_first.rs | 18 +- src/tools/clippy/tests/ui/get_first.stderr | 24 +- src/tools/clippy/tests/ui/if_not_else_bittest.rs | 11 + .../clippy/tests/ui/ignored_unit_patterns.fixed | 31 + src/tools/clippy/tests/ui/ignored_unit_patterns.rs | 31 + .../clippy/tests/ui/ignored_unit_patterns.stderr | 36 +- src/tools/clippy/tests/ui/impl_trait_in_params.rs | 35 +- .../clippy/tests/ui/impl_trait_in_params.stderr | 30 +- .../clippy/tests/ui/implied_bounds_in_impls.fixed | 1 - .../clippy/tests/ui/implied_bounds_in_impls.rs | 1 - .../clippy/tests/ui/implied_bounds_in_impls.stderr | 32 +- .../clippy/tests/ui/into_iter_without_iter.rs | 148 + .../clippy/tests/ui/into_iter_without_iter.stderr | 114 + .../items_after_test_module/after_proc_macros.rs | 11 + .../items_after_test_module/auxiliary/submodule.rs | 4 + .../ui/items_after_test_module/block_module.rs | 23 - .../ui/items_after_test_module/block_module.stderr | 2 - .../ui/items_after_test_module/in_submodule.rs | 8 + .../ui/items_after_test_module/in_submodule.stderr | 14 + .../ui/items_after_test_module/multiple_modules.rs | 11 + .../ui/items_after_test_module/root_module.fixed | 22 + .../ui/items_after_test_module/root_module.rs | 22 + .../ui/items_after_test_module/root_module.stderr | 20 + .../clippy/tests/ui/iter_without_into_iter.rs | 124 + .../clippy/tests/ui/iter_without_into_iter.stderr | 150 + src/tools/clippy/tests/ui/large_futures.fixed | 2 +- src/tools/clippy/tests/ui/large_futures.rs | 2 +- src/tools/clippy/tests/ui/let_and_return.fixed | 21 +- src/tools/clippy/tests/ui/let_and_return.rs | 19 + src/tools/clippy/tests/ui/let_and_return.stderr | 26 +- src/tools/clippy/tests/ui/manual_filter.rs | 4 +- src/tools/clippy/tests/ui/manual_filter.stderr | 6 +- src/tools/clippy/tests/ui/manual_filter_map.fixed | 1 + src/tools/clippy/tests/ui/manual_filter_map.rs | 1 + src/tools/clippy/tests/ui/manual_filter_map.stderr | 76 +- src/tools/clippy/tests/ui/manual_find_map.fixed | 1 + src/tools/clippy/tests/ui/manual_find_map.rs | 1 + src/tools/clippy/tests/ui/manual_find_map.stderr | 78 +- src/tools/clippy/tests/ui/manual_hash_one.fixed | 89 + src/tools/clippy/tests/ui/manual_hash_one.rs | 89 + src/tools/clippy/tests/ui/manual_hash_one.stderr | 56 + .../clippy/tests/ui/manual_is_ascii_check.fixed | 4 + src/tools/clippy/tests/ui/manual_is_ascii_check.rs | 4 + .../clippy/tests/ui/manual_is_ascii_check.stderr | 22 +- src/tools/clippy/tests/ui/manual_let_else.rs | 4 +- src/tools/clippy/tests/ui/manual_let_else.stderr | 48 +- .../clippy/tests/ui/manual_let_else_match.fixed | 4 + src/tools/clippy/tests/ui/manual_let_else_match.rs | 8 + .../clippy/tests/ui/manual_let_else_match.stderr | 12 +- .../clippy/tests/ui/manual_map_option_2.fixed | 4 +- src/tools/clippy/tests/ui/manual_map_option_2.rs | 4 +- .../clippy/tests/ui/manual_map_option_2.stderr | 19 +- .../clippy/tests/ui/manual_non_exhaustive_enum.rs | 3 +- .../tests/ui/manual_non_exhaustive_enum.stderr | 20 +- src/tools/clippy/tests/ui/manual_string_new.fixed | 1 + src/tools/clippy/tests/ui/manual_string_new.rs | 1 + src/tools/clippy/tests/ui/manual_string_new.stderr | 18 +- src/tools/clippy/tests/ui/map_identity.fixed | 27 + src/tools/clippy/tests/ui/map_identity.rs | 29 + src/tools/clippy/tests/ui/map_identity.stderr | 29 +- .../tests/ui/match_same_arms_non_exhaustive.rs | 22 +- .../tests/ui/match_same_arms_non_exhaustive.stderr | 16 +- src/tools/clippy/tests/ui/min_ident_chars.rs | 1 + src/tools/clippy/tests/ui/min_ident_chars.stderr | 58 +- .../tests/ui/min_rust_version_invalid_attr.rs | 5 +- .../tests/ui/min_rust_version_invalid_attr.stderr | 24 +- src/tools/clippy/tests/ui/misnamed_getters.fixed | 1 + src/tools/clippy/tests/ui/misnamed_getters.rs | 1 + src/tools/clippy/tests/ui/misnamed_getters.stderr | 36 +- .../ui/missing_const_for_fn/auxiliary/helper.rs | 4 +- .../tests/ui/multiple_unsafe_ops_per_block.rs | 7 + src/tools/clippy/tests/ui/must_use_unit.stderr | 4 +- .../clippy/tests/ui/needless_bool/fixable.fixed | 3 +- src/tools/clippy/tests/ui/needless_bool/fixable.rs | 3 +- .../clippy/tests/ui/needless_bool/fixable.stderr | 42 +- src/tools/clippy/tests/ui/needless_if.fixed | 16 +- src/tools/clippy/tests/ui/needless_if.rs | 16 +- src/tools/clippy/tests/ui/needless_if.stderr | 21 +- src/tools/clippy/tests/ui/needless_late_init.fixed | 4 +- src/tools/clippy/tests/ui/needless_late_init.rs | 4 +- .../clippy/tests/ui/needless_pass_by_ref_mut.rs | 45 + .../tests/ui/needless_pass_by_ref_mut.stderr | 62 +- .../clippy/tests/ui/needless_raw_string.fixed | 4 + src/tools/clippy/tests/ui/needless_raw_string.rs | 4 + .../clippy/tests/ui/needless_raw_string.stderr | 46 +- .../tests/ui/needless_raw_string_hashes.stderr | 30 +- src/tools/clippy/tests/ui/option_if_let_else.fixed | 17 +- src/tools/clippy/tests/ui/option_if_let_else.rs | 19 +- .../clippy/tests/ui/option_if_let_else.stderr | 67 +- src/tools/clippy/tests/ui/print_literal.fixed | 18 +- src/tools/clippy/tests/ui/print_literal.rs | 18 +- src/tools/clippy/tests/ui/print_literal.stderr | 124 +- src/tools/clippy/tests/ui/redundant_guards.fixed | 2 +- src/tools/clippy/tests/ui/redundant_guards.rs | 2 +- src/tools/clippy/tests/ui/redundant_locals.rs | 45 + src/tools/clippy/tests/ui/redundant_locals.stderr | 139 +- .../ui/redundant_pattern_matching_option.fixed | 13 +- .../tests/ui/redundant_pattern_matching_option.rs | 13 +- .../ui/redundant_pattern_matching_option.stderr | 56 +- .../tests/ui/rest_pat_in_fully_bound_structs.rs | 1 + .../ui/rest_pat_in_fully_bound_structs.stderr | 6 +- .../clippy/tests/ui/std_instead_of_core.fixed | 11 + src/tools/clippy/tests/ui/std_instead_of_core.rs | 11 + .../clippy/tests/ui/std_instead_of_core.stderr | 22 +- src/tools/clippy/tests/ui/struct_fields.rs | 331 ++ src/tools/clippy/tests/ui/struct_fields.stderr | 265 ++ .../ui/unnecessary_fallible_conversions.fixed | 6 + .../tests/ui/unnecessary_fallible_conversions.rs | 6 + .../ui/unnecessary_fallible_conversions.stderr | 17 + .../unnecessary_fallible_conversions_unfixable.rs | 43 + ...necessary_fallible_conversions_unfixable.stderr | 41 + .../clippy/tests/ui/unnecessary_lazy_eval.fixed | 3 + src/tools/clippy/tests/ui/unnecessary_lazy_eval.rs | 3 + .../clippy/tests/ui/unnecessary_lazy_eval.stderr | 94 +- .../tests/ui/unnecessary_lazy_eval_unfixable.rs | 5 + .../ui/unnecessary_lazy_eval_unfixable.stderr | 10 +- .../tests/ui/unnecessary_literal_unwrap.fixed | 4 +- .../tests/ui/unnecessary_literal_unwrap.stderr | 4 +- .../clippy/tests/ui/unnecessary_to_owned.fixed | 23 +- src/tools/clippy/tests/ui/unnecessary_to_owned.rs | 23 +- .../clippy/tests/ui/unnecessary_to_owned.stderr | 176 +- .../clippy/tests/ui/unnecessary_unsafety_doc.rs | 2 +- src/tools/clippy/tests/ui/unused_async.rs | 1 - src/tools/clippy/tests/ui/unused_async.stderr | 10 +- .../clippy/tests/ui/unused_enumerate_index.fixed | 58 + .../clippy/tests/ui/unused_enumerate_index.rs | 58 + .../clippy/tests/ui/unused_enumerate_index.stderr | 26 + .../clippy/tests/ui/useless_conversion_try.rs | 2 +- src/tools/clippy/tests/ui/waker_clone_wake.fixed | 29 + src/tools/clippy/tests/ui/waker_clone_wake.rs | 29 + src/tools/clippy/tests/ui/waker_clone_wake.stderr | 17 + src/tools/clippy/tests/ui/wildcard_imports.fixed | 28 + src/tools/clippy/tests/ui/wildcard_imports.rs | 28 + src/tools/clippy/tests/ui/wildcard_imports.stderr | 38 +- .../ui/wildcard_imports_2021.edition2018.fixed | 28 + .../ui/wildcard_imports_2021.edition2018.stderr | 38 +- .../ui/wildcard_imports_2021.edition2021.fixed | 28 + .../ui/wildcard_imports_2021.edition2021.stderr | 38 +- src/tools/clippy/tests/ui/wildcard_imports_2021.rs | 28 + src/tools/clippy/tests/ui/write_literal.fixed | 14 +- src/tools/clippy/tests/ui/write_literal.rs | 14 +- src/tools/clippy/tests/ui/write_literal.stderr | 74 +- src/tools/clippy/tests/ui/write_literal_2.rs | 16 +- src/tools/clippy/tests/ui/write_literal_2.stderr | 82 +- src/tools/clippy/tests/versioncheck.rs | 1 + src/tools/clippy/triagebot.toml | 2 + src/tools/compiletest/Cargo.toml | 1 + src/tools/compiletest/src/common.rs | 23 +- src/tools/compiletest/src/header.rs | 14 +- src/tools/compiletest/src/header/needs.rs | 4 +- src/tools/compiletest/src/header/tests.rs | 2 + src/tools/compiletest/src/lib.rs | 12 +- src/tools/compiletest/src/runtest.rs | 145 +- src/tools/lld-wrapper/src/main.rs | 4 +- src/tools/miropt-test-tools/src/lib.rs | 27 +- src/tools/opt-dist/Cargo.toml | 1 + src/tools/opt-dist/src/bolt.rs | 14 +- src/tools/opt-dist/src/exec.rs | 9 +- src/tools/opt-dist/src/main.rs | 76 +- src/tools/opt-dist/src/tests.rs | 4 +- src/tools/opt-dist/src/training.rs | 53 +- src/tools/opt-dist/src/utils/artifact_size.rs | 61 + src/tools/opt-dist/src/utils/mod.rs | 51 +- .../crates/hir-def/src/attr/builtin.rs | 18 - .../rust-analyzer/crates/parser/src/syntax_kind.rs | 2 +- .../crates/parser/src/syntax_kind/generated.rs | 1 - .../crates/syntax/src/tests/sourcegen_ast.rs | 1 - src/tools/rust-installer/Cargo.toml | 1 - src/tools/rustdoc-gui/tester.js | 9 +- src/tools/rustfmt/.github/workflows/check_diff.yml | 2 +- src/tools/rustfmt/CHANGELOG.md | 80 +- src/tools/rustfmt/CODE_OF_CONDUCT.md | 2 +- src/tools/rustfmt/Cargo.lock | 445 ++- src/tools/rustfmt/Cargo.toml | 17 +- src/tools/rustfmt/Contributing.md | 12 +- src/tools/rustfmt/README.md | 2 +- src/tools/rustfmt/ci/build_and_test.bat | 6 +- src/tools/rustfmt/ci/build_and_test.sh | 6 +- src/tools/rustfmt/ci/check_diff.sh | 13 +- src/tools/rustfmt/config_proc_macro/Cargo.toml | 2 +- src/tools/rustfmt/rust-toolchain | 2 +- src/tools/rustfmt/src/attr.rs | 8 +- src/tools/rustfmt/src/attr/doc_comment.rs | 6 +- src/tools/rustfmt/src/bin/main.rs | 26 +- src/tools/rustfmt/src/cargo-fmt/main.rs | 30 +- src/tools/rustfmt/src/chains.rs | 91 +- src/tools/rustfmt/src/closures.rs | 15 +- src/tools/rustfmt/src/comment.rs | 74 +- src/tools/rustfmt/src/config/config_type.rs | 10 +- src/tools/rustfmt/src/config/file_lines.rs | 2 +- src/tools/rustfmt/src/config/macro_names.rs | 16 +- src/tools/rustfmt/src/config/mod.rs | 15 +- src/tools/rustfmt/src/config/options.rs | 4 +- src/tools/rustfmt/src/emitter.rs | 2 +- src/tools/rustfmt/src/emitter/checkstyle.rs | 4 +- src/tools/rustfmt/src/emitter/checkstyle/xml.rs | 2 +- src/tools/rustfmt/src/emitter/diff.rs | 6 +- src/tools/rustfmt/src/emitter/json.rs | 6 +- src/tools/rustfmt/src/emitter/stdout.rs | 4 +- src/tools/rustfmt/src/expr.rs | 139 +- src/tools/rustfmt/src/format-diff/main.rs | 15 +- src/tools/rustfmt/src/formatting.rs | 2 +- src/tools/rustfmt/src/git-rustfmt/main.rs | 11 +- src/tools/rustfmt/src/imports.rs | 26 +- src/tools/rustfmt/src/items.rs | 380 +-- src/tools/rustfmt/src/lib.rs | 11 +- src/tools/rustfmt/src/lists.rs | 2 +- src/tools/rustfmt/src/macros.rs | 84 +- src/tools/rustfmt/src/matches.rs | 25 +- src/tools/rustfmt/src/pairs.rs | 45 +- src/tools/rustfmt/src/parse/session.rs | 3 +- src/tools/rustfmt/src/patterns.rs | 8 +- src/tools/rustfmt/src/rustfmt_diff.rs | 17 +- src/tools/rustfmt/src/skip.rs | 2 +- src/tools/rustfmt/src/source_file.rs | 2 +- src/tools/rustfmt/src/stmt.rs | 34 +- src/tools/rustfmt/src/string.rs | 4 +- .../rustfmt/src/test/configuration_snippet.rs | 10 +- src/tools/rustfmt/src/test/mod.rs | 24 +- src/tools/rustfmt/src/types.rs | 46 +- src/tools/rustfmt/src/utils.rs | 43 +- src/tools/rustfmt/tests/cargo-fmt/main.rs | 2 +- src/tools/rustfmt/tests/config/issue-5816.toml | 1 + src/tools/rustfmt/tests/rustfmt/main.rs | 6 +- .../rustfmt/tests/source/immovable_coroutines.rs | 7 + .../rustfmt/tests/source/immovable_generators.rs | 7 - src/tools/rustfmt/tests/source/issue-3984.rs | 12 + src/tools/rustfmt/tests/source/issue-4808.rs | 13 + src/tools/rustfmt/tests/source/issue-5655/one.rs | 9 + src/tools/rustfmt/tests/source/issue-5655/two.rs | 9 + src/tools/rustfmt/tests/source/issue-5791.rs | 3 + src/tools/rustfmt/tests/source/issue-5835.rs | 8 + .../rustfmt/tests/source/issue-5852/default.rs | 104 + .../rustfmt/tests/source/issue-5852/horizontal.rs | 106 + .../tests/source/issue-5852/horizontal_vertical.rs | 106 + .../tests/source/issue-5852/issue_example.rs | 8 + src/tools/rustfmt/tests/source/issue-5852/split.rs | 111 + .../rustfmt/tests/source/issue-5852/vertical.rs | 106 + src/tools/rustfmt/tests/source/issue-5935.rs | 9 + src/tools/rustfmt/tests/source/issue_5721.rs | 8 + src/tools/rustfmt/tests/source/issue_5730.rs | 3 + src/tools/rustfmt/tests/source/issue_5735.rs | 6 + src/tools/rustfmt/tests/source/issue_5882.rs | 7 + src/tools/rustfmt/tests/source/let_chains.rs | 121 + src/tools/rustfmt/tests/source/let_else.rs | 20 + src/tools/rustfmt/tests/source/let_else_v2.rs | 56 + src/tools/rustfmt/tests/source/match.rs | 3 + .../rustfmt/tests/source/non-lifetime-binders.rs | 10 + .../source/skip_macro_invocations/config_file.rs | 9 + src/tools/rustfmt/tests/target/extern-rust.rs | 1 + .../rustfmt/tests/target/immovable_coroutines.rs | 7 + .../rustfmt/tests/target/immovable_generators.rs | 7 - src/tools/rustfmt/tests/target/issue-3984.rs | 12 + src/tools/rustfmt/tests/target/issue-4808.rs | 13 + src/tools/rustfmt/tests/target/issue-5568.rs | 14 + src/tools/rustfmt/tests/target/issue-5655/one.rs | 9 + src/tools/rustfmt/tests/target/issue-5655/two.rs | 8 + src/tools/rustfmt/tests/target/issue-5791.rs | 3 + .../target/issue-5797/retain_trailing_semicolon.rs | 7 + src/tools/rustfmt/tests/target/issue-5835.rs | 8 + .../rustfmt/tests/target/issue-5852/default.rs | 97 + .../rustfmt/tests/target/issue-5852/horizontal.rs | 99 + .../tests/target/issue-5852/horizontal_vertical.rs | 99 + .../tests/target/issue-5852/issue_example.rs | 8 + src/tools/rustfmt/tests/target/issue-5852/split.rs | 102 + .../rustfmt/tests/target/issue-5852/vertical.rs | 105 + src/tools/rustfmt/tests/target/issue-5871.rs | 8 + src/tools/rustfmt/tests/target/issue-5935.rs | 8 + src/tools/rustfmt/tests/target/issue_4110.rs | 2 +- src/tools/rustfmt/tests/target/issue_5533.rs | 6 + src/tools/rustfmt/tests/target/issue_5542.rs | 10 + src/tools/rustfmt/tests/target/issue_5676.rs | 8 + src/tools/rustfmt/tests/target/issue_5721.rs | 10 + src/tools/rustfmt/tests/target/issue_5730.rs | 3 + src/tools/rustfmt/tests/target/issue_5735.rs | 6 + src/tools/rustfmt/tests/target/issue_5882.rs | 7 + src/tools/rustfmt/tests/target/issue_5907.rs | 6 + src/tools/rustfmt/tests/target/let_chains.rs | 129 + src/tools/rustfmt/tests/target/let_else.rs | 31 + src/tools/rustfmt/tests/target/let_else_v2.rs | 73 + src/tools/rustfmt/tests/target/match.rs | 8 + .../rustfmt/tests/target/non-lifetime-binders.rs | 10 + .../target/skip_macro_invocations/config_file.rs | 7 + src/tools/rustfmt/triagebot.toml | 3 + src/tools/suggest-tests/src/main.rs | 21 +- src/tools/tidy/src/alphabetical.rs | 94 +- src/tools/tidy/src/alphabetical/tests.rs | 188 ++ src/tools/tidy/src/deps.rs | 268 +- src/tools/tidy/src/extdeps.rs | 36 +- src/tools/tidy/src/features.rs | 16 +- src/tools/tidy/src/lib.rs | 16 +- src/tools/tidy/src/mir_opt_tests.rs | 3 +- src/tools/tidy/src/style.rs | 7 +- src/tools/tidy/src/ui_tests.rs | 2 +- src/version | 2 +- 1366 files changed, 55755 insertions(+), 40442 deletions(-) delete mode 100644 src/bootstrap/CHANGELOG.md delete mode 100644 src/bootstrap/bin/_helper.rs delete mode 100644 src/bootstrap/bin/main.rs delete mode 100644 src/bootstrap/bin/rustc.rs delete mode 100644 src/bootstrap/bin/rustdoc.rs delete mode 100644 src/bootstrap/bin/sccache-plus-cl.rs delete mode 100644 src/bootstrap/builder.rs delete mode 100644 src/bootstrap/builder/tests.rs delete mode 100644 src/bootstrap/cache.rs delete mode 100644 src/bootstrap/cc_detect.rs delete mode 100644 src/bootstrap/channel.rs delete mode 100644 src/bootstrap/check.rs delete mode 100644 src/bootstrap/clean.rs delete mode 100644 src/bootstrap/compile.rs delete mode 100644 src/bootstrap/config.rs delete mode 100644 src/bootstrap/config/tests.rs delete mode 100644 src/bootstrap/dist.rs delete mode 100644 src/bootstrap/doc.rs delete mode 100644 src/bootstrap/download.rs delete mode 100644 src/bootstrap/dylib_util.rs delete mode 100644 src/bootstrap/flags.rs delete mode 100644 src/bootstrap/format.rs delete mode 100644 src/bootstrap/install.rs delete mode 100644 src/bootstrap/job.rs delete mode 100644 src/bootstrap/lib.rs delete mode 100644 src/bootstrap/llvm.rs delete mode 100644 src/bootstrap/metadata.rs delete mode 100644 src/bootstrap/metrics.rs delete mode 100644 src/bootstrap/render_tests.rs delete mode 100644 src/bootstrap/run.rs delete mode 100644 src/bootstrap/sanity.rs delete mode 100644 src/bootstrap/setup.rs delete mode 100644 src/bootstrap/setup/tests.rs create mode 100644 src/bootstrap/src/bin/main.rs create mode 100644 src/bootstrap/src/bin/rustc.rs create mode 100644 src/bootstrap/src/bin/rustdoc.rs create mode 100644 src/bootstrap/src/bin/sccache-plus-cl.rs create mode 100644 src/bootstrap/src/core/build_steps/check.rs create mode 100644 src/bootstrap/src/core/build_steps/clean.rs create mode 100644 src/bootstrap/src/core/build_steps/compile.rs create mode 100644 src/bootstrap/src/core/build_steps/dist.rs create mode 100644 src/bootstrap/src/core/build_steps/doc.rs create mode 100644 src/bootstrap/src/core/build_steps/format.rs create mode 100644 src/bootstrap/src/core/build_steps/install.rs create mode 100644 src/bootstrap/src/core/build_steps/llvm.rs create mode 100644 src/bootstrap/src/core/build_steps/mod.rs create mode 100644 src/bootstrap/src/core/build_steps/run.rs create mode 100644 src/bootstrap/src/core/build_steps/setup.rs create mode 100644 src/bootstrap/src/core/build_steps/suggest.rs create mode 100644 src/bootstrap/src/core/build_steps/synthetic_targets.rs create mode 100644 src/bootstrap/src/core/build_steps/test.rs create mode 100644 src/bootstrap/src/core/build_steps/tool.rs create mode 100644 src/bootstrap/src/core/build_steps/toolstate.rs create mode 100644 src/bootstrap/src/core/builder.rs create mode 100644 src/bootstrap/src/core/config/config.rs create mode 100644 src/bootstrap/src/core/config/flags.rs create mode 100644 src/bootstrap/src/core/config/mod.rs create mode 100644 src/bootstrap/src/core/download.rs create mode 100644 src/bootstrap/src/core/metadata.rs create mode 100644 src/bootstrap/src/core/mod.rs create mode 100644 src/bootstrap/src/core/sanity.rs create mode 100644 src/bootstrap/src/lib.rs create mode 100644 src/bootstrap/src/tests/builder.rs create mode 100644 src/bootstrap/src/tests/config.rs create mode 100644 src/bootstrap/src/tests/setup.rs create mode 100644 src/bootstrap/src/utils/bin_helpers.rs create mode 100644 src/bootstrap/src/utils/cache.rs create mode 100644 src/bootstrap/src/utils/cc_detect.rs create mode 100644 src/bootstrap/src/utils/channel.rs create mode 100644 src/bootstrap/src/utils/dylib.rs create mode 100644 src/bootstrap/src/utils/exec.rs create mode 100644 src/bootstrap/src/utils/helpers.rs create mode 100644 src/bootstrap/src/utils/job.rs create mode 100644 src/bootstrap/src/utils/metrics.rs create mode 100644 src/bootstrap/src/utils/mod.rs create mode 100644 src/bootstrap/src/utils/render_tests.rs create mode 100644 src/bootstrap/src/utils/tarball.rs delete mode 100644 src/bootstrap/suggest.rs delete mode 100644 src/bootstrap/synthetic_targets.rs delete mode 100644 src/bootstrap/tarball.rs delete mode 100644 src/bootstrap/test.rs delete mode 100644 src/bootstrap/tool.rs delete mode 100644 src/bootstrap/toolstate.rs delete mode 100644 src/bootstrap/util.rs delete mode 100755 src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh delete mode 100755 src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh create mode 100644 src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock delete mode 100644 src/ci/docker/host-x86_64/wasm32/Dockerfile create mode 100644 src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile create mode 100755 src/ci/scripts/install-awscli.sh create mode 100755 src/ci/scripts/install-tidy.sh create mode 100644 src/doc/rustc-dev-guide/src/effects.md create mode 100644 src/doc/rustc-dev-guide/src/solve/invariants.md create mode 100644 src/doc/rustc-dev-guide/src/traits/unsize.md create mode 100644 src/doc/rustc/src/platform-support/aix.md create mode 100644 src/doc/unstable-book/src/compiler-flags/no-jump-tables.md create mode 100644 src/doc/unstable-book/src/compiler-flags/remap-path-scope.md create mode 100644 src/doc/unstable-book/src/language-features/coroutines.md create mode 100644 src/doc/unstable-book/src/language-features/diagnostic-namespace.md delete mode 100644 src/doc/unstable-book/src/language-features/generators.md delete mode 100644 src/doc/unstable-book/src/language-features/plugin.md create mode 100644 src/doc/unstable-book/src/language-features/string-deref-patterns.md create mode 100644 src/etc/completions/x.py.zsh create mode 100644 src/etc/test-float-parse/Cargo.lock create mode 100644 src/tools/cargo/ci/generate.py create mode 120000 src/tools/cargo/credential/cargo-credential-1password/LICENSE-APACHE create mode 120000 src/tools/cargo/credential/cargo-credential-1password/LICENSE-MIT create mode 120000 src/tools/cargo/credential/cargo-credential-libsecret/LICENSE-APACHE create mode 120000 src/tools/cargo/credential/cargo-credential-libsecret/LICENSE-MIT create mode 120000 src/tools/cargo/credential/cargo-credential-macos-keychain/LICENSE-APACHE create mode 120000 src/tools/cargo/credential/cargo-credential-macos-keychain/LICENSE-MIT create mode 120000 src/tools/cargo/credential/cargo-credential-wincred/LICENSE-APACHE create mode 120000 src/tools/cargo/credential/cargo-credential-wincred/LICENSE-MIT create mode 120000 src/tools/cargo/credential/cargo-credential/LICENSE-APACHE create mode 120000 src/tools/cargo/credential/cargo-credential/LICENSE-MIT create mode 100644 src/tools/cargo/src/cargo/util/cache_lock.rs create mode 100644 src/tools/cargo/src/cargo/util/hostname.rs delete mode 100644 src/tools/cargo/src/cargo/util/to_semver.rs create mode 100644 src/tools/cargo/src/cargo/util/toml/schema.rs create mode 100644 src/tools/cargo/src/cargo/util_semver.rs create mode 100644 src/tools/cargo/src/doc/contrib/src/implementation/formatting.md create mode 100644 src/tools/cargo/src/doc/contrib/src/implementation/packages.md create mode 100644 src/tools/cargo/tests/testsuite/cache_lock.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_add/preserve_dep_std_table/in/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_add/preserve_dep_std_table/in/src/lib.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_add/preserve_dep_std_table/mod.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_add/preserve_dep_std_table/out/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_add/preserve_dep_std_table/stderr.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_add/preserve_dep_std_table/stdout.log delete mode 100644 src/tools/cargo/tests/testsuite/cargo_init/empty_dir/.keep create mode 100644 src/tools/cargo/tests/testsuite/cargo_init/workspace_add_member/in/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_init/workspace_add_member/in/crates/foo/.keep create mode 100644 src/tools/cargo/tests/testsuite/cargo_init/workspace_add_member/mod.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_init/workspace_add_member/out/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_init/workspace_add_member/out/crates/foo/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_init/workspace_add_member/out/crates/foo/src/main.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_init/workspace_add_member/stderr.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_init/workspace_add_member/stdout.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_previous_items/in/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_previous_items/mod.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_previous_items/out/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_previous_items/out/crates/foo/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_previous_items/out/crates/foo/src/main.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_previous_items/stderr.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_previous_items/stdout.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/in/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/in/crates/bar/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/in/crates/bar/src/main.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/in/crates/qux/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/in/crates/qux/src/main.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/mod.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/out/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/out/crates/foo/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/out/crates/foo/src/main.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/stderr.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_format_sorted/stdout.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_absolute_package_path/in/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_absolute_package_path/mod.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_absolute_package_path/out/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_absolute_package_path/out/crates/foo/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_absolute_package_path/out/crates/foo/src/main.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_absolute_package_path/stderr.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_absolute_package_path/stdout.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_empty_members/in/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_empty_members/mod.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_empty_members/out/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_empty_members/out/crates/foo/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_empty_members/out/crates/foo/src/main.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_empty_members/stderr.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_empty_members/stdout.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_exclude_list/in/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_exclude_list/in/src/lib.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_exclude_list/mod.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_exclude_list/out/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_exclude_list/out/crates/foo/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_exclude_list/out/crates/foo/src/main.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_exclude_list/stderr.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_exclude_list/stdout.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_members_glob/in/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_members_glob/mod.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_members_glob/out/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_members_glob/out/crates/foo/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_members_glob/out/crates/foo/src/main.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_members_glob/stderr.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_new/add_members_to_workspace_with_members_glob/stdout.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_remove/optional_dep_feature_formatting/in/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_remove/optional_dep_feature_formatting/in/src/lib.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_remove/optional_dep_feature_formatting/mod.rs create mode 100644 src/tools/cargo/tests/testsuite/cargo_remove/optional_dep_feature_formatting/out/Cargo.toml create mode 100644 src/tools/cargo/tests/testsuite/cargo_remove/optional_dep_feature_formatting/stderr.log create mode 100644 src/tools/cargo/tests/testsuite/cargo_remove/optional_dep_feature_formatting/stdout.log delete mode 100644 src/tools/cargo/tests/testsuite/plugins.rs create mode 100644 src/tools/cargo/tests/testsuite/profile_trim_paths.rs create mode 100644 src/tools/clippy/clippy_config/Cargo.toml create mode 100644 src/tools/clippy/clippy_config/src/conf.rs create mode 100644 src/tools/clippy/clippy_config/src/lib.rs create mode 100644 src/tools/clippy/clippy_config/src/metadata.rs create mode 100644 src/tools/clippy/clippy_config/src/msrvs.rs create mode 100644 src/tools/clippy/clippy_config/src/types.rs delete mode 100644 src/tools/clippy/clippy_lints/src/enum_variants.rs create mode 100644 src/tools/clippy/clippy_lints/src/item_name_repetitions.rs create mode 100644 src/tools/clippy/clippy_lints/src/iter_without_into_iter.rs create mode 100644 src/tools/clippy/clippy_lints/src/loops/unused_enumerate_index.rs create mode 100644 src/tools/clippy/clippy_lints/src/manual_hash_one.rs create mode 100644 src/tools/clippy/clippy_lints/src/methods/unnecessary_fallible_conversions.rs create mode 100644 src/tools/clippy/clippy_lints/src/methods/waker_clone_wake.rs delete mode 100644 src/tools/clippy/clippy_lints/src/utils/conf.rs delete mode 100644 src/tools/clippy/clippy_lints/src/utils/internal_lints/clippy_lints_internal.rs create mode 100644 src/tools/clippy/clippy_lints/src/utils/internal_lints/unsorted_clippy_utils_paths.rs delete mode 100644 src/tools/clippy/clippy_utils/src/msrvs.rs create mode 100644 src/tools/clippy/tests/ui-toml/borrow_interior_mutable_const/clippy.toml create mode 100644 src/tools/clippy/tests/ui-toml/borrow_interior_mutable_const/ignore.rs create mode 100644 src/tools/clippy/tests/ui-toml/declare_interior_mutable_const/clippy.toml create mode 100644 src/tools/clippy/tests/ui-toml/declare_interior_mutable_const/ignore.rs delete mode 100644 src/tools/clippy/tests/ui-toml/enum_variant_names/clippy.toml delete mode 100644 src/tools/clippy/tests/ui-toml/enum_variant_names/enum_variant_names.rs delete mode 100644 src/tools/clippy/tests/ui-toml/enum_variant_names/enum_variant_names.stderr delete mode 100644 src/tools/clippy/tests/ui-toml/enum_variants_threshold0/clippy.toml delete mode 100644 src/tools/clippy/tests/ui-toml/enum_variants_threshold0/enum_variants_name_threshold.rs create mode 100644 src/tools/clippy/tests/ui-toml/impl_trait_in_params/clippy.toml create mode 100644 src/tools/clippy/tests/ui-toml/impl_trait_in_params/impl_trait_in_params.rs create mode 100644 src/tools/clippy/tests/ui-toml/impl_trait_in_params/impl_trait_in_params.stderr create mode 100644 src/tools/clippy/tests/ui-toml/item_name_repetitions/threshold0/clippy.toml create mode 100644 src/tools/clippy/tests/ui-toml/item_name_repetitions/threshold0/item_name_repetitions.rs create mode 100644 src/tools/clippy/tests/ui-toml/item_name_repetitions/threshold5/clippy.toml create mode 100644 src/tools/clippy/tests/ui-toml/item_name_repetitions/threshold5/item_name_repetitions.rs create mode 100644 src/tools/clippy/tests/ui-toml/item_name_repetitions/threshold5/item_name_repetitions.stderr create mode 100644 src/tools/clippy/tests/ui/author/macro_in_closure.rs create mode 100644 src/tools/clippy/tests/ui/author/macro_in_closure.stdout create mode 100644 src/tools/clippy/tests/ui/author/macro_in_loop.rs create mode 100644 src/tools/clippy/tests/ui/author/macro_in_loop.stdout create mode 100644 src/tools/clippy/tests/ui/crashes/ice-11230.rs create mode 100644 src/tools/clippy/tests/ui/crashes/ice-11755.rs create mode 100644 src/tools/clippy/tests/ui/if_not_else_bittest.rs create mode 100644 src/tools/clippy/tests/ui/into_iter_without_iter.rs create mode 100644 src/tools/clippy/tests/ui/into_iter_without_iter.stderr create mode 100644 src/tools/clippy/tests/ui/items_after_test_module/after_proc_macros.rs create mode 100644 src/tools/clippy/tests/ui/items_after_test_module/auxiliary/submodule.rs delete mode 100644 src/tools/clippy/tests/ui/items_after_test_module/block_module.rs delete mode 100644 src/tools/clippy/tests/ui/items_after_test_module/block_module.stderr create mode 100644 src/tools/clippy/tests/ui/items_after_test_module/in_submodule.rs create mode 100644 src/tools/clippy/tests/ui/items_after_test_module/in_submodule.stderr create mode 100644 src/tools/clippy/tests/ui/items_after_test_module/multiple_modules.rs create mode 100644 src/tools/clippy/tests/ui/items_after_test_module/root_module.fixed create mode 100644 src/tools/clippy/tests/ui/items_after_test_module/root_module.rs create mode 100644 src/tools/clippy/tests/ui/items_after_test_module/root_module.stderr create mode 100644 src/tools/clippy/tests/ui/iter_without_into_iter.rs create mode 100644 src/tools/clippy/tests/ui/iter_without_into_iter.stderr create mode 100644 src/tools/clippy/tests/ui/manual_hash_one.fixed create mode 100644 src/tools/clippy/tests/ui/manual_hash_one.rs create mode 100644 src/tools/clippy/tests/ui/manual_hash_one.stderr create mode 100644 src/tools/clippy/tests/ui/struct_fields.rs create mode 100644 src/tools/clippy/tests/ui/struct_fields.stderr create mode 100644 src/tools/clippy/tests/ui/unnecessary_fallible_conversions.fixed create mode 100644 src/tools/clippy/tests/ui/unnecessary_fallible_conversions.rs create mode 100644 src/tools/clippy/tests/ui/unnecessary_fallible_conversions.stderr create mode 100644 src/tools/clippy/tests/ui/unnecessary_fallible_conversions_unfixable.rs create mode 100644 src/tools/clippy/tests/ui/unnecessary_fallible_conversions_unfixable.stderr create mode 100644 src/tools/clippy/tests/ui/unused_enumerate_index.fixed create mode 100644 src/tools/clippy/tests/ui/unused_enumerate_index.rs create mode 100644 src/tools/clippy/tests/ui/unused_enumerate_index.stderr create mode 100644 src/tools/clippy/tests/ui/waker_clone_wake.fixed create mode 100644 src/tools/clippy/tests/ui/waker_clone_wake.rs create mode 100644 src/tools/clippy/tests/ui/waker_clone_wake.stderr create mode 100644 src/tools/opt-dist/src/utils/artifact_size.rs create mode 100644 src/tools/rustfmt/tests/config/issue-5816.toml create mode 100644 src/tools/rustfmt/tests/source/immovable_coroutines.rs delete mode 100644 src/tools/rustfmt/tests/source/immovable_generators.rs create mode 100644 src/tools/rustfmt/tests/source/issue-3984.rs create mode 100644 src/tools/rustfmt/tests/source/issue-4808.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5655/one.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5655/two.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5791.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5835.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5852/default.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5852/horizontal.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5852/horizontal_vertical.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5852/issue_example.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5852/split.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5852/vertical.rs create mode 100644 src/tools/rustfmt/tests/source/issue-5935.rs create mode 100644 src/tools/rustfmt/tests/source/issue_5721.rs create mode 100644 src/tools/rustfmt/tests/source/issue_5730.rs create mode 100644 src/tools/rustfmt/tests/source/issue_5735.rs create mode 100644 src/tools/rustfmt/tests/source/issue_5882.rs create mode 100644 src/tools/rustfmt/tests/source/let_chains.rs create mode 100644 src/tools/rustfmt/tests/source/let_else_v2.rs create mode 100644 src/tools/rustfmt/tests/source/non-lifetime-binders.rs create mode 100644 src/tools/rustfmt/tests/source/skip_macro_invocations/config_file.rs create mode 100644 src/tools/rustfmt/tests/target/extern-rust.rs create mode 100644 src/tools/rustfmt/tests/target/immovable_coroutines.rs delete mode 100644 src/tools/rustfmt/tests/target/immovable_generators.rs create mode 100644 src/tools/rustfmt/tests/target/issue-3984.rs create mode 100644 src/tools/rustfmt/tests/target/issue-4808.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5568.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5655/one.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5655/two.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5791.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5797/retain_trailing_semicolon.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5835.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5852/default.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5852/horizontal.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5852/horizontal_vertical.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5852/issue_example.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5852/split.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5852/vertical.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5871.rs create mode 100644 src/tools/rustfmt/tests/target/issue-5935.rs create mode 100644 src/tools/rustfmt/tests/target/issue_5533.rs create mode 100644 src/tools/rustfmt/tests/target/issue_5542.rs create mode 100644 src/tools/rustfmt/tests/target/issue_5676.rs create mode 100644 src/tools/rustfmt/tests/target/issue_5721.rs create mode 100644 src/tools/rustfmt/tests/target/issue_5730.rs create mode 100644 src/tools/rustfmt/tests/target/issue_5735.rs create mode 100644 src/tools/rustfmt/tests/target/issue_5882.rs create mode 100644 src/tools/rustfmt/tests/target/issue_5907.rs create mode 100644 src/tools/rustfmt/tests/target/let_chains.rs create mode 100644 src/tools/rustfmt/tests/target/let_else_v2.rs create mode 100644 src/tools/rustfmt/tests/target/non-lifetime-binders.rs create mode 100644 src/tools/rustfmt/tests/target/skip_macro_invocations/config_file.rs create mode 100644 src/tools/tidy/src/alphabetical/tests.rs (limited to 'src') diff --git a/src/bootstrap/CHANGELOG.md b/src/bootstrap/CHANGELOG.md deleted file mode 100644 index 1aba07138..000000000 --- a/src/bootstrap/CHANGELOG.md +++ /dev/null @@ -1,71 +0,0 @@ -# Changelog - -All notable changes to bootstrap will be documented in this file. - -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). - - -## [Changes since the last major version] - -- Vendoring is no longer done automatically when building from git sources. To use vendoring, run `cargo vendor` manually, or use the pre-vendored `rustc-src` tarball. -- `llvm-libunwind` now accepts `in-tree` (formerly true), `system` or `no` (formerly false) [#77703](https://github.com/rust-lang/rust/pull/77703) -- The options `infodir`, `localstatedir`, and `gpg-password-file` are no longer allowed in config.toml. Previously, they were ignored without warning. Note that `infodir` and `localstatedir` are still accepted by `./configure`, with a warning. [#82451](https://github.com/rust-lang/rust/pull/82451) -- Change the names for `dist` commands to match the component they generate. [#90684](https://github.com/rust-lang/rust/pull/90684) -- The `build.fast-submodules` option has been removed. Fast submodule checkouts are enabled unconditionally. Automatic submodule handling can still be disabled with `build.submodules = false`. -- Several unsupported `./configure` options have been removed: `optimize`, `parallel-compiler`. These can still be enabled with `--set`, although it isn't recommended. -- `remote-test-server`'s `verbose` argument has been removed in favor of the `--verbose` flag -- `remote-test-server`'s `remote` argument has been removed in favor of the `--bind` flag. Use `--bind 0.0.0.0:12345` to replicate the behavior of the `remote` argument. -- `x.py fmt` now formats only files modified between the merge-base of HEAD and the last commit in the master branch of the rust-lang repository and the current working directory. To restore old behaviour, use `x.py fmt .`. The check mode is not affected by this change. [#105702](https://github.com/rust-lang/rust/pull/105702) -- The `llvm.version-check` config option has been removed. Older versions were never supported. If you still need to support older versions (e.g. you are applying custom patches), patch `check_llvm_version` in bootstrap to change the minimum version. [#108619](https://github.com/rust-lang/rust/pull/108619) -- The `rust.ignore-git` option has been renamed to `rust.omit-git-hash`. [#110059](https://github.com/rust-lang/rust/pull/110059) -- `--exclude` no longer accepts a `Kind` as part of a Step; instead it uses the top-level Kind of the subcommand. If this matches how you were already using --exclude (e.g. `x test --exclude test::std`), simply remove the kind: `--exclude std`. If you were using a kind that did not match the top-level subcommand, please open an issue explaining why you wanted this feature. - -### Non-breaking changes - -- `x.py check` needs opt-in to check tests (--all-targets) [#77473](https://github.com/rust-lang/rust/pull/77473) -- The default bootstrap profiles are now located at `bootstrap/defaults/config.$PROFILE.toml` (previously they were located at `bootstrap/defaults/config.toml.$PROFILE`) [#77558](https://github.com/rust-lang/rust/pull/77558) -- If you have Rust already installed, `x.py` will now infer the host target - from the default rust toolchain. [#78513](https://github.com/rust-lang/rust/pull/78513) -- Add options for enabling overflow checks, one for std (`overflow-checks-std`) and one for everything else (`overflow-checks`). Both default to false. -- Add llvm option `enable-warnings` to have control on llvm compilation warnings. Default to false. -- Add `rpath` option in `target` section to support set rpath option for each target independently. [#111242](https://github.com/rust-lang/rust/pull/111242) - - -## [Version 2] - 2020-09-25 - -- `host` now defaults to the value of `build` in all cases - + Previously `host` defaulted to an empty list when `target` was overridden, and to `build` otherwise - -### Non-breaking changes - -- Add `x.py setup` [#76631](https://github.com/rust-lang/rust/pull/76631) -- Add a changelog for x.py [#76626](https://github.com/rust-lang/rust/pull/76626) -- Optionally, download LLVM from CI on Linux and NixOS. This can be enabled with `download-ci-llvm = true` under `[llvm]`. - + [#76439](https://github.com/rust-lang/rust/pull/76349) - + [#76667](https://github.com/rust-lang/rust/pull/76667) - + [#76708](https://github.com/rust-lang/rust/pull/76708) -- Distribute rustc sources as part of `rustc-dev` [#76856](https://github.com/rust-lang/rust/pull/76856) -- Make the default stage for x.py configurable [#76625](https://github.com/rust-lang/rust/pull/76625). This can be enabled with `build-stage = N`, `doc-stage = N`, etc. -- Add a dedicated debug-logging option [#76588](https://github.com/rust-lang/rust/pull/76588). Previously, `debug-logging` could only be set with `debug-assertions`, slowing down the compiler more than necessary. -- Add sample defaults for x.py [#76628](https://github.com/rust-lang/rust/pull/76628) -- Add `--keep-stage-std`, which behaves like `keep-stage` but allows the stage - 0 compiler artifacts (i.e., stage1/bin/rustc) to be rebuilt if changed - [#77120](https://github.com/rust-lang/rust/pull/77120). -- File locking is now used to avoid collisions between multiple running instances of `x.py` (e.g. when using `rust-analyzer` and `x.py` at the same time). Note that Solaris and possibly other non Unix and non Windows systems don't support it [#108607](https://github.com/rust-lang/rust/pull/108607). This might possibly lead to build data corruption. - - -## [Version 1] - 2020-09-11 - -This is the first changelog entry, and it does not attempt to be an exhaustive list of features in x.py. -Instead, this documents the changes to bootstrap in the past 2 months. - -- Improve defaults in `x.py` [#73964](https://github.com/rust-lang/rust/pull/73964) - (see [blog post] for details) -- Set `ninja = true` by default [#74922](https://github.com/rust-lang/rust/pull/74922) -- Avoid trying to inversely cross-compile for build triple from host triples [#76415](https://github.com/rust-lang/rust/pull/76415) -- Allow blessing expect-tests in tools [#75975](https://github.com/rust-lang/rust/pull/75975) -- `x.py check` checks tests/examples/benches [#76258](https://github.com/rust-lang/rust/pull/76258) -- Fix `rust.use-lld` when linker is not set [#76326](https://github.com/rust-lang/rust/pull/76326) -- Build tests with LLD if `use-lld = true` was passed [#76378](https://github.com/rust-lang/rust/pull/76378) - -[blog post]: https://blog.rust-lang.org/inside-rust/2020/08/30/changes-to-x-py-defaults.html diff --git a/src/bootstrap/Cargo.lock b/src/bootstrap/Cargo.lock index f5220e361..57113b0ec 100644 --- a/src/bootstrap/Cargo.lock +++ b/src/bootstrap/Cargo.lock @@ -29,6 +29,12 @@ version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" + [[package]] name = "block-buffer" version = "0.10.2" @@ -50,6 +56,7 @@ dependencies = [ "fd-lock", "filetime", "hex", + "home", "ignore", "junction", "libc", @@ -104,40 +111,38 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clap" -version = "4.2.4" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "956ac1f6381d8d82ab4684768f89c0ea3afe66925ceadb4eeb3fc452ffc55d62" +checksum = "ac495e00dcec98c83465d5ad66c5c4fabd652fd6686e7c6269b117e729a6f17b" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.2.4" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84080e799e54cff944f4b4a4b0e71630b0e0443b25b985175c7dddc1a859b749" +checksum = "c77ed9a32a62e6ca27175d00d29d05ca32e396ea1eb5fb01d8256b669cec7663" dependencies = [ "anstyle", - "bitflags", "clap_lex", ] [[package]] name = "clap_complete" -version = "4.2.2" +version = "4.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36774babb166352bb4f7b9cb16f781ffa3439d2a8f12cd31bea85a38c888fea3" +checksum = "e3ae8ba90b9d8b007efe66e55e48fb936272f5ca00349b5b0e89877520d35ea7" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.2.0" +version = "4.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9644cd56d6b87dbe899ef8b053e331c0637664e9e21a33dfcdc36093f5c5c4" +checksum = "cf9804afaaf59a91e75b022a30fb7229a7901f60c755489cc61c9b423b836442" dependencies = [ "heck", "proc-macro2", @@ -147,9 +152,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.4.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a2dd5a6fe8c6e3502f568a6353e5273bbb15193ad9a89e457b9970798efbea1" +checksum = "702fc72eb24e5a1e48ce58027a675bc24edd52096d5397d4aea7c6dd9eca0bd1" [[package]] name = "cmake" @@ -175,16 +180,6 @@ dependencies = [ "libc", ] -[[package]] -name = "crossbeam-channel" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521" -dependencies = [ - "cfg-if", - "crossbeam-utils", -] - [[package]] name = "crossbeam-deque" version = "0.8.2" @@ -252,30 +247,19 @@ checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" [[package]] name = "errno" -version = "0.3.0" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d6a0976c999d473fe89ad888d5a284e55366d9dc9038b1ba2aa15128c4afa0" +checksum = "ac3e13f66a2f95e32a39eaa81f6b95d42878ca0e1db0c7543723dfe12557e860" dependencies = [ - "errno-dragonfly", "libc", "windows-sys", ] -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "fd-lock" -version = "3.0.11" +version = "3.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9799aefb4a2e4a01cc47610b1dd47c18ab13d991f27bbcaed9296f5a53d5cbad" +checksum = "ef033ed5e9bad94e55838ca0ca906db0e043f517adda0c8b79c7a8c66c93c1b5" dependencies = [ "cfg-if", "rustix", @@ -329,27 +313,21 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" -[[package]] -name = "hermit-abi" -version = "0.1.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" -dependencies = [ - "libc", -] - -[[package]] -name = "hermit-abi" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" - [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "home" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747309b4b440c06d57b0b25f2aee03ee9b5e5397d288c60e21fc709bb98a7408" +dependencies = [ + "winapi", +] + [[package]] name = "ignore" version = "0.4.18" @@ -368,17 +346,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "io-lifetimes" -version = "1.0.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09270fd4fa1111bc614ed2246c7ef56239a3063d5be0d1ec3b589c505d400aeb" -dependencies = [ - "hermit-abi 0.3.2", - "libc", - "windows-sys", -] - [[package]] name = "itoa" version = "1.0.2" @@ -403,15 +370,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.140" +version = "0.2.149" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99227334921fae1a979cf0bfdfcc6b3e5ce376ef57e16fb6fb3ea2ed6095f80c" +checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" [[package]] name = "linux-raw-sys" -version = "0.3.2" +version = "0.4.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f508063cc7bb32987c71511216bd5a32be15bccb6a80b52df8b9d7f01fc3aa2" +checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" [[package]] name = "log" @@ -457,16 +424,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "num_cpus" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" -dependencies = [ - "hermit-abi 0.1.19", - "libc", -] - [[package]] name = "object" version = "0.32.0" @@ -528,25 +485,22 @@ dependencies = [ [[package]] name = "rayon" -version = "1.6.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e060280438193c554f654141c9ea9417886713b7acd75974c85b18a69a88e0b" +checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1" dependencies = [ - "crossbeam-deque", "either", "rayon-core", ] [[package]] name = "rayon-core" -version = "1.10.1" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cac410af5d00ab6884528b4ab69d1e8e146e8d471201800fa1b4524126de6ad3" +checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed" dependencies = [ - "crossbeam-channel", "crossbeam-deque", "crossbeam-utils", - "num_cpus", ] [[package]] @@ -555,7 +509,7 @@ version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62f25bc4c7e55e0b0b7a1d43fb893f4fa1361d0abe38b9ce4f323c2adfe6ef42" dependencies = [ - "bitflags", + "bitflags 1.3.2", ] [[package]] @@ -583,13 +537,12 @@ checksum = "49b3de9ec5dc0a3417da371aab17d729997c15010e7fd24ff707773a33bddb64" [[package]] name = "rustix" -version = "0.37.6" +version = "0.38.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d097081ed288dfe45699b72f5b5d648e5f15d64d900c7080273baa20c16a6849" +checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" dependencies = [ - "bitflags", + "bitflags 2.4.1", "errno", - "io-lifetimes", "libc", "linux-raw-sys", "windows-sys", @@ -787,27 +740,37 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "windows" -version = "0.46.0" +version = "0.51.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca229916c5ee38c2f2bc1e9d8f04df975b4bd93f9955dc69fabb5d91270045c9" +dependencies = [ + "windows-core", + "windows-targets", +] + +[[package]] +name = "windows-core" +version = "0.51.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdacb41e6a96a052c6cb63a144f24900236121c6f63f4f8219fef5977ecb0c25" +checksum = "f1f8cf84f35d2db49a46868f947758c7a1138116f7fac3bc844f43ade1292e64" dependencies = [ "windows-targets", ] [[package]] name = "windows-sys" -version = "0.45.0" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "windows-targets", ] [[package]] name = "windows-targets" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm", "windows_aarch64_msvc", @@ -820,45 +783,45 @@ dependencies = [ [[package]] name = "windows_aarch64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_i686_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_x86_64_gnu" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnullvm" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_msvc" -version = "0.42.2" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "xattr" diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 9bf26948a..e4d359141 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -5,38 +5,48 @@ edition = "2021" build = "build.rs" default-run = "bootstrap" +[features] +build-metrics = ["sysinfo"] + [lib] -path = "lib.rs" +path = "src/lib.rs" doctest = false [[bin]] name = "bootstrap" -path = "bin/main.rs" +path = "src/bin/main.rs" test = false [[bin]] name = "rustc" -path = "bin/rustc.rs" +path = "src/bin/rustc.rs" test = false [[bin]] name = "rustdoc" -path = "bin/rustdoc.rs" +path = "src/bin/rustdoc.rs" test = false [[bin]] name = "sccache-plus-cl" -path = "bin/sccache-plus-cl.rs" +path = "src/bin/sccache-plus-cl.rs" test = false [dependencies] build_helper = { path = "../tools/build_helper" } +cc = "1.0.69" +clap = { version = "4.4.7", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } +clap_complete = "4.4.3" cmake = "0.1.38" filetime = "0.2" -cc = "1.0.69" -libc = "0.2" hex = "0.4" +home = "0.5.4" +ignore = "0.4.10" +libc = "0.2" object = { version = "0.32.0", default-features = false, features = ["archive", "coff", "read_core", "unaligned"] } +once_cell = "1.7.2" +opener = "0.5" +semver = "1.0.17" serde = "1.0.137" # Directly use serde_derive rather than through the derive feature of serde to allow building both # in parallel and to allow serde_json and toml to start building as soon as serde has been built. @@ -46,27 +56,21 @@ sha2 = "0.10" tar = "0.4" termcolor = "1.2.0" toml = "0.5" -ignore = "0.4.10" -opener = "0.5" -once_cell = "1.7.2" -xz2 = "0.1" walkdir = "2" +xz2 = "0.1" # Dependencies needed by the build-metrics feature sysinfo = { version = "0.26.0", optional = true } -clap = { version = "4.2.4", default-features = false, features = ["std", "usage", "help", "derive", "error-context"] } -clap_complete = "4.2.2" -semver = "1.0.17" # Solaris doesn't support flock() and thus fd-lock is not option now [target.'cfg(not(target_os = "solaris"))'.dependencies] -fd-lock = "3.0.8" +fd-lock = "3.0.13" [target.'cfg(windows)'.dependencies.junction] version = "1.0.0" [target.'cfg(windows)'.dependencies.windows] -version = "0.46.0" +version = "0.51.1" features = [ "Win32_Foundation", "Win32_Security", @@ -80,9 +84,6 @@ features = [ [dev-dependencies] pretty_assertions = "1.4" -[features] -build-metrics = ["sysinfo"] - # We care a lot about bootstrap's compile times, so don't include debuginfo for # dependencies, only bootstrap itself. [profile.dev] diff --git a/src/bootstrap/README.md b/src/bootstrap/README.md index 548281ca5..e7998a40a 100644 --- a/src/bootstrap/README.md +++ b/src/bootstrap/README.md @@ -181,11 +181,10 @@ Some general areas that you may be interested in modifying are: `Config` struct. * Adding a sanity check? Take a look at `bootstrap/sanity.rs`. -If you make a major change, please remember to: +If you make a major change on bootstrap configuration, please remember to: -+ Update `VERSION` in `src/bootstrap/main.rs`. -* Update `changelog-seen = N` in `config.example.toml`. -* Add an entry in `src/bootstrap/CHANGELOG.md`. ++ Update `CONFIG_CHANGE_HISTORY` in `src/bootstrap/lib.rs`. +* Update `change-id = {pull-request-id}` in `config.example.toml`. A 'major change' includes @@ -193,7 +192,7 @@ A 'major change' includes * A change in the default options. Changes that do not affect contributors to the compiler or users -building rustc from source don't need an update to `VERSION`. +building rustc from source don't need an update to `CONFIG_CHANGE_HISTORY`. If you have any questions, feel free to reach out on the `#t-infra/bootstrap` channel at [Rust Bootstrap Zulip server][rust-bootstrap-zulip]. When you encounter bugs, @@ -201,3 +200,8 @@ please file issues on the [Rust issue tracker][rust-issue-tracker]. [rust-bootstrap-zulip]: https://rust-lang.zulipchat.com/#narrow/stream/t-infra.2Fbootstrap [rust-issue-tracker]: https://github.com/rust-lang/rust/issues + +## Changelog + +Because we do not release bootstrap with versions, we also do not maintain CHANGELOG files. To +review the changes made to bootstrap, simply run `git log --no-merges --oneline -- src/bootstrap`. diff --git a/src/bootstrap/bin/_helper.rs b/src/bootstrap/bin/_helper.rs deleted file mode 100644 index 09aa471db..000000000 --- a/src/bootstrap/bin/_helper.rs +++ /dev/null @@ -1,24 +0,0 @@ -/// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`. -/// If it was not defined, returns 0 by default. -/// -/// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer. -fn parse_rustc_verbose() -> usize { - use std::str::FromStr; - - match std::env::var("RUSTC_VERBOSE") { - Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), - Err(_) => 0, - } -} - -/// Parses the value of the "RUSTC_STAGE" environment variable and returns it as a `String`. -/// -/// If "RUSTC_STAGE" was not set, the program will be terminated with 101. -fn parse_rustc_stage() -> String { - std::env::var("RUSTC_STAGE").unwrap_or_else(|_| { - // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead. - eprintln!("rustc shim: fatal: RUSTC_STAGE was not set"); - eprintln!("rustc shim: note: use `x.py build -vvv` to see all environment variables set by bootstrap"); - exit(101); - }) -} diff --git a/src/bootstrap/bin/main.rs b/src/bootstrap/bin/main.rs deleted file mode 100644 index c497cabbd..000000000 --- a/src/bootstrap/bin/main.rs +++ /dev/null @@ -1,124 +0,0 @@ -//! rustbuild, the Rust build system -//! -//! This is the entry point for the build system used to compile the `rustc` -//! compiler. Lots of documentation can be found in the `README.md` file in the -//! parent directory, and otherwise documentation can be found throughout the `build` -//! directory in each respective module. - -#[cfg(all(any(unix, windows), not(target_os = "solaris")))] -use std::io::Write; -#[cfg(all(any(unix, windows), not(target_os = "solaris")))] -use std::process; -use std::{env, fs}; - -#[cfg(all(any(unix, windows), not(target_os = "solaris")))] -use bootstrap::t; -use bootstrap::{Build, Config, Subcommand, VERSION}; - -fn main() { - let args = env::args().skip(1).collect::>(); - let config = Config::parse(&args); - - #[cfg(all(any(unix, windows), not(target_os = "solaris")))] - let mut build_lock; - #[cfg(all(any(unix, windows), not(target_os = "solaris")))] - let _build_lock_guard; - #[cfg(all(any(unix, windows), not(target_os = "solaris")))] - // Display PID of process holding the lock - // PID will be stored in a lock file - { - let path = config.out.join("lock"); - let pid = match fs::read_to_string(&path) { - Ok(contents) => contents, - Err(_) => String::new(), - }; - - build_lock = - fd_lock::RwLock::new(t!(fs::OpenOptions::new().write(true).create(true).open(&path))); - _build_lock_guard = match build_lock.try_write() { - Ok(mut lock) => { - t!(lock.write(&process::id().to_string().as_ref())); - lock - } - err => { - drop(err); - println!("warning: build directory locked by process {pid}, waiting for lock"); - let mut lock = t!(build_lock.write()); - t!(lock.write(&process::id().to_string().as_ref())); - lock - } - }; - } - - #[cfg(any(not(any(unix, windows)), target_os = "solaris"))] - println!("warning: file locking not supported for target, not locking build directory"); - - // check_version warnings are not printed during setup - let changelog_suggestion = - if matches!(config.cmd, Subcommand::Setup { .. }) { None } else { check_version(&config) }; - - // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the - // changelog warning, not the `x.py setup` message. - let suggest_setup = config.config.is_none() && !matches!(config.cmd, Subcommand::Setup { .. }); - if suggest_setup { - println!("warning: you have not made a `config.toml`"); - println!( - "help: consider running `./x.py setup` or copying `config.example.toml` by running \ - `cp config.example.toml config.toml`" - ); - } else if let Some(suggestion) = &changelog_suggestion { - println!("{suggestion}"); - } - - let pre_commit = config.src.join(".git").join("hooks").join("pre-commit"); - Build::new(config).build(); - - if suggest_setup { - println!("warning: you have not made a `config.toml`"); - println!( - "help: consider running `./x.py setup` or copying `config.example.toml` by running \ - `cp config.example.toml config.toml`" - ); - } else if let Some(suggestion) = &changelog_suggestion { - println!("{suggestion}"); - } - - // Give a warning if the pre-commit script is in pre-commit and not pre-push. - // HACK: Since the commit script uses hard links, we can't actually tell if it was installed by x.py setup or not. - // We could see if it's identical to src/etc/pre-push.sh, but pre-push may have been modified in the meantime. - // Instead, look for this comment, which is almost certainly not in any custom hook. - if fs::read_to_string(pre_commit).map_or(false, |contents| { - contents.contains("https://github.com/rust-lang/rust/issues/77620#issuecomment-705144570") - }) { - println!( - "warning: You have the pre-push script installed to .git/hooks/pre-commit. \ - Consider moving it to .git/hooks/pre-push instead, which runs less often." - ); - } - - if suggest_setup || changelog_suggestion.is_some() { - println!("note: this message was printed twice to make it more likely to be seen"); - } -} - -fn check_version(config: &Config) -> Option { - let mut msg = String::new(); - - let suggestion = if let Some(seen) = config.changelog_seen { - if seen != VERSION { - msg.push_str("warning: there have been changes to x.py since you last updated.\n"); - format!("update `config.toml` to use `changelog-seen = {VERSION}` instead") - } else { - return None; - } - } else { - msg.push_str("warning: x.py has made several changes recently you may want to look at\n"); - format!("add `changelog-seen = {VERSION}` at the top of `config.toml`") - }; - - msg.push_str("help: consider looking at the changes in `src/bootstrap/CHANGELOG.md`\n"); - msg.push_str("note: to silence this warning, "); - msg.push_str(&suggestion); - - Some(msg) -} diff --git a/src/bootstrap/bin/rustc.rs b/src/bootstrap/bin/rustc.rs deleted file mode 100644 index 20cd63b96..000000000 --- a/src/bootstrap/bin/rustc.rs +++ /dev/null @@ -1,410 +0,0 @@ -//! Shim which is passed to Cargo as "rustc" when running the bootstrap. -//! -//! This shim will take care of some various tasks that our build process -//! requires that Cargo can't quite do through normal configuration: -//! -//! 1. When compiling build scripts and build dependencies, we need a guaranteed -//! full standard library available. The only compiler which actually has -//! this is the snapshot, so we detect this situation and always compile with -//! the snapshot compiler. -//! 2. We pass a bunch of `--cfg` and other flags based on what we're compiling -//! (and this slightly differs based on a whether we're using a snapshot or -//! not), so we do that all here. -//! -//! This may one day be replaced by RUSTFLAGS, but the dynamic nature of -//! switching compilers for the bootstrap and for build scripts will probably -//! never get replaced. - -include!("../dylib_util.rs"); -include!("./_helper.rs"); - -use std::env; -use std::path::PathBuf; -use std::process::{exit, Child, Command}; -use std::time::Instant; - -fn main() { - let args = env::args_os().skip(1).collect::>(); - let arg = |name| args.windows(2).find(|args| args[0] == name).and_then(|args| args[1].to_str()); - - let stage = parse_rustc_stage(); - let verbose = parse_rustc_verbose(); - - // Detect whether or not we're a build script depending on whether --target - // is passed (a bit janky...) - let target = arg("--target"); - let version = args.iter().find(|w| &**w == "-vV"); - - // Use a different compiler for build scripts, since there may not yet be a - // libstd for the real compiler to use. However, if Cargo is attempting to - // determine the version of the compiler, the real compiler needs to be - // used. Currently, these two states are differentiated based on whether - // --target and -vV is/isn't passed. - let (rustc, libdir) = if target.is_none() && version.is_none() { - ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR") - } else { - ("RUSTC_REAL", "RUSTC_LIBDIR") - }; - - let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); - let on_fail = env::var_os("RUSTC_ON_FAIL").map(Command::new); - - let rustc = env::var_os(rustc).unwrap_or_else(|| panic!("{:?} was not set", rustc)); - let libdir = env::var_os(libdir).unwrap_or_else(|| panic!("{:?} was not set", libdir)); - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(&libdir)); - - let mut cmd = Command::new(rustc); - cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - // Get the name of the crate we're compiling, if any. - let crate_name = arg("--crate-name"); - - if let Some(crate_name) = crate_name { - if let Some(target) = env::var_os("RUSTC_TIME") { - if target == "all" - || target.into_string().unwrap().split(',').any(|c| c.trim() == crate_name) - { - cmd.arg("-Ztime-passes"); - } - } - } - - // Print backtrace in case of ICE - if env::var("RUSTC_BACKTRACE_ON_ICE").is_ok() && env::var("RUST_BACKTRACE").is_err() { - cmd.env("RUST_BACKTRACE", "1"); - } - - if let Ok(lint_flags) = env::var("RUSTC_LINT_FLAGS") { - cmd.args(lint_flags.split_whitespace()); - } - - if target.is_some() { - // The stage0 compiler has a special sysroot distinct from what we - // actually downloaded, so we just always pass the `--sysroot` option, - // unless one is already set. - if !args.iter().any(|arg| arg == "--sysroot") { - cmd.arg("--sysroot").arg(&sysroot); - } - - // If we're compiling specifically the `panic_abort` crate then we pass - // the `-C panic=abort` option. Note that we do not do this for any - // other crate intentionally as this is the only crate for now that we - // ship with panic=abort. - // - // This... is a bit of a hack how we detect this. Ideally this - // information should be encoded in the crate I guess? Would likely - // require an RFC amendment to RFC 1513, however. - if crate_name == Some("panic_abort") { - cmd.arg("-C").arg("panic=abort"); - } - - // `-Ztls-model=initial-exec` must not be applied to proc-macros, see - // issue https://github.com/rust-lang/rust/issues/100530 - if env::var("RUSTC_TLS_MODEL_INITIAL_EXEC").is_ok() - && arg("--crate-type") != Some("proc-macro") - && !matches!(crate_name, Some("proc_macro2" | "quote" | "syn" | "synstructure")) - { - cmd.arg("-Ztls-model=initial-exec"); - } - } else { - // FIXME(rust-lang/cargo#5754) we shouldn't be using special env vars - // here, but rather Cargo should know what flags to pass rustc itself. - - // Override linker if necessary. - if let Ok(host_linker) = env::var("RUSTC_HOST_LINKER") { - cmd.arg(format!("-Clinker={host_linker}")); - } - if env::var_os("RUSTC_HOST_FUSE_LD_LLD").is_some() { - cmd.arg("-Clink-args=-fuse-ld=lld"); - } - - if let Ok(s) = env::var("RUSTC_HOST_CRT_STATIC") { - if s == "true" { - cmd.arg("-C").arg("target-feature=+crt-static"); - } - if s == "false" { - cmd.arg("-C").arg("target-feature=-crt-static"); - } - } - - // Cargo doesn't pass RUSTFLAGS to proc_macros: - // https://github.com/rust-lang/cargo/issues/4423 - // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. - // We also declare that the flag is expected, which we need to do to not - // get warnings about it being unexpected. - if stage == "0" { - cmd.arg("--cfg=bootstrap"); - } - cmd.arg("-Zunstable-options"); - cmd.arg("--check-cfg=values(bootstrap)"); - } - - if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") { - cmd.arg("--remap-path-prefix").arg(&map); - } - - // Force all crates compiled by this compiler to (a) be unstable and (b) - // allow the `rustc_private` feature to link to other unstable crates - // also in the sysroot. We also do this for host crates, since those - // may be proc macros, in which case we might ship them. - if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { - cmd.arg("-Z").arg("force-unstable-if-unmarked"); - } - - // allow-features is handled from within this rustc wrapper because of - // issues with build scripts. Some packages use build scripts to - // dynamically detect if certain nightly features are available. - // There are different ways this causes problems: - // - // * rustix runs `rustc` on a small test program to see if the feature is - // available (and sets a `cfg` if it is). It does not honor - // CARGO_ENCODED_RUSTFLAGS. - // * proc-macro2 detects if `rustc -vV` says "nighty" or "dev" and enables - // nightly features. It will scan CARGO_ENCODED_RUSTFLAGS for - // -Zallow-features. Unfortunately CARGO_ENCODED_RUSTFLAGS is not set - // for build-dependencies when --target is used. - // - // The issues above means we can't just use RUSTFLAGS, and we can't use - // `cargo -Zallow-features=…`. Passing it through here ensures that it - // always gets set. Unfortunately that also means we need to enable more - // features than we really want (like those for proc-macro2), but there - // isn't much of a way around it. - // - // I think it is unfortunate that build scripts are doing this at all, - // since changes to nightly features can cause crates to break even if the - // user didn't want or care about the use of the nightly features. I think - // nightly features should be opt-in only. Unfortunately the dynamic - // checks are now too wide spread that we just need to deal with it. - // - // If you want to try to remove this, I suggest working with the crate - // authors to remove the dynamic checking. Another option is to pursue - // https://github.com/rust-lang/cargo/issues/11244 and - // https://github.com/rust-lang/cargo/issues/4423, which will likely be - // very difficult, but could help expose -Zallow-features into build - // scripts so they could try to honor them. - if let Ok(allow_features) = env::var("RUSTC_ALLOW_FEATURES") { - cmd.arg(format!("-Zallow-features={allow_features}")); - } - - if let Ok(flags) = env::var("MAGIC_EXTRA_RUSTFLAGS") { - for flag in flags.split(' ') { - cmd.arg(flag); - } - } - - let is_test = args.iter().any(|a| a == "--test"); - if verbose > 2 { - let rust_env_vars = - env::vars().filter(|(k, _)| k.starts_with("RUST") || k.starts_with("CARGO")); - let prefix = if is_test { "[RUSTC-SHIM] rustc --test" } else { "[RUSTC-SHIM] rustc" }; - let prefix = match crate_name { - Some(crate_name) => format!("{prefix} {crate_name}"), - None => prefix.to_string(), - }; - for (i, (k, v)) in rust_env_vars.enumerate() { - eprintln!("{prefix} env[{i}]: {k:?}={v:?}"); - } - eprintln!("{} working directory: {}", prefix, env::current_dir().unwrap().display()); - eprintln!( - "{} command: {:?}={:?} {:?}", - prefix, - dylib_path_var(), - env::join_paths(&dylib_path).unwrap(), - cmd, - ); - eprintln!("{prefix} sysroot: {sysroot:?}"); - eprintln!("{prefix} libdir: {libdir:?}"); - } - - let start = Instant::now(); - let (child, status) = { - let errmsg = format!("\nFailed to run:\n{cmd:?}\n-------------"); - let mut child = cmd.spawn().expect(&errmsg); - let status = child.wait().expect(&errmsg); - (child, status) - }; - - if env::var_os("RUSTC_PRINT_STEP_TIMINGS").is_some() - || env::var_os("RUSTC_PRINT_STEP_RUSAGE").is_some() - { - if let Some(crate_name) = crate_name { - let dur = start.elapsed(); - // If the user requested resource usage data, then - // include that in addition to the timing output. - let rusage_data = - env::var_os("RUSTC_PRINT_STEP_RUSAGE").and_then(|_| format_rusage_data(child)); - eprintln!( - "[RUSTC-TIMING] {} test:{} {}.{:03}{}{}", - crate_name, - is_test, - dur.as_secs(), - dur.subsec_millis(), - if rusage_data.is_some() { " " } else { "" }, - rusage_data.unwrap_or(String::new()), - ); - } - } - - if status.success() { - std::process::exit(0); - // note: everything below here is unreachable. do not put code that - // should run on success, after this block. - } - if verbose > 0 { - println!("\nDid not run successfully: {status}\n{cmd:?}\n-------------"); - } - - if let Some(mut on_fail) = on_fail { - on_fail.status().expect("Could not run the on_fail command"); - } - - // Preserve the exit code. In case of signal, exit with 0xfe since it's - // awkward to preserve this status in a cross-platform way. - match status.code() { - Some(i) => std::process::exit(i), - None => { - eprintln!("rustc exited with {status}"); - std::process::exit(0xfe); - } - } -} - -#[cfg(all(not(unix), not(windows)))] -// In the future we can add this for more platforms -fn format_rusage_data(_child: Child) -> Option { - None -} - -#[cfg(windows)] -fn format_rusage_data(child: Child) -> Option { - use std::os::windows::io::AsRawHandle; - - use windows::{ - Win32::Foundation::HANDLE, - Win32::System::ProcessStatus::{ - K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS, PROCESS_MEMORY_COUNTERS_EX, - }, - Win32::System::Threading::GetProcessTimes, - Win32::System::Time::FileTimeToSystemTime, - }; - - let handle = HANDLE(child.as_raw_handle() as isize); - - let mut user_filetime = Default::default(); - let mut user_time = Default::default(); - let mut kernel_filetime = Default::default(); - let mut kernel_time = Default::default(); - let mut memory_counters = PROCESS_MEMORY_COUNTERS::default(); - - unsafe { - GetProcessTimes( - handle, - &mut Default::default(), - &mut Default::default(), - &mut kernel_filetime, - &mut user_filetime, - ) - } - .ok() - .ok()?; - unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok().ok()?; - unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok().ok()?; - - // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process - // with the given handle and none of that process's children. - unsafe { - K32GetProcessMemoryInfo( - handle, - &mut memory_counters, - std::mem::size_of::() as u32, - ) - } - .ok() - .ok()?; - - // Guide on interpreting these numbers: - // https://docs.microsoft.com/en-us/windows/win32/psapi/process-memory-usage-information - let peak_working_set = memory_counters.PeakWorkingSetSize / 1024; - let peak_page_file = memory_counters.PeakPagefileUsage / 1024; - let peak_paged_pool = memory_counters.QuotaPeakPagedPoolUsage / 1024; - let peak_nonpaged_pool = memory_counters.QuotaPeakNonPagedPoolUsage / 1024; - Some(format!( - "user: {USER_SEC}.{USER_USEC:03} \ - sys: {SYS_SEC}.{SYS_USEC:03} \ - peak working set (kb): {PEAK_WORKING_SET} \ - peak page file usage (kb): {PEAK_PAGE_FILE} \ - peak paged pool usage (kb): {PEAK_PAGED_POOL} \ - peak non-paged pool usage (kb): {PEAK_NONPAGED_POOL} \ - page faults: {PAGE_FAULTS}", - USER_SEC = user_time.wSecond + (user_time.wMinute * 60), - USER_USEC = user_time.wMilliseconds, - SYS_SEC = kernel_time.wSecond + (kernel_time.wMinute * 60), - SYS_USEC = kernel_time.wMilliseconds, - PEAK_WORKING_SET = peak_working_set, - PEAK_PAGE_FILE = peak_page_file, - PEAK_PAGED_POOL = peak_paged_pool, - PEAK_NONPAGED_POOL = peak_nonpaged_pool, - PAGE_FAULTS = memory_counters.PageFaultCount, - )) -} - -#[cfg(unix)] -/// Tries to build a string with human readable data for several of the rusage -/// fields. Note that we are focusing mainly on data that we believe to be -/// supplied on Linux (the `rusage` struct has other fields in it but they are -/// currently unsupported by Linux). -fn format_rusage_data(_child: Child) -> Option { - let rusage: libc::rusage = unsafe { - let mut recv = std::mem::zeroed(); - // -1 is RUSAGE_CHILDREN, which means to get the rusage for all children - // (and grandchildren, etc) processes that have respectively terminated - // and been waited for. - let retval = libc::getrusage(-1, &mut recv); - if retval != 0 { - return None; - } - recv - }; - // Mac OS X reports the maxrss in bytes, not kb. - let divisor = if env::consts::OS == "macos" { 1024 } else { 1 }; - let maxrss = (rusage.ru_maxrss + (divisor - 1)) / divisor; - - let mut init_str = format!( - "user: {USER_SEC}.{USER_USEC:03} \ - sys: {SYS_SEC}.{SYS_USEC:03} \ - max rss (kb): {MAXRSS}", - USER_SEC = rusage.ru_utime.tv_sec, - USER_USEC = rusage.ru_utime.tv_usec, - SYS_SEC = rusage.ru_stime.tv_sec, - SYS_USEC = rusage.ru_stime.tv_usec, - MAXRSS = maxrss - ); - - // The remaining rusage stats vary in platform support. So we treat - // uniformly zero values in each category as "not worth printing", since it - // either means no events of that type occurred, or that the platform - // does not support it. - - let minflt = rusage.ru_minflt; - let majflt = rusage.ru_majflt; - if minflt != 0 || majflt != 0 { - init_str.push_str(&format!(" page reclaims: {minflt} page faults: {majflt}")); - } - - let inblock = rusage.ru_inblock; - let oublock = rusage.ru_oublock; - if inblock != 0 || oublock != 0 { - init_str.push_str(&format!(" fs block inputs: {inblock} fs block outputs: {oublock}")); - } - - let nvcsw = rusage.ru_nvcsw; - let nivcsw = rusage.ru_nivcsw; - if nvcsw != 0 || nivcsw != 0 { - init_str.push_str(&format!( - " voluntary ctxt switches: {nvcsw} involuntary ctxt switches: {nivcsw}" - )); - } - - return Some(init_str); -} diff --git a/src/bootstrap/bin/rustdoc.rs b/src/bootstrap/bin/rustdoc.rs deleted file mode 100644 index 6561c1c19..000000000 --- a/src/bootstrap/bin/rustdoc.rs +++ /dev/null @@ -1,86 +0,0 @@ -//! Shim which is passed to Cargo as "rustdoc" when running the bootstrap. -//! -//! See comments in `src/bootstrap/rustc.rs` for more information. - -use std::env; -use std::ffi::OsString; -use std::path::PathBuf; -use std::process::{exit, Command}; - -include!("../dylib_util.rs"); - -include!("./_helper.rs"); - -fn main() { - let args = env::args_os().skip(1).collect::>(); - - let stage = parse_rustc_stage(); - let verbose = parse_rustc_verbose(); - - let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set"); - let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set"); - let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); - - // Detect whether or not we're a build script depending on whether --target - // is passed (a bit janky...) - let target = args.windows(2).find(|w| &*w[0] == "--target").and_then(|w| w[1].to_str()); - - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(libdir.clone())); - - let mut cmd = Command::new(rustdoc); - - if target.is_some() { - // The stage0 compiler has a special sysroot distinct from what we - // actually downloaded, so we just always pass the `--sysroot` option, - // unless one is already set. - if !args.iter().any(|arg| arg == "--sysroot") { - cmd.arg("--sysroot").arg(&sysroot); - } - } - - cmd.args(&args); - cmd.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - // Force all crates compiled by this compiler to (a) be unstable and (b) - // allow the `rustc_private` feature to link to other unstable crates - // also in the sysroot. - if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { - cmd.arg("-Z").arg("force-unstable-if-unmarked"); - } - if let Some(linker) = env::var_os("RUSTDOC_LINKER") { - let mut arg = OsString::from("-Clinker="); - arg.push(&linker); - cmd.arg(arg); - } - if let Ok(no_threads) = env::var("RUSTDOC_LLD_NO_THREADS") { - cmd.arg("-Clink-arg=-fuse-ld=lld"); - cmd.arg(format!("-Clink-arg=-Wl,{no_threads}")); - } - // Cargo doesn't pass RUSTDOCFLAGS to proc_macros: - // https://github.com/rust-lang/cargo/issues/4423 - // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. - // We also declare that the flag is expected, which we need to do to not - // get warnings about it being unexpected. - if stage == "0" { - cmd.arg("--cfg=bootstrap"); - } - cmd.arg("-Zunstable-options"); - cmd.arg("--check-cfg=values(bootstrap)"); - - if verbose > 1 { - eprintln!( - "rustdoc command: {:?}={:?} {:?}", - dylib_path_var(), - env::join_paths(&dylib_path).unwrap(), - cmd, - ); - eprintln!("sysroot: {sysroot:?}"); - eprintln!("libdir: {libdir:?}"); - } - - std::process::exit(match cmd.status() { - Ok(s) => s.code().unwrap_or(1), - Err(e) => panic!("\n\nfailed to run {cmd:?}: {e}\n\n"), - }) -} diff --git a/src/bootstrap/bin/sccache-plus-cl.rs b/src/bootstrap/bin/sccache-plus-cl.rs deleted file mode 100644 index 554c2dd4d..000000000 --- a/src/bootstrap/bin/sccache-plus-cl.rs +++ /dev/null @@ -1,38 +0,0 @@ -use std::env; -use std::process::{self, Command}; - -fn main() { - let target = env::var("SCCACHE_TARGET").unwrap(); - // Locate the actual compiler that we're invoking - env::set_var("CC", env::var_os("SCCACHE_CC").unwrap()); - env::set_var("CXX", env::var_os("SCCACHE_CXX").unwrap()); - let mut cfg = cc::Build::new(); - cfg.cargo_metadata(false) - .out_dir("/") - .target(&target) - .host(&target) - .opt_level(0) - .warnings(false) - .debug(false); - let compiler = cfg.get_compiler(); - - // Invoke sccache with said compiler - let sccache_path = env::var_os("SCCACHE_PATH").unwrap(); - let mut cmd = Command::new(&sccache_path); - cmd.arg(compiler.path()); - for &(ref k, ref v) in compiler.env() { - cmd.env(k, v); - } - for arg in env::args().skip(1) { - cmd.arg(arg); - } - - if let Ok(s) = env::var("SCCACHE_EXTRA_ARGS") { - for s in s.split_whitespace() { - cmd.arg(s); - } - } - - let status = cmd.status().expect("failed to spawn"); - process::exit(status.code().unwrap_or(2)) -} diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index fac0cdf20..5a84e37f8 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -211,7 +211,7 @@ def require(cmd, exit=True, exception=False): if exception: raise elif exit: - eprint("error: unable to run `{}`: {}".format(' '.join(cmd), exc)) + eprint("ERROR: unable to run `{}`: {}".format(' '.join(cmd), exc)) eprint("Please make sure it's installed and in the path.") sys.exit(1) return None @@ -681,7 +681,7 @@ class RustBuild(object): answer = self._should_fix_bins_and_dylibs = get_answer() if answer: - eprint("info: You seem to be using Nix.") + eprint("INFO: You seem to be using Nix.") return answer def fix_bin_or_dylib(self, fname): @@ -727,7 +727,7 @@ class RustBuild(object): "nix-build", "-E", nix_expr, "-o", nix_deps_dir, ]) except subprocess.CalledProcessError as reason: - eprint("warning: failed to call nix-build:", reason) + eprint("WARNING: failed to call nix-build:", reason) return self.nix_deps_dir = nix_deps_dir @@ -747,7 +747,7 @@ class RustBuild(object): try: subprocess.check_output([patchelf] + patchelf_args + [fname]) except subprocess.CalledProcessError as reason: - eprint("warning: failed to call patchelf:", reason) + eprint("WARNING: failed to call patchelf:", reason) return def rustc_stamp(self): @@ -954,6 +954,13 @@ class RustBuild(object): if deny_warnings: env["RUSTFLAGS"] += " -Dwarnings" + # Add RUSTFLAGS_BOOTSTRAP to RUSTFLAGS for bootstrap compilation. + # Note that RUSTFLAGS_BOOTSTRAP should always be added to the end of + # RUSTFLAGS to be actually effective (e.g., if we have `-Dwarnings` in + # RUSTFLAGS, passing `-Awarnings` from RUSTFLAGS_BOOTSTRAP should override it). + if "RUSTFLAGS_BOOTSTRAP" in env: + env["RUSTFLAGS"] += " " + env["RUSTFLAGS_BOOTSTRAP"] + env["PATH"] = os.path.join(self.bin_root(), "bin") + \ os.pathsep + env["PATH"] if not os.path.isfile(self.cargo()): @@ -998,7 +1005,7 @@ class RustBuild(object): if 'SUDO_USER' in os.environ and not self.use_vendored_sources: if os.getuid() == 0: self.use_vendored_sources = True - eprint('info: looks like you\'re trying to run this command as root') + eprint('INFO: looks like you\'re trying to run this command as root') eprint(' and so in order to preserve your $HOME this will now') eprint(' use vendored sources by default.') @@ -1010,14 +1017,14 @@ class RustBuild(object): "--sync ./src/tools/rust-analyzer/Cargo.toml " \ "--sync ./compiler/rustc_codegen_cranelift/Cargo.toml " \ "--sync ./src/bootstrap/Cargo.toml " - eprint('error: vendoring required, but vendor directory does not exist.') + eprint('ERROR: vendoring required, but vendor directory does not exist.') eprint(' Run `cargo vendor {}` to initialize the ' 'vendor directory.'.format(sync_dirs)) eprint('Alternatively, use the pre-vendored `rustc-src` dist component.') raise Exception("{} not found".format(vendor_dir)) if not os.path.exists(cargo_dir): - eprint('error: vendoring required, but .cargo/config does not exist.') + eprint('ERROR: vendoring required, but .cargo/config does not exist.') raise Exception("{} not found".format(cargo_dir)) else: if os.path.exists(cargo_dir): @@ -1042,6 +1049,12 @@ def bootstrap(args): """Configure, fetch, build and run the initial bootstrap""" rust_root = os.path.abspath(os.path.join(__file__, '../../..')) + if not os.path.exists(os.path.join(rust_root, '.git')) and \ + os.path.exists(os.path.join(rust_root, '.github')): + eprint("warn: Looks like you are trying to bootstrap Rust from a source that is neither a " + "git clone nor distributed tarball.\nThis build may fail due to missing submodules " + "unless you put them in place manually.") + # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, # then `config.toml` in the root directory. toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG') @@ -1112,7 +1125,7 @@ def main(): # process has to happen before anything is printed out. if help_triggered: eprint( - "info: Downloading and building bootstrap before processing --help command.\n" + "INFO: Downloading and building bootstrap before processing --help command.\n" " See src/bootstrap/README.md for help with common commands.") exit_code = 0 diff --git a/src/bootstrap/bootstrap_test.py b/src/bootstrap/bootstrap_test.py index dc06a4c97..6da410ed2 100644 --- a/src/bootstrap/bootstrap_test.py +++ b/src/bootstrap/bootstrap_test.py @@ -34,7 +34,7 @@ def serialize_and_parse(configure_args, bootstrap_args=None): # Verify this is actually valid TOML. tomllib.loads(build.config_toml) except ImportError: - print("warning: skipping TOML validation, need at least python 3.11", file=sys.stderr) + print("WARNING: skipping TOML validation, need at least python 3.11", file=sys.stderr) return build @@ -103,7 +103,6 @@ class GenerateAndParseConfig(unittest.TestCase): """Test that we can serialize and deserialize a config.toml file""" def test_no_args(self): build = serialize_and_parse([]) - self.assertEqual(build.get_toml("changelog-seen"), '2') self.assertEqual(build.get_toml("profile"), 'dist') self.assertIsNone(build.get_toml("llvm.download-ci-llvm")) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs deleted file mode 100644 index 46a62eed9..000000000 --- a/src/bootstrap/builder.rs +++ /dev/null @@ -1,2318 +0,0 @@ -use std::any::{type_name, Any}; -use std::cell::{Cell, RefCell}; -use std::collections::BTreeSet; -use std::env; -use std::ffi::OsStr; -use std::fmt::{Debug, Write}; -use std::fs::{self, File}; -use std::hash::Hash; -use std::io::{BufRead, BufReader}; -use std::ops::Deref; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::time::{Duration, Instant}; - -use crate::cache::{Cache, Interned, INTERNER}; -use crate::config::{DryRun, SplitDebuginfo, TargetSelection}; -use crate::doc; -use crate::flags::{Color, Subcommand}; -use crate::install; -use crate::llvm; -use crate::run; -use crate::setup; -use crate::test; -use crate::tool::{self, SourceType}; -use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; -use crate::EXTRA_CHECK_CFGS; -use crate::{check, compile, Crate}; -use crate::{clean, dist}; -use crate::{Build, CLang, DocTests, GitRepo, Mode}; - -pub use crate::Compiler; -// FIXME: -// - use std::lazy for `Lazy` -// - use std::cell for `OnceCell` -// Once they get stabilized and reach beta. -use clap::ValueEnum; -use once_cell::sync::{Lazy, OnceCell}; - -pub struct Builder<'a> { - pub build: &'a Build, - pub top_stage: u32, - pub kind: Kind, - cache: Cache, - stack: RefCell>>, - time_spent_on_dependencies: Cell, - pub paths: Vec, -} - -impl<'a> Deref for Builder<'a> { - type Target = Build; - - fn deref(&self) -> &Self::Target { - self.build - } -} - -pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { - /// `PathBuf` when directories are created or to return a `Compiler` once - /// it's been assembled. - type Output: Clone; - - /// Whether this step is run by default as part of its respective phase. - /// `true` here can still be overwritten by `should_run` calling `default_condition`. - const DEFAULT: bool = false; - - /// If true, then this rule should be skipped if --target was specified, but --host was not - const ONLY_HOSTS: bool = false; - - /// Primary function to execute this rule. Can call `builder.ensure()` - /// with other steps to run those. - fn run(self, builder: &Builder<'_>) -> Self::Output; - - /// When bootstrap is passed a set of paths, this controls whether this rule - /// will execute. However, it does not get called in a "default" context - /// when we are not passed any paths; in that case, `make_run` is called - /// directly. - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_>; - - /// Builds up a "root" rule, either as a default rule or from a path passed - /// to us. - /// - /// When path is `None`, we are executing in a context where no paths were - /// passed. When `./x.py build` is run, for example, this rule could get - /// called if it is in the correct list below with a path of `None`. - fn make_run(_run: RunConfig<'_>) { - // It is reasonable to not have an implementation of make_run for rules - // who do not want to get called from the root context. This means that - // they are likely dependencies (e.g., sysroot creation) or similar, and - // as such calling them from ./x.py isn't logical. - unimplemented!() - } -} - -pub struct RunConfig<'a> { - pub builder: &'a Builder<'a>, - pub target: TargetSelection, - pub paths: Vec, -} - -impl RunConfig<'_> { - pub fn build_triple(&self) -> TargetSelection { - self.builder.build.build - } - - /// Return a list of crate names selected by `run.paths`. - #[track_caller] - pub fn cargo_crates_in_set(&self) -> Interned> { - let mut crates = Vec::new(); - for krate in &self.paths { - let path = krate.assert_single_path(); - let Some(crate_name) = self.builder.crate_paths.get(&path.path) else { - panic!("missing crate for path {}", path.path.display()) - }; - crates.push(crate_name.to_string()); - } - INTERNER.intern_list(crates) - } - - /// Given an `alias` selected by the `Step` and the paths passed on the command line, - /// return a list of the crates that should be built. - /// - /// Normally, people will pass *just* `library` if they pass it. - /// But it's possible (although strange) to pass something like `library std core`. - /// Build all crates anyway, as if they hadn't passed the other args. - pub fn make_run_crates(&self, alias: Alias) -> Interned> { - let has_alias = - self.paths.iter().any(|set| set.assert_single_path().path.ends_with(alias.as_str())); - if !has_alias { - return self.cargo_crates_in_set(); - } - - let crates = match alias { - Alias::Library => self.builder.in_tree_crates("sysroot", Some(self.target)), - Alias::Compiler => self.builder.in_tree_crates("rustc-main", Some(self.target)), - }; - - let crate_names = crates.into_iter().map(|krate| krate.name.to_string()).collect(); - INTERNER.intern_list(crate_names) - } -} - -#[derive(Debug, Copy, Clone)] -pub enum Alias { - Library, - Compiler, -} - -impl Alias { - fn as_str(self) -> &'static str { - match self { - Alias::Library => "library", - Alias::Compiler => "compiler", - } - } -} - -/// A description of the crates in this set, suitable for passing to `builder.info`. -/// -/// `crates` should be generated by [`RunConfig::cargo_crates_in_set`]. -pub fn crate_description(crates: &[impl AsRef]) -> String { - if crates.is_empty() { - return "".into(); - } - - let mut descr = String::from(" {"); - descr.push_str(crates[0].as_ref()); - for krate in &crates[1..] { - descr.push_str(", "); - descr.push_str(krate.as_ref()); - } - descr.push('}'); - descr -} - -struct StepDescription { - default: bool, - only_hosts: bool, - should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>, - make_run: fn(RunConfig<'_>), - name: &'static str, - kind: Kind, -} - -#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)] -pub struct TaskPath { - pub path: PathBuf, - pub kind: Option, -} - -impl Debug for TaskPath { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(kind) = &self.kind { - write!(f, "{}::", kind.as_str())?; - } - write!(f, "{}", self.path.display()) - } -} - -/// Collection of paths used to match a task rule. -#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] -pub enum PathSet { - /// A collection of individual paths or aliases. - /// - /// These are generally matched as a path suffix. For example, a - /// command-line value of `std` will match if `library/std` is in the - /// set. - /// - /// NOTE: the paths within a set should always be aliases of one another. - /// For example, `src/librustdoc` and `src/tools/rustdoc` should be in the same set, - /// but `library/core` and `library/std` generally should not, unless there's no way (for that Step) - /// to build them separately. - Set(BTreeSet), - /// A "suite" of paths. - /// - /// These can match as a path suffix (like `Set`), or as a prefix. For - /// example, a command-line value of `tests/ui/abi/variadic-ffi.rs` - /// will match `tests/ui`. A command-line value of `ui` would also - /// match `tests/ui`. - Suite(TaskPath), -} - -impl PathSet { - fn empty() -> PathSet { - PathSet::Set(BTreeSet::new()) - } - - fn one>(path: P, kind: Kind) -> PathSet { - let mut set = BTreeSet::new(); - set.insert(TaskPath { path: path.into(), kind: Some(kind) }); - PathSet::Set(set) - } - - fn has(&self, needle: &Path, module: Kind) -> bool { - match self { - PathSet::Set(set) => set.iter().any(|p| Self::check(p, needle, module)), - PathSet::Suite(suite) => Self::check(suite, needle, module), - } - } - - // internal use only - fn check(p: &TaskPath, needle: &Path, module: Kind) -> bool { - if let Some(p_kind) = &p.kind { - p.path.ends_with(needle) && *p_kind == module - } else { - p.path.ends_with(needle) - } - } - - /// Return all `TaskPath`s in `Self` that contain any of the `needles`, removing the - /// matched needles. - /// - /// This is used for `StepDescription::krate`, which passes all matching crates at once to - /// `Step::make_run`, rather than calling it many times with a single crate. - /// See `tests.rs` for examples. - fn intersection_removing_matches(&self, needles: &mut Vec<&Path>, module: Kind) -> PathSet { - let mut check = |p| { - for (i, n) in needles.iter().enumerate() { - let matched = Self::check(p, n, module); - if matched { - needles.remove(i); - return true; - } - } - false - }; - match self { - PathSet::Set(set) => PathSet::Set(set.iter().filter(|&p| check(p)).cloned().collect()), - PathSet::Suite(suite) => { - if check(suite) { - self.clone() - } else { - PathSet::empty() - } - } - } - } - - /// A convenience wrapper for Steps which know they have no aliases and all their sets contain only a single path. - /// - /// This can be used with [`ShouldRun::crate_or_deps`], [`ShouldRun::path`], or [`ShouldRun::alias`]. - #[track_caller] - pub fn assert_single_path(&self) -> &TaskPath { - match self { - PathSet::Set(set) => { - assert_eq!(set.len(), 1, "called assert_single_path on multiple paths"); - set.iter().next().unwrap() - } - PathSet::Suite(_) => unreachable!("called assert_single_path on a Suite path"), - } - } -} - -impl StepDescription { - fn from(kind: Kind) -> StepDescription { - StepDescription { - default: S::DEFAULT, - only_hosts: S::ONLY_HOSTS, - should_run: S::should_run, - make_run: S::make_run, - name: std::any::type_name::(), - kind, - } - } - - fn maybe_run(&self, builder: &Builder<'_>, mut pathsets: Vec) { - pathsets.retain(|set| !self.is_excluded(builder, set)); - - if pathsets.is_empty() { - return; - } - - // Determine the targets participating in this rule. - let targets = if self.only_hosts { &builder.hosts } else { &builder.targets }; - - for target in targets { - let run = RunConfig { builder, paths: pathsets.clone(), target: *target }; - (self.make_run)(run); - } - } - - fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool { - if builder.config.skip.iter().any(|e| pathset.has(&e, builder.kind)) { - if !matches!(builder.config.dry_run, DryRun::SelfCheck) { - println!("Skipping {pathset:?} because it is excluded"); - } - return true; - } - - if !builder.config.skip.is_empty() && !matches!(builder.config.dry_run, DryRun::SelfCheck) { - builder.verbose(&format!( - "{:?} not skipped for {:?} -- not in {:?}", - pathset, self.name, builder.config.skip - )); - } - false - } - - fn run(v: &[StepDescription], builder: &Builder<'_>, paths: &[PathBuf]) { - let should_runs = v - .iter() - .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind))) - .collect::>(); - - // sanity checks on rules - for (desc, should_run) in v.iter().zip(&should_runs) { - assert!( - !should_run.paths.is_empty(), - "{:?} should have at least one pathset", - desc.name - ); - } - - if paths.is_empty() || builder.config.include_default_paths { - for (desc, should_run) in v.iter().zip(&should_runs) { - if desc.default && should_run.is_really_default() { - desc.maybe_run(builder, should_run.paths.iter().cloned().collect()); - } - } - } - - // strip CurDir prefix if present - let mut paths: Vec<_> = - paths.into_iter().map(|p| p.strip_prefix(".").unwrap_or(p)).collect(); - - // Handle all test suite paths. - // (This is separate from the loop below to avoid having to handle multiple paths in `is_suite_path` somehow.) - paths.retain(|path| { - for (desc, should_run) in v.iter().zip(&should_runs) { - if let Some(suite) = should_run.is_suite_path(&path) { - desc.maybe_run(builder, vec![suite.clone()]); - return false; - } - } - true - }); - - if paths.is_empty() { - return; - } - - // Handle all PathSets. - for (desc, should_run) in v.iter().zip(&should_runs) { - let pathsets = should_run.pathset_for_paths_removing_matches(&mut paths, desc.kind); - if !pathsets.is_empty() { - desc.maybe_run(builder, pathsets); - } - } - - if !paths.is_empty() { - eprintln!("error: no `{}` rules matched {:?}", builder.kind.as_str(), paths,); - eprintln!( - "help: run `x.py {} --help --verbose` to show a list of available paths", - builder.kind.as_str() - ); - eprintln!( - "note: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`" - ); - crate::exit!(1); - } - } -} - -enum ReallyDefault<'a> { - Bool(bool), - Lazy(Lazy bool + 'a>>), -} - -pub struct ShouldRun<'a> { - pub builder: &'a Builder<'a>, - kind: Kind, - - // use a BTreeSet to maintain sort order - paths: BTreeSet, - - // If this is a default rule, this is an additional constraint placed on - // its run. Generally something like compiler docs being enabled. - is_really_default: ReallyDefault<'a>, -} - -impl<'a> ShouldRun<'a> { - fn new(builder: &'a Builder<'_>, kind: Kind) -> ShouldRun<'a> { - ShouldRun { - builder, - kind, - paths: BTreeSet::new(), - is_really_default: ReallyDefault::Bool(true), // by default no additional conditions - } - } - - pub fn default_condition(mut self, cond: bool) -> Self { - self.is_really_default = ReallyDefault::Bool(cond); - self - } - - pub fn lazy_default_condition(mut self, lazy_cond: Box bool + 'a>) -> Self { - self.is_really_default = ReallyDefault::Lazy(Lazy::new(lazy_cond)); - self - } - - pub fn is_really_default(&self) -> bool { - match &self.is_really_default { - ReallyDefault::Bool(val) => *val, - ReallyDefault::Lazy(lazy) => *lazy.deref(), - } - } - - /// Indicates it should run if the command-line selects the given crate or - /// any of its (local) dependencies. - /// - /// `make_run` will be called a single time with all matching command-line paths. - pub fn crate_or_deps(self, name: &str) -> Self { - let crates = self.builder.in_tree_crates(name, None); - self.crates(crates) - } - - /// Indicates it should run if the command-line selects any of the given crates. - /// - /// `make_run` will be called a single time with all matching command-line paths. - /// - /// Prefer [`ShouldRun::crate_or_deps`] to this function where possible. - pub(crate) fn crates(mut self, crates: Vec<&Crate>) -> Self { - for krate in crates { - let path = krate.local_path(self.builder); - self.paths.insert(PathSet::one(path, self.kind)); - } - self - } - - // single alias, which does not correspond to any on-disk path - pub fn alias(mut self, alias: &str) -> Self { - // exceptional case for `Kind::Setup` because its `library` - // and `compiler` options would otherwise naively match with - // `compiler` and `library` folders respectively. - assert!( - self.kind == Kind::Setup || !self.builder.src.join(alias).exists(), - "use `builder.path()` for real paths: {alias}" - ); - self.paths.insert(PathSet::Set( - std::iter::once(TaskPath { path: alias.into(), kind: Some(self.kind) }).collect(), - )); - self - } - - // single, non-aliased path - pub fn path(self, path: &str) -> Self { - self.paths(&[path]) - } - - /// Multiple aliases for the same job. - /// - /// This differs from [`path`] in that multiple calls to path will end up calling `make_run` - /// multiple times, whereas a single call to `paths` will only ever generate a single call to - /// `paths`. - /// - /// This is analogous to `all_krates`, although `all_krates` is gone now. Prefer [`path`] where possible. - /// - /// [`path`]: ShouldRun::path - pub fn paths(mut self, paths: &[&str]) -> Self { - static SUBMODULES_PATHS: OnceCell> = OnceCell::new(); - - let init_submodules_paths = |src: &PathBuf| { - let file = File::open(src.join(".gitmodules")).unwrap(); - - let mut submodules_paths = vec![]; - for line in BufReader::new(file).lines() { - if let Ok(line) = line { - let line = line.trim(); - - if line.starts_with("path") { - let actual_path = - line.split(' ').last().expect("Couldn't get value of path"); - submodules_paths.push(actual_path.to_owned()); - } - } - } - - submodules_paths - }; - - let submodules_paths = - SUBMODULES_PATHS.get_or_init(|| init_submodules_paths(&self.builder.src)); - - self.paths.insert(PathSet::Set( - paths - .iter() - .map(|p| { - // assert only if `p` isn't submodule - if submodules_paths.iter().find(|sm_p| p.contains(*sm_p)).is_none() { - assert!( - self.builder.src.join(p).exists(), - "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", - p - ); - } - - TaskPath { path: p.into(), kind: Some(self.kind) } - }) - .collect(), - )); - self - } - - /// Handles individual files (not directories) within a test suite. - fn is_suite_path(&self, requested_path: &Path) -> Option<&PathSet> { - self.paths.iter().find(|pathset| match pathset { - PathSet::Suite(suite) => requested_path.starts_with(&suite.path), - PathSet::Set(_) => false, - }) - } - - pub fn suite_path(mut self, suite: &str) -> Self { - self.paths.insert(PathSet::Suite(TaskPath { path: suite.into(), kind: Some(self.kind) })); - self - } - - // allows being more explicit about why should_run in Step returns the value passed to it - pub fn never(mut self) -> ShouldRun<'a> { - self.paths.insert(PathSet::empty()); - self - } - - /// Given a set of requested paths, return the subset which match the Step for this `ShouldRun`, - /// removing the matches from `paths`. - /// - /// NOTE: this returns multiple PathSets to allow for the possibility of multiple units of work - /// within the same step. For example, `test::Crate` allows testing multiple crates in the same - /// cargo invocation, which are put into separate sets because they aren't aliases. - /// - /// The reason we return PathSet instead of PathBuf is to allow for aliases that mean the same thing - /// (for now, just `all_krates` and `paths`, but we may want to add an `aliases` function in the future?) - fn pathset_for_paths_removing_matches( - &self, - paths: &mut Vec<&Path>, - kind: Kind, - ) -> Vec { - let mut sets = vec![]; - for pathset in &self.paths { - let subset = pathset.intersection_removing_matches(paths, kind); - if subset != PathSet::empty() { - sets.push(subset); - } - } - sets - } -} - -#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] -pub enum Kind { - #[clap(alias = "b")] - Build, - #[clap(alias = "c")] - Check, - Clippy, - Fix, - Format, - #[clap(alias = "t")] - Test, - Bench, - #[clap(alias = "d")] - Doc, - Clean, - Dist, - Install, - #[clap(alias = "r")] - Run, - Setup, - Suggest, -} - -impl Kind { - pub fn parse(string: &str) -> Option { - // these strings, including the one-letter aliases, must match the x.py help text - Some(match string { - "build" | "b" => Kind::Build, - "check" | "c" => Kind::Check, - "clippy" => Kind::Clippy, - "fix" => Kind::Fix, - "fmt" => Kind::Format, - "test" | "t" => Kind::Test, - "bench" => Kind::Bench, - "doc" | "d" => Kind::Doc, - "clean" => Kind::Clean, - "dist" => Kind::Dist, - "install" => Kind::Install, - "run" | "r" => Kind::Run, - "setup" => Kind::Setup, - "suggest" => Kind::Suggest, - _ => return None, - }) - } - - pub fn as_str(&self) -> &'static str { - match self { - Kind::Build => "build", - Kind::Check => "check", - Kind::Clippy => "clippy", - Kind::Fix => "fix", - Kind::Format => "fmt", - Kind::Test => "test", - Kind::Bench => "bench", - Kind::Doc => "doc", - Kind::Clean => "clean", - Kind::Dist => "dist", - Kind::Install => "install", - Kind::Run => "run", - Kind::Setup => "setup", - Kind::Suggest => "suggest", - } - } - - pub fn description(&self) -> String { - match self { - Kind::Test => "Testing", - Kind::Bench => "Benchmarking", - Kind::Doc => "Documenting", - Kind::Run => "Running", - Kind::Suggest => "Suggesting", - _ => { - let title_letter = self.as_str()[0..1].to_ascii_uppercase(); - return format!("{title_letter}{}ing", &self.as_str()[1..]); - } - } - .to_owned() - } -} - -impl<'a> Builder<'a> { - fn get_step_descriptions(kind: Kind) -> Vec { - macro_rules! describe { - ($($rule:ty),+ $(,)?) => {{ - vec![$(StepDescription::from::<$rule>(kind)),+] - }}; - } - match kind { - Kind::Build => describe!( - compile::Std, - compile::Rustc, - compile::Assemble, - compile::CodegenBackend, - compile::StartupObjects, - tool::BuildManifest, - tool::Rustbook, - tool::ErrorIndex, - tool::UnstableBookGen, - tool::Tidy, - tool::Linkchecker, - tool::CargoTest, - tool::Compiletest, - tool::RemoteTestServer, - tool::RemoteTestClient, - tool::RustInstaller, - tool::Cargo, - tool::Rls, - tool::RustAnalyzer, - tool::RustAnalyzerProcMacroSrv, - tool::RustDemangler, - tool::Rustdoc, - tool::Clippy, - tool::CargoClippy, - llvm::Llvm, - llvm::Sanitizers, - tool::Rustfmt, - tool::Miri, - tool::CargoMiri, - llvm::Lld, - llvm::CrtBeginEnd, - tool::RustdocGUITest, - tool::OptimizedDist, - tool::CoverageDump, - ), - Kind::Check | Kind::Clippy | Kind::Fix => describe!( - check::Std, - check::Rustc, - check::Rustdoc, - check::CodegenBackend, - check::Clippy, - check::Miri, - check::CargoMiri, - check::MiroptTestTools, - check::Rls, - check::Rustfmt, - check::RustAnalyzer, - check::Bootstrap - ), - Kind::Test => describe!( - crate::toolstate::ToolStateCheck, - test::ExpandYamlAnchors, - test::Tidy, - test::Ui, - test::RunPassValgrind, - test::CoverageMap, - test::RunCoverage, - test::MirOpt, - test::Codegen, - test::CodegenUnits, - test::Assembly, - test::Incremental, - test::Debuginfo, - test::UiFullDeps, - test::CodegenCranelift, - test::Rustdoc, - test::RunCoverageRustdoc, - test::Pretty, - test::Crate, - test::CrateLibrustc, - test::CrateRustdoc, - test::CrateRustdocJsonTypes, - test::CrateBootstrap, - test::Linkcheck, - test::TierCheck, - test::Cargotest, - test::Cargo, - test::RustAnalyzer, - test::ErrorIndex, - test::Distcheck, - test::RunMakeFullDeps, - test::Nomicon, - test::Reference, - test::RustdocBook, - test::RustByExample, - test::TheBook, - test::UnstableBook, - test::RustcBook, - test::LintDocs, - test::RustcGuide, - test::EmbeddedBook, - test::EditionGuide, - test::Rustfmt, - test::Miri, - test::Clippy, - test::RustDemangler, - test::CompiletestTest, - test::RustdocJSStd, - test::RustdocJSNotStd, - test::RustdocGUI, - test::RustdocTheme, - test::RustdocUi, - test::RustdocJson, - test::HtmlCheck, - test::RustInstaller, - // Run bootstrap close to the end as it's unlikely to fail - test::Bootstrap, - // Run run-make last, since these won't pass without make on Windows - test::RunMake, - ), - Kind::Bench => describe!(test::Crate, test::CrateLibrustc), - Kind::Doc => describe!( - doc::UnstableBook, - doc::UnstableBookGen, - doc::TheBook, - doc::Standalone, - doc::Std, - doc::Rustc, - doc::Rustdoc, - doc::Rustfmt, - doc::ErrorIndex, - doc::Nomicon, - doc::Reference, - doc::RustdocBook, - doc::RustByExample, - doc::RustcBook, - doc::Cargo, - doc::CargoBook, - doc::Clippy, - doc::ClippyBook, - doc::Miri, - doc::EmbeddedBook, - doc::EditionGuide, - doc::StyleGuide, - doc::Tidy, - doc::Bootstrap, - ), - Kind::Dist => describe!( - dist::Docs, - dist::RustcDocs, - dist::JsonDocs, - dist::Mingw, - dist::Rustc, - dist::Std, - dist::RustcDev, - dist::Analysis, - dist::Src, - dist::Cargo, - dist::Rls, - dist::RustAnalyzer, - dist::Rustfmt, - dist::RustDemangler, - dist::Clippy, - dist::Miri, - dist::LlvmTools, - dist::RustDev, - dist::Bootstrap, - dist::Extended, - // It seems that PlainSourceTarball somehow changes how some of the tools - // perceive their dependencies (see #93033) which would invalidate fingerprints - // and force us to rebuild tools after vendoring dependencies. - // To work around this, create the Tarball after building all the tools. - dist::PlainSourceTarball, - dist::BuildManifest, - dist::ReproducibleArtifacts, - ), - Kind::Install => describe!( - install::Docs, - install::Std, - install::Cargo, - install::RustAnalyzer, - install::Rustfmt, - install::RustDemangler, - install::Clippy, - install::Miri, - install::LlvmTools, - install::Src, - install::Rustc - ), - Kind::Run => describe!( - run::ExpandYamlAnchors, - run::BuildManifest, - run::BumpStage0, - run::ReplaceVersionPlaceholder, - run::Miri, - run::CollectLicenseMetadata, - run::GenerateCopyright, - run::GenerateWindowsSys, - run::GenerateCompletions, - ), - Kind::Setup => describe!(setup::Profile, setup::Hook, setup::Link, setup::Vscode), - Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std), - // special-cased in Build::build() - Kind::Format | Kind::Suggest => vec![], - } - } - - pub fn get_help(build: &Build, kind: Kind) -> Option { - let step_descriptions = Builder::get_step_descriptions(kind); - if step_descriptions.is_empty() { - return None; - } - - let builder = Self::new_internal(build, kind, vec![]); - let builder = &builder; - // The "build" kind here is just a placeholder, it will be replaced with something else in - // the following statement. - let mut should_run = ShouldRun::new(builder, Kind::Build); - for desc in step_descriptions { - should_run.kind = desc.kind; - should_run = (desc.should_run)(should_run); - } - let mut help = String::from("Available paths:\n"); - let mut add_path = |path: &Path| { - t!(write!(help, " ./x.py {} {}\n", kind.as_str(), path.display())); - }; - for pathset in should_run.paths { - match pathset { - PathSet::Set(set) => { - for path in set { - add_path(&path.path); - } - } - PathSet::Suite(path) => { - add_path(&path.path.join("...")); - } - } - } - Some(help) - } - - fn new_internal(build: &Build, kind: Kind, paths: Vec) -> Builder<'_> { - Builder { - build, - top_stage: build.config.stage, - kind, - cache: Cache::new(), - stack: RefCell::new(Vec::new()), - time_spent_on_dependencies: Cell::new(Duration::new(0, 0)), - paths, - } - } - - pub fn new(build: &Build) -> Builder<'_> { - let paths = &build.config.paths; - let (kind, paths) = match build.config.cmd { - Subcommand::Build => (Kind::Build, &paths[..]), - Subcommand::Check { .. } => (Kind::Check, &paths[..]), - Subcommand::Clippy { .. } => (Kind::Clippy, &paths[..]), - Subcommand::Fix => (Kind::Fix, &paths[..]), - Subcommand::Doc { .. } => (Kind::Doc, &paths[..]), - Subcommand::Test { .. } => (Kind::Test, &paths[..]), - Subcommand::Bench { .. } => (Kind::Bench, &paths[..]), - Subcommand::Dist => (Kind::Dist, &paths[..]), - Subcommand::Install => (Kind::Install, &paths[..]), - Subcommand::Run { .. } => (Kind::Run, &paths[..]), - Subcommand::Clean { .. } => (Kind::Clean, &paths[..]), - Subcommand::Format { .. } => (Kind::Format, &[][..]), - Subcommand::Suggest { .. } => (Kind::Suggest, &[][..]), - Subcommand::Setup { profile: ref path } => ( - Kind::Setup, - path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)), - ), - }; - - Self::new_internal(build, kind, paths.to_owned()) - } - - pub fn execute_cli(&self) { - self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths); - } - - pub fn default_doc(&self, paths: &[PathBuf]) { - self.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), paths); - } - - pub fn doc_rust_lang_org_channel(&self) -> String { - let channel = match &*self.config.channel { - "stable" => &self.version, - "beta" => "beta", - "nightly" | "dev" => "nightly", - // custom build of rustdoc maybe? link to the latest stable docs just in case - _ => "stable", - }; - "https://doc.rust-lang.org/".to_owned() + channel - } - - fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) { - StepDescription::run(v, self, paths); - } - - /// Obtain a compiler at a given stage and for a given host. Explicitly does - /// not take `Compiler` since all `Compiler` instances are meant to be - /// obtained through this function, since it ensures that they are valid - /// (i.e., built and assembled). - pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler { - self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } }) - } - - /// Similar to `compiler`, except handles the full-bootstrap option to - /// silently use the stage1 compiler instead of a stage2 compiler if one is - /// requested. - /// - /// Note that this does *not* have the side effect of creating - /// `compiler(stage, host)`, unlike `compiler` above which does have such - /// a side effect. The returned compiler here can only be used to compile - /// new artifacts, it can't be used to rely on the presence of a particular - /// sysroot. - /// - /// See `force_use_stage1` and `force_use_stage2` for documentation on what each argument is. - pub fn compiler_for( - &self, - stage: u32, - host: TargetSelection, - target: TargetSelection, - ) -> Compiler { - if self.build.force_use_stage2(stage) { - self.compiler(2, self.config.build) - } else if self.build.force_use_stage1(stage, target) { - self.compiler(1, self.config.build) - } else { - self.compiler(stage, host) - } - } - - pub fn sysroot(&self, compiler: Compiler) -> Interned { - self.ensure(compile::Sysroot::new(compiler)) - } - - /// Returns the libdir where the standard library and other artifacts are - /// found for a compiler's sysroot. - pub fn sysroot_libdir(&self, compiler: Compiler, target: TargetSelection) -> Interned { - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - struct Libdir { - compiler: Compiler, - target: TargetSelection, - } - impl Step for Libdir { - type Output = Interned; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> Interned { - let lib = builder.sysroot_libdir_relative(self.compiler); - let sysroot = builder - .sysroot(self.compiler) - .join(lib) - .join("rustlib") - .join(self.target.triple) - .join("lib"); - // Avoid deleting the rustlib/ directory we just copied - // (in `impl Step for Sysroot`). - if !builder.download_rustc() { - builder.verbose(&format!( - "Removing sysroot {} to avoid caching bugs", - sysroot.display() - )); - let _ = fs::remove_dir_all(&sysroot); - t!(fs::create_dir_all(&sysroot)); - } - - if self.compiler.stage == 0 { - // The stage 0 compiler for the build triple is always pre-built. - // Ensure that `libLLVM.so` ends up in the target libdir, so that ui-fulldeps tests can use it when run. - dist::maybe_install_llvm_target( - builder, - self.compiler.host, - &builder.sysroot(self.compiler), - ); - } - - INTERNER.intern_path(sysroot) - } - } - self.ensure(Libdir { compiler, target }) - } - - pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf { - self.sysroot_libdir(compiler, compiler.host).with_file_name("codegen-backends") - } - - /// Returns the compiler's libdir where it stores the dynamic libraries that - /// it itself links against. - /// - /// For example this returns `/lib` on Unix and `/bin` on - /// Windows. - pub fn rustc_libdir(&self, compiler: Compiler) -> PathBuf { - if compiler.is_snapshot(self) { - self.rustc_snapshot_libdir() - } else { - match self.config.libdir_relative() { - Some(relative_libdir) if compiler.stage >= 1 => { - self.sysroot(compiler).join(relative_libdir) - } - _ => self.sysroot(compiler).join(libdir(compiler.host)), - } - } - } - - /// Returns the compiler's relative libdir where it stores the dynamic libraries that - /// it itself links against. - /// - /// For example this returns `lib` on Unix and `bin` on - /// Windows. - pub fn libdir_relative(&self, compiler: Compiler) -> &Path { - if compiler.is_snapshot(self) { - libdir(self.config.build).as_ref() - } else { - match self.config.libdir_relative() { - Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, - _ => libdir(compiler.host).as_ref(), - } - } - } - - /// Returns the compiler's relative libdir where the standard library and other artifacts are - /// found for a compiler's sysroot. - /// - /// For example this returns `lib` on Unix and Windows. - pub fn sysroot_libdir_relative(&self, compiler: Compiler) -> &Path { - match self.config.libdir_relative() { - Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, - _ if compiler.stage == 0 => &self.build.initial_libdir, - _ => Path::new("lib"), - } - } - - pub fn rustc_lib_paths(&self, compiler: Compiler) -> Vec { - let mut dylib_dirs = vec![self.rustc_libdir(compiler)]; - - // Ensure that the downloaded LLVM libraries can be found. - if self.config.llvm_from_ci { - let ci_llvm_lib = self.out.join(&*compiler.host.triple).join("ci-llvm").join("lib"); - dylib_dirs.push(ci_llvm_lib); - } - - dylib_dirs - } - - /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic - /// library lookup path. - pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Command) { - // Windows doesn't need dylib path munging because the dlls for the - // compiler live next to the compiler and the system will find them - // automatically. - if cfg!(windows) { - return; - } - - add_dylib_path(self.rustc_lib_paths(compiler), cmd); - } - - /// Gets a path to the compiler specified. - pub fn rustc(&self, compiler: Compiler) -> PathBuf { - if compiler.is_snapshot(self) { - self.initial_rustc.clone() - } else { - self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host)) - } - } - - /// Gets the paths to all of the compiler's codegen backends. - fn codegen_backends(&self, compiler: Compiler) -> impl Iterator { - fs::read_dir(self.sysroot_codegen_backends(compiler)) - .into_iter() - .flatten() - .filter_map(Result::ok) - .map(|entry| entry.path()) - } - - pub fn rustdoc(&self, compiler: Compiler) -> PathBuf { - self.ensure(tool::Rustdoc { compiler }) - } - - pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command { - let mut cmd = Command::new(&self.bootstrap_out.join("rustdoc")); - cmd.env("RUSTC_STAGE", compiler.stage.to_string()) - .env("RUSTC_SYSROOT", self.sysroot(compiler)) - // Note that this is *not* the sysroot_libdir because rustdoc must be linked - // equivalently to rustc. - .env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)) - .env("CFG_RELEASE_CHANNEL", &self.config.channel) - .env("RUSTDOC_REAL", self.rustdoc(compiler)) - .env("RUSTC_BOOTSTRAP", "1"); - - cmd.arg("-Wrustdoc::invalid_codeblock_attributes"); - - if self.config.deny_warnings { - cmd.arg("-Dwarnings"); - } - cmd.arg("-Znormalize-docs"); - - // Remove make-related flags that can cause jobserver problems. - cmd.env_remove("MAKEFLAGS"); - cmd.env_remove("MFLAGS"); - - if let Some(linker) = self.linker(compiler.host) { - cmd.env("RUSTDOC_LINKER", linker); - } - if self.is_fuse_ld_lld(compiler.host) { - cmd.env("RUSTDOC_FUSE_LD_LLD", "1"); - } - cmd - } - - /// Return the path to `llvm-config` for the target, if it exists. - /// - /// Note that this returns `None` if LLVM is disabled, or if we're in a - /// check build or dry-run, where there's no need to build all of LLVM. - fn llvm_config(&self, target: TargetSelection) -> Option { - if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() { - let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target }); - if llvm_config.is_file() { - return Some(llvm_config); - } - } - None - } - - /// Like `cargo`, but only passes flags that are valid for all commands. - pub fn bare_cargo( - &self, - compiler: Compiler, - mode: Mode, - target: TargetSelection, - cmd: &str, - ) -> Command { - let mut cargo = Command::new(&self.initial_cargo); - // Run cargo from the source root so it can find .cargo/config. - // This matters when using vendoring and the working directory is outside the repository. - cargo.current_dir(&self.src); - - let out_dir = self.stage_out(compiler, mode); - cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd); - - // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger` - // from out of tree it shouldn't matter, since x.py is only used for - // building in-tree. - let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"]; - match self.build.config.color { - Color::Always => { - cargo.arg("--color=always"); - for log in &color_logs { - cargo.env(log, "always"); - } - } - Color::Never => { - cargo.arg("--color=never"); - for log in &color_logs { - cargo.env(log, "never"); - } - } - Color::Auto => {} // nothing to do - } - - if cmd != "install" { - cargo.arg("--target").arg(target.rustc_target_arg()); - } else { - assert_eq!(target, compiler.host); - } - - if self.config.rust_optimize.is_release() { - // FIXME: cargo bench/install do not accept `--release` - if cmd != "bench" && cmd != "install" { - cargo.arg("--release"); - } - } - - // Remove make-related flags to ensure Cargo can correctly set things up - cargo.env_remove("MAKEFLAGS"); - cargo.env_remove("MFLAGS"); - - cargo - } - - /// Prepares an invocation of `cargo` to be run. - /// - /// This will create a `Command` that represents a pending execution of - /// Cargo. This cargo will be configured to use `compiler` as the actual - /// rustc compiler, its output will be scoped by `mode`'s output directory, - /// it will pass the `--target` flag for the specified `target`, and will be - /// executing the Cargo command `cmd`. - pub fn cargo( - &self, - compiler: Compiler, - mode: Mode, - source_type: SourceType, - target: TargetSelection, - cmd: &str, - ) -> Cargo { - let mut cargo = self.bare_cargo(compiler, mode, target, cmd); - let out_dir = self.stage_out(compiler, mode); - - // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, - // so we need to explicitly clear out if they've been updated. - for backend in self.codegen_backends(compiler) { - self.clear_if_dirty(&out_dir, &backend); - } - - if cmd == "doc" || cmd == "rustdoc" { - let my_out = match mode { - // This is the intended out directory for compiler documentation. - Mode::Rustc | Mode::ToolRustc => self.compiler_doc_out(target), - Mode::Std => { - if self.config.cmd.json() { - out_dir.join(target.triple).join("json-doc") - } else { - out_dir.join(target.triple).join("doc") - } - } - _ => panic!("doc mode {mode:?} not expected"), - }; - let rustdoc = self.rustdoc(compiler); - self.clear_if_dirty(&my_out, &rustdoc); - } - - let profile_var = |name: &str| { - let profile = if self.config.rust_optimize.is_release() { "RELEASE" } else { "DEV" }; - format!("CARGO_PROFILE_{}_{}", profile, name) - }; - - // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config - // needs to not accidentally link to libLLVM in stage0/lib. - cargo.env("REAL_LIBRARY_PATH_VAR", &util::dylib_path_var()); - if let Some(e) = env::var_os(util::dylib_path_var()) { - cargo.env("REAL_LIBRARY_PATH", e); - } - - // Set a flag for `check`/`clippy`/`fix`, so that certain build - // scripts can do less work (i.e. not building/requiring LLVM). - if cmd == "check" || cmd == "clippy" || cmd == "fix" { - // If we've not yet built LLVM, or it's stale, then bust - // the rustc_llvm cache. That will always work, even though it - // may mean that on the next non-check build we'll need to rebuild - // rustc_llvm. But if LLVM is stale, that'll be a tiny amount - // of work comparatively, and we'd likely need to rebuild it anyway, - // so that's okay. - if crate::llvm::prebuilt_llvm_config(self, target).is_err() { - cargo.env("RUST_CHECK", "1"); - } - } - - let stage = if compiler.stage == 0 && self.local_rebuild { - // Assume the local-rebuild rustc already has stage1 features. - 1 - } else { - compiler.stage - }; - - let mut rustflags = Rustflags::new(target); - if stage != 0 { - if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") { - cargo.args(s.split_whitespace()); - } - rustflags.env("RUSTFLAGS_NOT_BOOTSTRAP"); - } else { - if let Ok(s) = env::var("CARGOFLAGS_BOOTSTRAP") { - cargo.args(s.split_whitespace()); - } - rustflags.env("RUSTFLAGS_BOOTSTRAP"); - if cmd == "clippy" { - // clippy overwrites sysroot if we pass it to cargo. - // Pass it directly to clippy instead. - // NOTE: this can't be fixed in clippy because we explicitly don't set `RUSTC`, - // so it has no way of knowing the sysroot. - rustflags.arg("--sysroot"); - rustflags.arg( - self.sysroot(compiler) - .as_os_str() - .to_str() - .expect("sysroot must be valid UTF-8"), - ); - // Only run clippy on a very limited subset of crates (in particular, not build scripts). - cargo.arg("-Zunstable-options"); - // Explicitly does *not* set `--cfg=bootstrap`, since we're using a nightly clippy. - let host_version = Command::new("rustc").arg("--version").output().map_err(|_| ()); - let output = host_version.and_then(|output| { - if output.status.success() { - Ok(output) - } else { - Err(()) - } - }).unwrap_or_else(|_| { - eprintln!( - "error: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component" - ); - eprintln!("help: try `rustup component add clippy`"); - crate::exit!(1); - }); - if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") { - rustflags.arg("--cfg=bootstrap"); - } - } else { - rustflags.arg("--cfg=bootstrap"); - } - } - - let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling { - Some(setting) => { - // If an explicit setting is given, use that - setting - } - None => { - if mode == Mode::Std { - // The standard library defaults to the legacy scheme - false - } else { - // The compiler and tools default to the new scheme - true - } - } - }; - - // By default, windows-rs depends on a native library that doesn't get copied into the - // sysroot. Passing this cfg enables raw-dylib support instead, which makes the native - // library unnecessary. This can be removed when windows-rs enables raw-dylib - // unconditionally. - if let Mode::Rustc | Mode::ToolRustc = mode { - rustflags.arg("--cfg=windows_raw_dylib"); - } - - if use_new_symbol_mangling { - rustflags.arg("-Csymbol-mangling-version=v0"); - } else { - rustflags.arg("-Csymbol-mangling-version=legacy"); - rustflags.arg("-Zunstable-options"); - } - - // Enable cfg checking of cargo features for everything but std and also enable cfg - // checking of names and values. - // - // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like - // backtrace, core_simd, std_float, ...), those dependencies have their own - // features but cargo isn't involved in the #[path] process and so cannot pass the - // complete list of features, so for that reason we don't enable checking of - // features for std crates. - cargo.arg(if mode != Mode::Std { - "-Zcheck-cfg=names,values,output,features" - } else { - "-Zcheck-cfg=names,values,output" - }); - - // Add extra cfg not defined in/by rustc - // - // Note: Although it would seems that "-Zunstable-options" to `rustflags` is useless as - // cargo would implicitly add it, it was discover that sometimes bootstrap only use - // `rustflags` without `cargo` making it required. - rustflags.arg("-Zunstable-options"); - for (restricted_mode, name, values) in EXTRA_CHECK_CFGS { - if *restricted_mode == None || *restricted_mode == Some(mode) { - // Creating a string of the values by concatenating each value: - // ',"tvos","watchos"' or '' (nothing) when there are no values - let values = match values { - Some(values) => values - .iter() - .map(|val| [",", "\"", val, "\""]) - .flatten() - .collect::(), - None => String::new(), - }; - rustflags.arg(&format!("--check-cfg=values({name}{values})")); - } - } - - // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`, - // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See - // #71458. - let mut rustdocflags = rustflags.clone(); - rustdocflags.propagate_cargo_env("RUSTDOCFLAGS"); - if stage == 0 { - rustdocflags.env("RUSTDOCFLAGS_BOOTSTRAP"); - } else { - rustdocflags.env("RUSTDOCFLAGS_NOT_BOOTSTRAP"); - } - - if let Ok(s) = env::var("CARGOFLAGS") { - cargo.args(s.split_whitespace()); - } - - match mode { - Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {} - Mode::Rustc | Mode::Codegen | Mode::ToolRustc => { - // Build proc macros both for the host and the target - if target != compiler.host && cmd != "check" { - cargo.arg("-Zdual-proc-macros"); - rustflags.arg("-Zdual-proc-macros"); - } - } - } - - // This tells Cargo (and in turn, rustc) to output more complete - // dependency information. Most importantly for rustbuild, this - // includes sysroot artifacts, like libstd, which means that we don't - // need to track those in rustbuild (an error prone process!). This - // feature is currently unstable as there may be some bugs and such, but - // it represents a big improvement in rustbuild's reliability on - // rebuilds, so we're using it here. - // - // For some additional context, see #63470 (the PR originally adding - // this), as well as #63012 which is the tracking issue for this - // feature on the rustc side. - cargo.arg("-Zbinary-dep-depinfo"); - let allow_features = match mode { - Mode::ToolBootstrap | Mode::ToolStd => { - // Restrict the allowed features so we don't depend on nightly - // accidentally. - // - // binary-dep-depinfo is used by rustbuild itself for all - // compilations. - // - // Lots of tools depend on proc_macro2 and proc-macro-error. - // Those have build scripts which assume nightly features are - // available if the `rustc` version is "nighty" or "dev". See - // bin/rustc.rs for why that is a problem. Instead of labeling - // those features for each individual tool that needs them, - // just blanket allow them here. - // - // If this is ever removed, be sure to add something else in - // its place to keep the restrictions in place (or make a way - // to unset RUSTC_BOOTSTRAP). - "binary-dep-depinfo,proc_macro_span,proc_macro_span_shrink,proc_macro_diagnostic" - .to_string() - } - Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => String::new(), - }; - - cargo.arg("-j").arg(self.jobs().to_string()); - - // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005 - // Force cargo to output binaries with disambiguating hashes in the name - let mut metadata = if compiler.stage == 0 { - // Treat stage0 like a special channel, whether it's a normal prior- - // release rustc or a local rebuild with the same version, so we - // never mix these libraries by accident. - "bootstrap".to_string() - } else { - self.config.channel.to_string() - }; - // We want to make sure that none of the dependencies between - // std/test/rustc unify with one another. This is done for weird linkage - // reasons but the gist of the problem is that if librustc, libtest, and - // libstd all depend on libc from crates.io (which they actually do) we - // want to make sure they all get distinct versions. Things get really - // weird if we try to unify all these dependencies right now, namely - // around how many times the library is linked in dynamic libraries and - // such. If rustc were a static executable or if we didn't ship dylibs - // this wouldn't be a problem, but we do, so it is. This is in general - // just here to make sure things build right. If you can remove this and - // things still build right, please do! - match mode { - Mode::Std => metadata.push_str("std"), - // When we're building rustc tools, they're built with a search path - // that contains things built during the rustc build. For example, - // bitflags is built during the rustc build, and is a dependency of - // rustdoc as well. We're building rustdoc in a different target - // directory, though, which means that Cargo will rebuild the - // dependency. When we go on to build rustdoc, we'll look for - // bitflags, and find two different copies: one built during the - // rustc step and one that we just built. This isn't always a - // problem, somehow -- not really clear why -- but we know that this - // fixes things. - Mode::ToolRustc => metadata.push_str("tool-rustc"), - // Same for codegen backends. - Mode::Codegen => metadata.push_str("codegen"), - _ => {} - } - cargo.env("__CARGO_DEFAULT_LIB_METADATA", &metadata); - - if cmd == "clippy" { - rustflags.arg("-Zforce-unstable-if-unmarked"); - } - - rustflags.arg("-Zmacro-backtrace"); - - let want_rustdoc = self.doc_tests != DocTests::No; - - // We synthetically interpret a stage0 compiler used to build tools as a - // "raw" compiler in that it's the exact snapshot we download. Normally - // the stage0 build means it uses libraries build by the stage0 - // compiler, but for tools we just use the precompiled libraries that - // we've downloaded - let use_snapshot = mode == Mode::ToolBootstrap; - assert!(!use_snapshot || stage == 0 || self.local_rebuild); - - let maybe_sysroot = self.sysroot(compiler); - let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot }; - let libdir = self.rustc_libdir(compiler); - - // Clear the output directory if the real rustc we're using has changed; - // Cargo cannot detect this as it thinks rustc is bootstrap/debug/rustc. - // - // Avoid doing this during dry run as that usually means the relevant - // compiler is not yet linked/copied properly. - // - // Only clear out the directory if we're compiling std; otherwise, we - // should let Cargo take care of things for us (via depdep info) - if !self.config.dry_run() && mode == Mode::Std && cmd == "build" { - self.clear_if_dirty(&out_dir, &self.rustc(compiler)); - } - - // Customize the compiler we're running. Specify the compiler to cargo - // as our shim and then pass it some various options used to configure - // how the actual compiler itself is called. - // - // These variables are primarily all read by - // src/bootstrap/bin/{rustc.rs,rustdoc.rs} - cargo - .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target)) - .env("RUSTC_REAL", self.rustc(compiler)) - .env("RUSTC_STAGE", stage.to_string()) - .env("RUSTC_SYSROOT", &sysroot) - .env("RUSTC_LIBDIR", &libdir) - .env("RUSTDOC", self.bootstrap_out.join("rustdoc")) - .env( - "RUSTDOC_REAL", - if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) { - self.rustdoc(compiler) - } else { - PathBuf::from("/path/to/nowhere/rustdoc/not/required") - }, - ) - .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir()) - .env("RUSTC_BREAK_ON_ICE", "1"); - // Clippy support is a hack and uses the default `cargo-clippy` in path. - // Don't override RUSTC so that the `cargo-clippy` in path will be run. - if cmd != "clippy" { - cargo.env("RUSTC", self.bootstrap_out.join("rustc")); - } - - // Dealing with rpath here is a little special, so let's go into some - // detail. First off, `-rpath` is a linker option on Unix platforms - // which adds to the runtime dynamic loader path when looking for - // dynamic libraries. We use this by default on Unix platforms to ensure - // that our nightlies behave the same on Windows, that is they work out - // of the box. This can be disabled by setting `rpath = false` in `[rust]` - // table of `config.toml` - // - // Ok, so the astute might be wondering "why isn't `-C rpath` used - // here?" and that is indeed a good question to ask. This codegen - // option is the compiler's current interface to generating an rpath. - // Unfortunately it doesn't quite suffice for us. The flag currently - // takes no value as an argument, so the compiler calculates what it - // should pass to the linker as `-rpath`. This unfortunately is based on - // the **compile time** directory structure which when building with - // Cargo will be very different than the runtime directory structure. - // - // All that's a really long winded way of saying that if we use - // `-Crpath` then the executables generated have the wrong rpath of - // something like `$ORIGIN/deps` when in fact the way we distribute - // rustc requires the rpath to be `$ORIGIN/../lib`. - // - // So, all in all, to set up the correct rpath we pass the linker - // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it - // fun to pass a flag to a tool to pass a flag to pass a flag to a tool - // to change a flag in a binary? - if self.config.rpath_enabled(target) && util::use_host_linker(target) { - let libdir = self.sysroot_libdir_relative(compiler).to_str().unwrap(); - let rpath = if target.contains("apple") { - // Note that we need to take one extra step on macOS to also pass - // `-Wl,-instal_name,@rpath/...` to get things to work right. To - // do that we pass a weird flag to the compiler to get it to do - // so. Note that this is definitely a hack, and we should likely - // flesh out rpath support more fully in the future. - rustflags.arg("-Zosx-rpath-install-name"); - Some(format!("-Wl,-rpath,@loader_path/../{libdir}")) - } else if !target.contains("windows") - && !target.contains("aix") - && !target.contains("xous") - { - rustflags.arg("-Clink-args=-Wl,-z,origin"); - Some(format!("-Wl,-rpath,$ORIGIN/../{libdir}")) - } else { - None - }; - if let Some(rpath) = rpath { - rustflags.arg(&format!("-Clink-args={rpath}")); - } - } - - if let Some(host_linker) = self.linker(compiler.host) { - cargo.env("RUSTC_HOST_LINKER", host_linker); - } - if self.is_fuse_ld_lld(compiler.host) { - cargo.env("RUSTC_HOST_FUSE_LD_LLD", "1"); - cargo.env("RUSTDOC_FUSE_LD_LLD", "1"); - } - - if let Some(target_linker) = self.linker(target) { - let target = crate::envify(&target.triple); - cargo.env(&format!("CARGO_TARGET_{target}_LINKER"), target_linker); - } - if self.is_fuse_ld_lld(target) { - rustflags.arg("-Clink-args=-fuse-ld=lld"); - } - self.lld_flags(target).for_each(|flag| { - rustdocflags.arg(&flag); - }); - - if !(["build", "check", "clippy", "fix", "rustc"].contains(&cmd)) && want_rustdoc { - cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)); - } - - let debuginfo_level = match mode { - Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc, - Mode::Std => self.config.rust_debuginfo_level_std, - Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => { - self.config.rust_debuginfo_level_tools - } - }; - cargo.env(profile_var("DEBUG"), debuginfo_level.to_string()); - if let Some(opt_level) = &self.config.rust_optimize.get_opt_level() { - cargo.env(profile_var("OPT_LEVEL"), opt_level); - } - if !self.config.dry_run() && self.cc.borrow()[&target].args().iter().any(|arg| arg == "-gz") - { - rustflags.arg("-Clink-arg=-gz"); - } - cargo.env( - profile_var("DEBUG_ASSERTIONS"), - if mode == Mode::Std { - self.config.rust_debug_assertions_std.to_string() - } else { - self.config.rust_debug_assertions.to_string() - }, - ); - cargo.env( - profile_var("OVERFLOW_CHECKS"), - if mode == Mode::Std { - self.config.rust_overflow_checks_std.to_string() - } else { - self.config.rust_overflow_checks.to_string() - }, - ); - - let split_debuginfo_is_stable = target.contains("linux") - || target.contains("apple") - || (target.contains("msvc") - && self.config.rust_split_debuginfo == SplitDebuginfo::Packed) - || (target.contains("windows") - && self.config.rust_split_debuginfo == SplitDebuginfo::Off); - - if !split_debuginfo_is_stable { - rustflags.arg("-Zunstable-options"); - } - match self.config.rust_split_debuginfo { - SplitDebuginfo::Packed => rustflags.arg("-Csplit-debuginfo=packed"), - SplitDebuginfo::Unpacked => rustflags.arg("-Csplit-debuginfo=unpacked"), - SplitDebuginfo::Off => rustflags.arg("-Csplit-debuginfo=off"), - }; - - if self.config.cmd.bless() { - // Bless `expect!` tests. - cargo.env("UPDATE_EXPECT", "1"); - } - - if !mode.is_tool() { - cargo.env("RUSTC_FORCE_UNSTABLE", "1"); - } - - if let Some(x) = self.crt_static(target) { - if x { - rustflags.arg("-Ctarget-feature=+crt-static"); - } else { - rustflags.arg("-Ctarget-feature=-crt-static"); - } - } - - if let Some(x) = self.crt_static(compiler.host) { - cargo.env("RUSTC_HOST_CRT_STATIC", x.to_string()); - } - - if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) { - let map = format!("{}={}", self.build.src.display(), map_to); - cargo.env("RUSTC_DEBUGINFO_MAP", map); - - // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to, - // in order to opportunistically reverse it later. - cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); - } - - // Enable usage of unstable features - cargo.env("RUSTC_BOOTSTRAP", "1"); - self.add_rust_test_threads(&mut cargo); - - // Almost all of the crates that we compile as part of the bootstrap may - // have a build script, including the standard library. To compile a - // build script, however, it itself needs a standard library! This - // introduces a bit of a pickle when we're compiling the standard - // library itself. - // - // To work around this we actually end up using the snapshot compiler - // (stage0) for compiling build scripts of the standard library itself. - // The stage0 compiler is guaranteed to have a libstd available for use. - // - // For other crates, however, we know that we've already got a standard - // library up and running, so we can use the normal compiler to compile - // build scripts in that situation. - if mode == Mode::Std { - cargo - .env("RUSTC_SNAPSHOT", &self.initial_rustc) - .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()); - } else { - cargo - .env("RUSTC_SNAPSHOT", self.rustc(compiler)) - .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler)); - } - - // Tools that use compiler libraries may inherit the `-lLLVM` link - // requirement, but the `-L` library path is not propagated across - // separate Cargo projects. We can add LLVM's library path to the - // platform-specific environment variable as a workaround. - if mode == Mode::ToolRustc || mode == Mode::Codegen { - if let Some(llvm_config) = self.llvm_config(target) { - let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); - add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cargo); - } - } - - // Compile everything except libraries and proc macros with the more - // efficient initial-exec TLS model. This doesn't work with `dlopen`, - // so we can't use it by default in general, but we can use it for tools - // and our own internal libraries. - if !mode.must_support_dlopen() && !target.triple.starts_with("powerpc-") { - cargo.env("RUSTC_TLS_MODEL_INITIAL_EXEC", "1"); - } - - // Ignore incremental modes except for stage0, since we're - // not guaranteeing correctness across builds if the compiler - // is changing under your feet. - if self.config.incremental && compiler.stage == 0 { - cargo.env("CARGO_INCREMENTAL", "1"); - } else { - // Don't rely on any default setting for incr. comp. in Cargo - cargo.env("CARGO_INCREMENTAL", "0"); - } - - if let Some(ref on_fail) = self.config.on_fail { - cargo.env("RUSTC_ON_FAIL", on_fail); - } - - if self.config.print_step_timings { - cargo.env("RUSTC_PRINT_STEP_TIMINGS", "1"); - } - - if self.config.print_step_rusage { - cargo.env("RUSTC_PRINT_STEP_RUSAGE", "1"); - } - - if self.config.backtrace_on_ice { - cargo.env("RUSTC_BACKTRACE_ON_ICE", "1"); - } - - cargo.env("RUSTC_VERBOSE", self.verbosity.to_string()); - - // Downstream forks of the Rust compiler might want to use a custom libc to add support for - // targets that are not yet available upstream. Adding a patch to replace libc with a - // custom one would cause compilation errors though, because Cargo would interpret the - // custom libc as part of the workspace, and apply the check-cfg lints on it. - // - // The libc build script emits check-cfg flags only when this environment variable is set, - // so this line allows the use of custom libcs. - cargo.env("LIBC_CHECK_CFG", "1"); - - if source_type == SourceType::InTree { - let mut lint_flags = Vec::new(); - // When extending this list, add the new lints to the RUSTFLAGS of the - // build_bootstrap function of src/bootstrap/bootstrap.py as well as - // some code doesn't go through this `rustc` wrapper. - lint_flags.push("-Wrust_2018_idioms"); - lint_flags.push("-Wunused_lifetimes"); - lint_flags.push("-Wsemicolon_in_expressions_from_macros"); - - if self.config.deny_warnings { - lint_flags.push("-Dwarnings"); - rustdocflags.arg("-Dwarnings"); - } - - // This does not use RUSTFLAGS due to caching issues with Cargo. - // Clippy is treated as an "in tree" tool, but shares the same - // cache as other "submodule" tools. With these options set in - // RUSTFLAGS, that causes *every* shared dependency to be rebuilt. - // By injecting this into the rustc wrapper, this circumvents - // Cargo's fingerprint detection. This is fine because lint flags - // are always ignored in dependencies. Eventually this should be - // fixed via better support from Cargo. - cargo.env("RUSTC_LINT_FLAGS", lint_flags.join(" ")); - - rustdocflags.arg("-Wrustdoc::invalid_codeblock_attributes"); - } - - if mode == Mode::Rustc { - rustflags.arg("-Zunstable-options"); - rustflags.arg("-Wrustc::internal"); - } - - // Throughout the build Cargo can execute a number of build scripts - // compiling C/C++ code and we need to pass compilers, archivers, flags, etc - // obtained previously to those build scripts. - // Build scripts use either the `cc` crate or `configure/make` so we pass - // the options through environment variables that are fetched and understood by both. - // - // FIXME: the guard against msvc shouldn't need to be here - if target.contains("msvc") { - if let Some(ref cl) = self.config.llvm_clang_cl { - cargo.env("CC", cl).env("CXX", cl); - } - } else { - let ccache = self.config.ccache.as_ref(); - let ccacheify = |s: &Path| { - let ccache = match ccache { - Some(ref s) => s, - None => return s.display().to_string(), - }; - // FIXME: the cc-rs crate only recognizes the literal strings - // `ccache` and `sccache` when doing caching compilations, so we - // mirror that here. It should probably be fixed upstream to - // accept a new env var or otherwise work with custom ccache - // vars. - match &ccache[..] { - "ccache" | "sccache" => format!("{} {}", ccache, s.display()), - _ => s.display().to_string(), - } - }; - let triple_underscored = target.triple.replace("-", "_"); - let cc = ccacheify(&self.cc(target)); - cargo.env(format!("CC_{triple_underscored}"), &cc); - - let cflags = self.cflags(target, GitRepo::Rustc, CLang::C).join(" "); - cargo.env(format!("CFLAGS_{triple_underscored}"), &cflags); - - if let Some(ar) = self.ar(target) { - let ranlib = format!("{} s", ar.display()); - cargo - .env(format!("AR_{triple_underscored}"), ar) - .env(format!("RANLIB_{triple_underscored}"), ranlib); - } - - if let Ok(cxx) = self.cxx(target) { - let cxx = ccacheify(&cxx); - let cxxflags = self.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" "); - cargo - .env(format!("CXX_{triple_underscored}"), &cxx) - .env(format!("CXXFLAGS_{triple_underscored}"), cxxflags); - } - } - - // If Control Flow Guard is enabled, pass the `control-flow-guard` flag to rustc - // when compiling the standard library, since this might be linked into the final outputs - // produced by rustc. Since this mitigation is only available on Windows, only enable it - // for the standard library in case the compiler is run on a non-Windows platform. - // This is not needed for stage 0 artifacts because these will only be used for building - // the stage 1 compiler. - if cfg!(windows) - && mode == Mode::Std - && self.config.control_flow_guard - && compiler.stage >= 1 - { - rustflags.arg("-Ccontrol-flow-guard"); - } - - // For `cargo doc` invocations, make rustdoc print the Rust version into the docs - // This replaces spaces with tabs because RUSTDOCFLAGS does not - // support arguments with regular spaces. Hopefully someday Cargo will - // have space support. - let rust_version = self.rust_version().replace(' ', "\t"); - rustdocflags.arg("--crate-version").arg(&rust_version); - - // Environment variables *required* throughout the build - // - // FIXME: should update code to not require this env var - cargo.env("CFG_COMPILER_HOST_TRIPLE", target.triple); - - // Set this for all builds to make sure doc builds also get it. - cargo.env("CFG_RELEASE_CHANNEL", &self.config.channel); - - // This one's a bit tricky. As of the time of this writing the compiler - // links to the `winapi` crate on crates.io. This crate provides raw - // bindings to Windows system functions, sort of like libc does for - // Unix. This crate also, however, provides "import libraries" for the - // MinGW targets. There's an import library per dll in the windows - // distribution which is what's linked to. These custom import libraries - // are used because the winapi crate can reference Windows functions not - // present in the MinGW import libraries. - // - // For example MinGW may ship libdbghelp.a, but it may not have - // references to all the functions in the dbghelp dll. Instead the - // custom import library for dbghelp in the winapi crates has all this - // information. - // - // Unfortunately for us though the import libraries are linked by - // default via `-ldylib=winapi_foo`. That is, they're linked with the - // `dylib` type with a `winapi_` prefix (so the winapi ones don't - // conflict with the system MinGW ones). This consequently means that - // the binaries we ship of things like rustc_codegen_llvm (aka the rustc_codegen_llvm - // DLL) when linked against *again*, for example with procedural macros - // or plugins, will trigger the propagation logic of `-ldylib`, passing - // `-lwinapi_foo` to the linker again. This isn't actually available in - // our distribution, however, so the link fails. - // - // To solve this problem we tell winapi to not use its bundled import - // libraries. This means that it will link to the system MinGW import - // libraries by default, and the `-ldylib=foo` directives will still get - // passed to the final linker, but they'll look like `-lfoo` which can - // be resolved because MinGW has the import library. The downside is we - // don't get newer functions from Windows, but we don't use any of them - // anyway. - if !mode.is_tool() { - cargo.env("WINAPI_NO_BUNDLED_LIBRARIES", "1"); - } - - for _ in 0..self.verbosity { - cargo.arg("-v"); - } - - match (mode, self.config.rust_codegen_units_std, self.config.rust_codegen_units) { - (Mode::Std, Some(n), _) | (_, _, Some(n)) => { - cargo.env(profile_var("CODEGEN_UNITS"), n.to_string()); - } - _ => { - // Don't set anything - } - } - - if self.config.locked_deps { - cargo.arg("--locked"); - } - if self.config.vendor || self.is_sudo { - cargo.arg("--frozen"); - } - - // Try to use a sysroot-relative bindir, in case it was configured absolutely. - cargo.env("RUSTC_INSTALL_BINDIR", self.config.bindir_relative()); - - self.ci_env.force_coloring_in_ci(&mut cargo); - - // When we build Rust dylibs they're all intended for intermediate - // usage, so make sure we pass the -Cprefer-dynamic flag instead of - // linking all deps statically into the dylib. - if matches!(mode, Mode::Std | Mode::Rustc) { - rustflags.arg("-Cprefer-dynamic"); - } - - // When building incrementally we default to a lower ThinLTO import limit - // (unless explicitly specified otherwise). This will produce a somewhat - // slower code but give way better compile times. - { - let limit = match self.config.rust_thin_lto_import_instr_limit { - Some(limit) => Some(limit), - None if self.config.incremental => Some(10), - _ => None, - }; - - if let Some(limit) = limit { - if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm" - { - rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}")); - } - } - } - - if matches!(mode, Mode::Std) { - if let Some(mir_opt_level) = self.config.rust_validate_mir_opts { - rustflags.arg("-Zvalidate-mir"); - rustflags.arg(&format!("-Zmir-opt-level={mir_opt_level}")); - } - // Always enable inlining MIR when building the standard library. - // Without this flag, MIR inlining is disabled when incremental compilation is enabled. - // That causes some mir-opt tests which inline functions from the standard library to - // break when incremental compilation is enabled. So this overrides the "no inlining - // during incremental builds" heuristic for the standard library. - rustflags.arg("-Zinline-mir"); - } - - // set rustc args passed from command line - let rustc_args = - self.config.cmd.rustc_args().iter().map(|s| s.to_string()).collect::>(); - if !rustc_args.is_empty() { - cargo.env("RUSTFLAGS", &rustc_args.join(" ")); - } - - Cargo { command: cargo, rustflags, rustdocflags, allow_features } - } - - /// Ensure that a given step is built, returning its output. This will - /// cache the step, so it is safe (and good!) to call this as often as - /// needed to ensure that all dependencies are built. - pub fn ensure(&'a self, step: S) -> S::Output { - { - let mut stack = self.stack.borrow_mut(); - for stack_step in stack.iter() { - // should skip - if stack_step.downcast_ref::().map_or(true, |stack_step| *stack_step != step) { - continue; - } - let mut out = String::new(); - out += &format!("\n\nCycle in build detected when adding {step:?}\n"); - for el in stack.iter().rev() { - out += &format!("\t{el:?}\n"); - } - panic!("{}", out); - } - if let Some(out) = self.cache.get(&step) { - self.verbose_than(1, &format!("{}c {:?}", " ".repeat(stack.len()), step)); - - return out; - } - self.verbose_than(1, &format!("{}> {:?}", " ".repeat(stack.len()), step)); - stack.push(Box::new(step.clone())); - } - - #[cfg(feature = "build-metrics")] - self.metrics.enter_step(&step, self); - - let (out, dur) = { - let start = Instant::now(); - let zero = Duration::new(0, 0); - let parent = self.time_spent_on_dependencies.replace(zero); - let out = step.clone().run(self); - let dur = start.elapsed(); - let deps = self.time_spent_on_dependencies.replace(parent + dur); - (out, dur - deps) - }; - - if self.config.print_step_timings && !self.config.dry_run() { - let step_string = format!("{step:?}"); - let brace_index = step_string.find("{").unwrap_or(0); - let type_string = type_name::(); - println!( - "[TIMING] {} {} -- {}.{:03}", - &type_string.strip_prefix("bootstrap::").unwrap_or(type_string), - &step_string[brace_index..], - dur.as_secs(), - dur.subsec_millis() - ); - } - - #[cfg(feature = "build-metrics")] - self.metrics.exit_step(self); - - { - let mut stack = self.stack.borrow_mut(); - let cur_step = stack.pop().expect("step stack empty"); - assert_eq!(cur_step.downcast_ref(), Some(&step)); - } - self.verbose_than(1, &format!("{}< {:?}", " ".repeat(self.stack.borrow().len()), step)); - self.cache.put(step, out.clone()); - out - } - - /// Ensure that a given step is built *only if it's supposed to be built by default*, returning - /// its output. This will cache the step, so it's safe (and good!) to call this as often as - /// needed to ensure that all dependencies are build. - pub(crate) fn ensure_if_default>>( - &'a self, - step: S, - kind: Kind, - ) -> S::Output { - let desc = StepDescription::from::(kind); - let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); - - // Avoid running steps contained in --skip - for pathset in &should_run.paths { - if desc.is_excluded(self, pathset) { - return None; - } - } - - // Only execute if it's supposed to run as default - if desc.default && should_run.is_really_default() { self.ensure(step) } else { None } - } - - /// Checks if any of the "should_run" paths is in the `Builder` paths. - pub(crate) fn was_invoked_explicitly(&'a self, kind: Kind) -> bool { - let desc = StepDescription::from::(kind); - let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); - - for path in &self.paths { - if should_run.paths.iter().any(|s| s.has(path, desc.kind)) - && !desc.is_excluded( - self, - &PathSet::Suite(TaskPath { path: path.clone(), kind: Some(desc.kind) }), - ) - { - return true; - } - } - - false - } - - pub(crate) fn maybe_open_in_browser(&self, path: impl AsRef) { - if self.was_invoked_explicitly::(Kind::Doc) { - self.open_in_browser(path); - } - } - - pub(crate) fn open_in_browser(&self, path: impl AsRef) { - if self.config.dry_run() || !self.config.cmd.open() { - return; - } - - let path = path.as_ref(); - self.info(&format!("Opening doc {}", path.display())); - if let Err(err) = opener::open(path) { - self.info(&format!("{err}\n")); - } - } -} - -#[cfg(test)] -mod tests; - -/// Represents flag values in `String` form with whitespace delimiter to pass it to the compiler later. -/// -/// `-Z crate-attr` flags will be applied recursively on the target code using the `rustc_parse::parser::Parser`. -/// See `rustc_builtin_macros::cmdline_attrs::inject` for more information. -#[derive(Debug, Clone)] -struct Rustflags(String, TargetSelection); - -impl Rustflags { - fn new(target: TargetSelection) -> Rustflags { - let mut ret = Rustflags(String::new(), target); - ret.propagate_cargo_env("RUSTFLAGS"); - ret - } - - /// By default, cargo will pick up on various variables in the environment. However, bootstrap - /// reuses those variables to pass additional flags to rustdoc, so by default they get overridden. - /// Explicitly add back any previous value in the environment. - /// - /// `prefix` is usually `RUSTFLAGS` or `RUSTDOCFLAGS`. - fn propagate_cargo_env(&mut self, prefix: &str) { - // Inherit `RUSTFLAGS` by default ... - self.env(prefix); - - // ... and also handle target-specific env RUSTFLAGS if they're configured. - let target_specific = format!("CARGO_TARGET_{}_{}", crate::envify(&self.1.triple), prefix); - self.env(&target_specific); - } - - fn env(&mut self, env: &str) { - if let Ok(s) = env::var(env) { - for part in s.split(' ') { - self.arg(part); - } - } - } - - fn arg(&mut self, arg: &str) -> &mut Self { - assert_eq!(arg.split(' ').count(), 1); - if !self.0.is_empty() { - self.0.push(' '); - } - self.0.push_str(arg); - self - } -} - -#[derive(Debug)] -pub struct Cargo { - command: Command, - rustflags: Rustflags, - rustdocflags: Rustflags, - allow_features: String, -} - -impl Cargo { - pub fn rustdocflag(&mut self, arg: &str) -> &mut Cargo { - self.rustdocflags.arg(arg); - self - } - pub fn rustflag(&mut self, arg: &str) -> &mut Cargo { - self.rustflags.arg(arg); - self - } - - pub fn arg(&mut self, arg: impl AsRef) -> &mut Cargo { - self.command.arg(arg.as_ref()); - self - } - - pub fn args(&mut self, args: I) -> &mut Cargo - where - I: IntoIterator, - S: AsRef, - { - for arg in args { - self.arg(arg.as_ref()); - } - self - } - - pub fn env(&mut self, key: impl AsRef, value: impl AsRef) -> &mut Cargo { - // These are managed through rustflag/rustdocflag interfaces. - assert_ne!(key.as_ref(), "RUSTFLAGS"); - assert_ne!(key.as_ref(), "RUSTDOCFLAGS"); - self.command.env(key.as_ref(), value.as_ref()); - self - } - - pub fn add_rustc_lib_path(&mut self, builder: &Builder<'_>, compiler: Compiler) { - builder.add_rustc_lib_path(compiler, &mut self.command); - } - - pub fn current_dir(&mut self, dir: &Path) -> &mut Cargo { - self.command.current_dir(dir); - self - } - - /// Adds nightly-only features that this invocation is allowed to use. - /// - /// By default, all nightly features are allowed. Once this is called, it - /// will be restricted to the given set. - pub fn allow_features(&mut self, features: &str) -> &mut Cargo { - if !self.allow_features.is_empty() { - self.allow_features.push(','); - } - self.allow_features.push_str(features); - self - } -} - -impl From for Command { - fn from(mut cargo: Cargo) -> Command { - let rustflags = &cargo.rustflags.0; - if !rustflags.is_empty() { - cargo.command.env("RUSTFLAGS", rustflags); - } - - let rustdocflags = &cargo.rustdocflags.0; - if !rustdocflags.is_empty() { - cargo.command.env("RUSTDOCFLAGS", rustdocflags); - } - - if !cargo.allow_features.is_empty() { - cargo.command.env("RUSTC_ALLOW_FEATURES", cargo.allow_features); - } - - cargo.command - } -} diff --git a/src/bootstrap/builder/tests.rs b/src/bootstrap/builder/tests.rs deleted file mode 100644 index 80e66622e..000000000 --- a/src/bootstrap/builder/tests.rs +++ /dev/null @@ -1,702 +0,0 @@ -use super::*; -use crate::config::{Config, DryRun, TargetSelection}; -use crate::doc::DocumentationFormat; -use std::thread; - -fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { - configure_with_args(&[cmd.to_owned()], host, target) -} - -fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config { - let mut config = Config::parse(cmd); - // don't save toolstates - config.save_toolstates = None; - config.dry_run = DryRun::SelfCheck; - - // Ignore most submodules, since we don't need them for a dry run. - // But make sure to check out the `doc` and `rust-analyzer` submodules, since some steps need them - // just to know which commands to run. - let submodule_build = Build::new(Config { - // don't include LLVM, so CI doesn't require ninja/cmake to be installed - rust_codegen_backends: vec![], - ..Config::parse(&["check".to_owned()]) - }); - submodule_build.update_submodule(Path::new("src/doc/book")); - submodule_build.update_submodule(Path::new("src/tools/rust-analyzer")); - config.submodules = Some(false); - - config.ninja_in_file = false; - // try to avoid spurious failures in dist where we create/delete each others file - // HACK: rather than pull in `tempdir`, use the one that cargo has conveniently created for us - let dir = Path::new(env!("OUT_DIR")) - .join("tmp-rustbuild-tests") - .join(&thread::current().name().unwrap_or("unknown").replace(":", "-")); - t!(fs::create_dir_all(&dir)); - config.out = dir; - config.build = TargetSelection::from_user("A"); - config.hosts = host.iter().map(|s| TargetSelection::from_user(s)).collect(); - config.targets = target.iter().map(|s| TargetSelection::from_user(s)).collect(); - config -} - -fn first(v: Vec<(A, B)>) -> Vec { - v.into_iter().map(|(a, _)| a).collect::>() -} - -fn run_build(paths: &[PathBuf], config: Config) -> Cache { - let kind = config.cmd.kind(); - let build = Build::new(config); - let builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(kind), paths); - builder.cache -} - -fn check_cli(paths: [&str; N]) { - run_build( - &paths.map(PathBuf::from), - configure_with_args(&paths.map(String::from), &["A"], &["A"]), - ); -} - -macro_rules! std { - ($host:ident => $target:ident, stage = $stage:literal) => { - compile::Std::new( - Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, - TargetSelection::from_user(stringify!($target)), - ) - }; -} - -macro_rules! doc_std { - ($host:ident => $target:ident, stage = $stage:literal) => {{ - let config = configure("doc", &["A"], &["A"]); - let build = Build::new(config); - let builder = Builder::new(&build); - doc::Std::new( - $stage, - TargetSelection::from_user(stringify!($target)), - &builder, - DocumentationFormat::HTML, - ) - }}; -} - -macro_rules! rustc { - ($host:ident => $target:ident, stage = $stage:literal) => { - compile::Rustc::new( - Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, - TargetSelection::from_user(stringify!($target)), - ) - }; -} - -#[test] -fn test_valid() { - // make sure multi suite paths are accepted - check_cli(["test", "tests/ui/attr-start.rs", "tests/ui/attr-shebang.rs"]); -} - -#[test] -#[should_panic] -fn test_invalid() { - // make sure that invalid paths are caught, even when combined with valid paths - check_cli(["test", "library/std", "x"]); -} - -#[test] -fn test_intersection() { - let set = |paths: &[&str]| { - PathSet::Set(paths.into_iter().map(|p| TaskPath { path: p.into(), kind: None }).collect()) - }; - let library_set = set(&["library/core", "library/alloc", "library/std"]); - let mut command_paths = - vec![Path::new("library/core"), Path::new("library/alloc"), Path::new("library/stdarch")]; - let subset = library_set.intersection_removing_matches(&mut command_paths, Kind::Build); - assert_eq!(subset, set(&["library/core", "library/alloc"]),); - assert_eq!(command_paths, vec![Path::new("library/stdarch")]); -} - -#[test] -fn test_exclude() { - let mut config = configure("test", &["A"], &["A"]); - config.skip = vec!["src/tools/tidy".into()]; - let cache = run_build(&[], config); - - // Ensure we have really excluded tidy - assert!(!cache.contains::()); - - // Ensure other tests are not affected. - assert!(cache.contains::()); -} - -#[test] -fn test_exclude_kind() { - let path = PathBuf::from("compiler/rustc_data_structures"); - - let mut config = configure("test", &["A"], &["A"]); - // Ensure our test is valid, and `test::Rustc` would be run without the exclude. - assert!(run_build(&[], config.clone()).contains::()); - // Ensure tests for rustc are not skipped. - config.skip = vec![path.clone()]; - assert!(run_build(&[], config.clone()).contains::()); - // Ensure builds for rustc are not skipped. - assert!(run_build(&[], config).contains::()); -} - -/// Ensure that if someone passes both a single crate and `library`, all library crates get built. -#[test] -fn alias_and_path_for_library() { - let mut cache = - run_build(&["library".into(), "core".into()], configure("build", &["A"], &["A"])); - assert_eq!( - first(cache.all::()), - &[std!(A => A, stage = 0), std!(A => A, stage = 1)] - ); - - let mut cache = run_build(&["library".into(), "core".into()], configure("doc", &["A"], &["A"])); - assert_eq!(first(cache.all::()), &[doc_std!(A => A, stage = 0)]); -} - -#[test] -fn test_beta_rev_parsing() { - use crate::extract_beta_rev; - - // single digit revision - assert_eq!(extract_beta_rev("1.99.9-beta.7 (xxxxxx)"), Some("7".to_string())); - // multiple digits - assert_eq!(extract_beta_rev("1.99.9-beta.777 (xxxxxx)"), Some("777".to_string())); - // nightly channel (no beta revision) - assert_eq!(extract_beta_rev("1.99.9-nightly (xxxxxx)"), None); - // stable channel (no beta revision) - assert_eq!(extract_beta_rev("1.99.9 (xxxxxxx)"), None); - // invalid string - assert_eq!(extract_beta_rev("invalid"), None); -} - -mod defaults { - use super::{configure, first, run_build}; - use crate::builder::*; - use crate::Config; - use pretty_assertions::assert_eq; - - #[test] - fn build_default() { - let mut cache = run_build(&[], configure("build", &["A"], &["A"])); - - let a = TargetSelection::from_user("A"); - assert_eq!( - first(cache.all::()), - &[std!(A => A, stage = 0), std!(A => A, stage = 1),] - ); - assert!(!cache.all::().is_empty()); - // Make sure rustdoc is only built once. - assert_eq!( - first(cache.all::()), - // Recall that rustdoc stages are off-by-one - // - this is the compiler it's _linked_ to, not built with. - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }], - ); - assert_eq!(first(cache.all::()), &[rustc!(A => A, stage = 0)],); - } - - #[test] - fn build_stage_0() { - let config = Config { stage: 0, ..configure("build", &["A"], &["A"]) }; - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user("A"); - assert_eq!(first(cache.all::()), &[std!(A => A, stage = 0)]); - assert!(!cache.all::().is_empty()); - assert_eq!( - first(cache.all::()), - // This is the beta rustdoc. - // Add an assert here to make sure this is the only rustdoc built. - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }], - ); - assert!(cache.all::().is_empty()); - } - - #[test] - fn build_cross_compile() { - let config = Config { stage: 1, ..configure("build", &["A", "B"], &["A", "B"]) }; - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - // Ideally, this build wouldn't actually have `target: a` - // rustdoc/rustcc/std here (the user only requested a host=B build, so - // there's not really a need for us to build for target A in this case - // (since we're producing stage 1 libraries/binaries). But currently - // rustbuild is just a bit buggy here; this should be fixed though. - assert_eq!( - first(cache.all::()), - &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => B, stage = 0), - std!(A => B, stage = 1), - ] - ); - assert_eq!( - first(cache.all::()), - &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, - tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } }, - ], - ); - assert_eq!( - first(cache.all::()), - &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 0),] - ); - } - - #[test] - fn doc_default() { - let mut config = configure("doc", &["A"], &["A"]); - config.compiler_docs = true; - config.cmd = Subcommand::Doc { open: false, json: false }; - let mut cache = run_build(&[], config); - let a = TargetSelection::from_user("A"); - - // error_index_generator uses stage 0 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!(first(cache.all::()), &[doc::ErrorIndex { target: a },]); - assert_eq!( - first(cache.all::()), - &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 0 } }] - ); - // docs should be built with the beta compiler, not with the stage0 artifacts. - // recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to, - // not the one it was built by. - assert_eq!( - first(cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },] - ); - } -} - -mod dist { - use super::{first, run_build, Config}; - use crate::builder::*; - use pretty_assertions::assert_eq; - - fn configure(host: &[&str], target: &[&str]) -> Config { - Config { stage: 2, ..super::configure("dist", host, target) } - } - - #[test] - fn dist_baseline() { - let mut cache = run_build(&[], configure(&["A"], &["A"])); - - let a = TargetSelection::from_user("A"); - - assert_eq!(first(cache.all::()), &[dist::Docs { host: a },]); - assert_eq!(first(cache.all::()), &[dist::Mingw { host: a },]); - assert_eq!( - first(cache.all::()), - &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },] - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - // Make sure rustdoc is only built once. - assert_eq!( - first(cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] - ); - } - - #[test] - fn dist_with_targets() { - let mut cache = run_build(&[], configure(&["A"], &["A", "B"])); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - assert_eq!( - first(cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, - ] - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - } - - #[test] - fn dist_with_hosts() { - let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - assert_eq!( - first(cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => A, stage = 2), - std!(A => B, stage = 1), - std!(A => B, stage = 2), - ], - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - } - - #[test] - fn dist_only_cross_host() { - let b = TargetSelection::from_user("B"); - let mut config = configure(&["A", "B"], &["A", "B"]); - config.docs = false; - config.extended = true; - config.hosts = vec![b]; - let mut cache = run_build(&[], config); - - assert_eq!( - first(cache.all::()), - &[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },] - ); - assert_eq!( - first(cache.all::()), - &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 1),] - ); - } - - #[test] - fn dist_with_targets_and_hosts() { - let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B", "C"])); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - let c = TargetSelection::from_user("C"); - - assert_eq!( - first(cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, - ] - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - } - - #[test] - fn dist_with_empty_host() { - let config = configure(&[], &["C"]); - let mut cache = run_build(&[], config); - - let a = TargetSelection::from_user("A"); - let c = TargetSelection::from_user("C"); - - assert_eq!(first(cache.all::()), &[dist::Docs { host: c },]); - assert_eq!(first(cache.all::()), &[dist::Mingw { host: c },]); - assert_eq!( - first(cache.all::()), - &[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },] - ); - } - - #[test] - fn dist_with_same_targets_and_hosts() { - let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); - - let a = TargetSelection::from_user("A"); - let b = TargetSelection::from_user("B"); - - assert_eq!( - first(cache.all::()), - &[dist::Docs { host: a }, dist::Docs { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[dist::Mingw { host: a }, dist::Mingw { host: b },] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, - dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, - ] - ); - assert_eq!( - first(cache.all::()), - &[ - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, - dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, - ] - ); - assert_eq!(first(cache.all::()), &[dist::Src]); - assert_eq!( - first(cache.all::()), - &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => A, stage = 2), - std!(A => B, stage = 1), - std!(A => B, stage = 2), - ] - ); - assert_eq!( - first(cache.all::()), - &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, - compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, - ] - ); - } - - #[test] - fn build_all() { - let build = Build::new(configure(&["A", "B"], &["A", "B", "C"])); - let mut builder = Builder::new(&build); - builder.run_step_descriptions( - &Builder::get_step_descriptions(Kind::Build), - &["compiler/rustc".into(), "library".into()], - ); - - assert_eq!( - first(builder.cache.all::()), - &[ - std!(A => A, stage = 0), - std!(A => A, stage = 1), - std!(A => A, stage = 2), - std!(A => B, stage = 1), - std!(A => B, stage = 2), - std!(A => C, stage = 2), - ] - ); - assert_eq!(builder.cache.all::().len(), 5); - assert_eq!( - first(builder.cache.all::()), - &[ - rustc!(A => A, stage = 0), - rustc!(A => A, stage = 1), - rustc!(A => A, stage = 2), - rustc!(A => B, stage = 1), - rustc!(A => B, stage = 2), - ] - ); - } - - #[test] - fn build_with_empty_host() { - let config = configure(&[], &["C"]); - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); - - let a = TargetSelection::from_user("A"); - - assert_eq!( - first(builder.cache.all::()), - &[std!(A => A, stage = 0), std!(A => A, stage = 1), std!(A => C, stage = 2),] - ); - assert_eq!( - first(builder.cache.all::()), - &[ - compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, - compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, - ] - ); - assert_eq!( - first(builder.cache.all::()), - &[rustc!(A => A, stage = 0), rustc!(A => A, stage = 1),] - ); - } - - #[test] - fn test_with_no_doc_stage0() { - let mut config = configure(&["A"], &["A"]); - config.stage = 0; - config.paths = vec!["library/std".into()]; - config.cmd = Subcommand::Test { - test_args: vec![], - rustc_args: vec![], - no_fail_fast: false, - no_doc: true, - doc: false, - bless: false, - force_rerun: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - run: None, - only_modified: false, - skip: vec![], - extra_checks: None, - }; - - let build = Build::new(config); - let mut builder = Builder::new(&build); - - let host = TargetSelection::from_user("A"); - - builder.run_step_descriptions( - &[StepDescription::from::(Kind::Test)], - &["library/std".into()], - ); - - // Ensure we don't build any compiler artifacts. - assert!(!builder.cache.contains::()); - assert_eq!( - first(builder.cache.all::()), - &[test::Crate { - compiler: Compiler { host, stage: 0 }, - target: host, - mode: Mode::Std, - crates: vec![INTERNER.intern_str("std")], - },] - ); - } - - #[test] - fn doc_ci() { - let mut config = configure(&["A"], &["A"]); - config.compiler_docs = true; - config.cmd = Subcommand::Doc { open: false, json: false }; - let build = Build::new(config); - let mut builder = Builder::new(&build); - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); - let a = TargetSelection::from_user("A"); - - // error_index_generator uses stage 1 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!( - first(builder.cache.all::()), - &[doc::ErrorIndex { target: a },] - ); - assert_eq!( - first(builder.cache.all::()), - &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] - ); - // This is actually stage 1, but Rustdoc::run swaps out the compiler with - // stage minus 1 if --stage is not 0. Very confusing! - assert_eq!( - first(builder.cache.all::()), - &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] - ); - } - - #[test] - fn test_docs() { - // Behavior of `x.py test` doing various documentation tests. - let mut config = configure(&["A"], &["A"]); - config.cmd = Subcommand::Test { - test_args: vec![], - rustc_args: vec![], - no_fail_fast: false, - doc: true, - no_doc: false, - skip: vec![], - bless: false, - force_rerun: false, - compare_mode: None, - rustfix_coverage: false, - pass: None, - run: None, - only_modified: false, - extra_checks: None, - }; - // Make sure rustfmt binary not being found isn't an error. - config.channel = "beta".to_string(); - let build = Build::new(config); - let mut builder = Builder::new(&build); - - builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); - let a = TargetSelection::from_user("A"); - - // error_index_generator uses stage 1 to share rustdoc artifacts with the - // rustdoc tool. - assert_eq!( - first(builder.cache.all::()), - &[doc::ErrorIndex { target: a },] - ); - assert_eq!( - first(builder.cache.all::()), - &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] - ); - // Unfortunately rustdoc is built twice. Once from stage1 for compiletest - // (and other things), and once from stage0 for std crates. Ideally it - // would only be built once. If someone wants to fix this, it might be - // worth investigating if it would be possible to test std from stage1. - // Note that the stages here are +1 than what they actually are because - // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is - // not 0. - // - // The stage 0 copy is the one downloaded for bootstrapping. It is - // (currently) needed to run "cargo test" on the linkchecker, and - // should be relatively "free". - assert_eq!( - first(builder.cache.all::()), - &[ - tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }, - tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, - tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, - ] - ); - } -} diff --git a/src/bootstrap/cache.rs b/src/bootstrap/cache.rs deleted file mode 100644 index 53e4ff034..000000000 --- a/src/bootstrap/cache.rs +++ /dev/null @@ -1,272 +0,0 @@ -use std::any::{Any, TypeId}; -use std::borrow::Borrow; -use std::cell::RefCell; -use std::cmp::Ordering; -use std::collections::HashMap; -use std::fmt; -use std::hash::{Hash, Hasher}; -use std::marker::PhantomData; -use std::mem; -use std::ops::Deref; -use std::path::PathBuf; -use std::sync::Mutex; - -// FIXME: replace with std::lazy after it gets stabilized and reaches beta -use once_cell::sync::Lazy; - -use crate::builder::Step; - -pub struct Interned(usize, PhantomData<*const T>); - -impl Default for Interned { - fn default() -> Self { - T::default().intern() - } -} - -impl Copy for Interned {} -impl Clone for Interned { - fn clone(&self) -> Interned { - *self - } -} - -impl PartialEq for Interned { - fn eq(&self, other: &Self) -> bool { - self.0 == other.0 - } -} -impl Eq for Interned {} - -impl PartialEq for Interned { - fn eq(&self, other: &str) -> bool { - *self == other - } -} -impl<'a> PartialEq<&'a str> for Interned { - fn eq(&self, other: &&str) -> bool { - **self == **other - } -} -impl<'a, T> PartialEq<&'a Interned> for Interned { - fn eq(&self, other: &&Self) -> bool { - self.0 == other.0 - } -} -impl<'a, T> PartialEq> for &'a Interned { - fn eq(&self, other: &Interned) -> bool { - self.0 == other.0 - } -} - -unsafe impl Send for Interned {} -unsafe impl Sync for Interned {} - -impl fmt::Display for Interned { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s: &str = &*self; - f.write_str(s) - } -} - -impl fmt::Debug for Interned -where - Self: Deref, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s: &U = &*self; - f.write_fmt(format_args!("{s:?}")) - } -} - -impl Hash for Interned { - fn hash(&self, state: &mut H) { - let l = T::intern_cache().lock().unwrap(); - l.get(*self).hash(state) - } -} - -impl Deref for Interned { - type Target = T::Target; - fn deref(&self) -> &Self::Target { - let l = T::intern_cache().lock().unwrap(); - unsafe { mem::transmute::<&Self::Target, &Self::Target>(l.get(*self)) } - } -} - -impl, U: ?Sized> AsRef for Interned { - fn as_ref(&self) -> &U { - let l = T::intern_cache().lock().unwrap(); - unsafe { mem::transmute::<&U, &U>(l.get(*self).as_ref()) } - } -} - -impl PartialOrd for Interned { - fn partial_cmp(&self, other: &Self) -> Option { - let l = T::intern_cache().lock().unwrap(); - l.get(*self).partial_cmp(l.get(*other)) - } -} - -impl Ord for Interned { - fn cmp(&self, other: &Self) -> Ordering { - let l = T::intern_cache().lock().unwrap(); - l.get(*self).cmp(l.get(*other)) - } -} - -struct TyIntern { - items: Vec, - set: HashMap>, -} - -impl Default for TyIntern { - fn default() -> Self { - TyIntern { items: Vec::new(), set: Default::default() } - } -} - -impl TyIntern { - fn intern_borrow(&mut self, item: &B) -> Interned - where - B: Eq + Hash + ToOwned + ?Sized, - T: Borrow, - { - if let Some(i) = self.set.get(&item) { - return *i; - } - let item = item.to_owned(); - let interned = Interned(self.items.len(), PhantomData::<*const T>); - self.set.insert(item.clone(), interned); - self.items.push(item); - interned - } - - fn intern(&mut self, item: T) -> Interned { - if let Some(i) = self.set.get(&item) { - return *i; - } - let interned = Interned(self.items.len(), PhantomData::<*const T>); - self.set.insert(item.clone(), interned); - self.items.push(item); - interned - } - - fn get(&self, i: Interned) -> &T { - &self.items[i.0] - } -} - -#[derive(Default)] -pub struct Interner { - strs: Mutex>, - paths: Mutex>, - lists: Mutex>>, -} - -trait Internable: Clone + Eq + Hash + 'static { - fn intern_cache() -> &'static Mutex>; - - fn intern(self) -> Interned { - Self::intern_cache().lock().unwrap().intern(self) - } -} - -impl Internable for String { - fn intern_cache() -> &'static Mutex> { - &INTERNER.strs - } -} - -impl Internable for PathBuf { - fn intern_cache() -> &'static Mutex> { - &INTERNER.paths - } -} - -impl Internable for Vec { - fn intern_cache() -> &'static Mutex> { - &INTERNER.lists - } -} - -impl Interner { - pub fn intern_str(&self, s: &str) -> Interned { - self.strs.lock().unwrap().intern_borrow(s) - } - pub fn intern_string(&self, s: String) -> Interned { - self.strs.lock().unwrap().intern(s) - } - - pub fn intern_path(&self, s: PathBuf) -> Interned { - self.paths.lock().unwrap().intern(s) - } - - pub fn intern_list(&self, v: Vec) -> Interned> { - self.lists.lock().unwrap().intern(v) - } -} - -pub static INTERNER: Lazy = Lazy::new(Interner::default); - -/// This is essentially a `HashMap` which allows storing any type in its input and -/// any type in its output. It is a write-once cache; values are never evicted, -/// which means that references to the value can safely be returned from the -/// `get()` method. -#[derive(Debug)] -pub struct Cache( - RefCell< - HashMap< - TypeId, - Box, // actually a HashMap> - >, - >, -); - -impl Cache { - pub fn new() -> Cache { - Cache(RefCell::new(HashMap::new())) - } - - pub fn put(&self, step: S, value: S::Output) { - let mut cache = self.0.borrow_mut(); - let type_id = TypeId::of::(); - let stepcache = cache - .entry(type_id) - .or_insert_with(|| Box::new(HashMap::::new())) - .downcast_mut::>() - .expect("invalid type mapped"); - assert!(!stepcache.contains_key(&step), "processing {step:?} a second time"); - stepcache.insert(step, value); - } - - pub fn get(&self, step: &S) -> Option { - let mut cache = self.0.borrow_mut(); - let type_id = TypeId::of::(); - let stepcache = cache - .entry(type_id) - .or_insert_with(|| Box::new(HashMap::::new())) - .downcast_mut::>() - .expect("invalid type mapped"); - stepcache.get(step).cloned() - } -} - -#[cfg(test)] -impl Cache { - pub fn all(&mut self) -> Vec<(S, S::Output)> { - let cache = self.0.get_mut(); - let type_id = TypeId::of::(); - let mut v = cache - .remove(&type_id) - .map(|b| b.downcast::>().expect("correct type")) - .map(|m| m.into_iter().collect::>()) - .unwrap_or_default(); - v.sort_by_key(|(s, _)| s.clone()); - v - } - - pub fn contains(&self) -> bool { - self.0.borrow().contains_key(&TypeId::of::()) - } -} diff --git a/src/bootstrap/cc_detect.rs b/src/bootstrap/cc_detect.rs deleted file mode 100644 index 2496c2a9d..000000000 --- a/src/bootstrap/cc_detect.rs +++ /dev/null @@ -1,273 +0,0 @@ -//! C-compiler probing and detection. -//! -//! This module will fill out the `cc` and `cxx` maps of `Build` by looking for -//! C and C++ compilers for each target configured. A compiler is found through -//! a number of vectors (in order of precedence) -//! -//! 1. Configuration via `target.$target.cc` in `config.toml`. -//! 2. Configuration via `target.$target.android-ndk` in `config.toml`, if -//! applicable -//! 3. Special logic to probe on OpenBSD -//! 4. The `CC_$target` environment variable. -//! 5. The `CC` environment variable. -//! 6. "cc" -//! -//! Some of this logic is implemented here, but much of it is farmed out to the -//! `cc` crate itself, so we end up having the same fallbacks as there. -//! Similar logic is then used to find a C++ compiler, just some s/cc/c++/ is -//! used. -//! -//! It is intended that after this module has run no C/C++ compiler will -//! ever be probed for. Instead the compilers found here will be used for -//! everything. - -use std::collections::HashSet; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::{env, iter}; - -use crate::config::{Target, TargetSelection}; -use crate::util::output; -use crate::{Build, CLang, GitRepo}; - -// The `cc` crate doesn't provide a way to obtain a path to the detected archiver, -// so use some simplified logic here. First we respect the environment variable `AR`, then -// try to infer the archiver path from the C compiler path. -// In the future this logic should be replaced by calling into the `cc` crate. -fn cc2ar(cc: &Path, target: TargetSelection) -> Option { - if let Some(ar) = env::var_os(format!("AR_{}", target.triple.replace("-", "_"))) { - Some(PathBuf::from(ar)) - } else if let Some(ar) = env::var_os("AR") { - Some(PathBuf::from(ar)) - } else if target.contains("msvc") { - None - } else if target.contains("musl") { - Some(PathBuf::from("ar")) - } else if target.contains("openbsd") { - Some(PathBuf::from("ar")) - } else if target.contains("vxworks") { - Some(PathBuf::from("wr-ar")) - } else if target.contains("android") { - Some(cc.parent().unwrap().join(PathBuf::from("llvm-ar"))) - } else { - let parent = cc.parent().unwrap(); - let file = cc.file_name().unwrap().to_str().unwrap(); - for suffix in &["gcc", "cc", "clang"] { - if let Some(idx) = file.rfind(suffix) { - let mut file = file[..idx].to_owned(); - file.push_str("ar"); - return Some(parent.join(&file)); - } - } - Some(parent.join(file)) - } -} - -fn new_cc_build(build: &Build, target: TargetSelection) -> cc::Build { - let mut cfg = cc::Build::new(); - cfg.cargo_metadata(false) - .opt_level(2) - .warnings(false) - .debug(false) - // Compress debuginfo - .flag_if_supported("-gz") - .target(&target.triple) - .host(&build.build.triple); - match build.crt_static(target) { - Some(a) => { - cfg.static_crt(a); - } - None => { - if target.contains("msvc") { - cfg.static_crt(true); - } - if target.contains("musl") { - cfg.static_flag(true); - } - } - } - cfg -} - -pub fn find(build: &Build) { - // For all targets we're going to need a C compiler for building some shims - // and such as well as for being a linker for Rust code. - let targets = build - .targets - .iter() - .chain(&build.hosts) - .cloned() - .chain(iter::once(build.build)) - .collect::>(); - for target in targets.into_iter() { - find_target(build, target); - } -} - -pub fn find_target(build: &Build, target: TargetSelection) { - let mut cfg = new_cc_build(build, target); - let config = build.config.target_config.get(&target); - if let Some(cc) = config.and_then(|c| c.cc.as_ref()) { - cfg.compiler(cc); - } else { - set_compiler(&mut cfg, Language::C, target, config, build); - } - - let compiler = cfg.get_compiler(); - let ar = if let ar @ Some(..) = config.and_then(|c| c.ar.clone()) { - ar - } else { - cc2ar(compiler.path(), target) - }; - - build.cc.borrow_mut().insert(target, compiler.clone()); - let cflags = build.cflags(target, GitRepo::Rustc, CLang::C); - - // If we use llvm-libunwind, we will need a C++ compiler as well for all targets - // We'll need one anyways if the target triple is also a host triple - let mut cfg = new_cc_build(build, target); - cfg.cpp(true); - let cxx_configured = if let Some(cxx) = config.and_then(|c| c.cxx.as_ref()) { - cfg.compiler(cxx); - true - } else if build.hosts.contains(&target) || build.build == target { - set_compiler(&mut cfg, Language::CPlusPlus, target, config, build); - true - } else { - // Use an auto-detected compiler (or one configured via `CXX_target_triple` env vars). - cfg.try_get_compiler().is_ok() - }; - - // for VxWorks, record CXX compiler which will be used in lib.rs:linker() - if cxx_configured || target.contains("vxworks") { - let compiler = cfg.get_compiler(); - build.cxx.borrow_mut().insert(target, compiler); - } - - build.verbose(&format!("CC_{} = {:?}", &target.triple, build.cc(target))); - build.verbose(&format!("CFLAGS_{} = {:?}", &target.triple, cflags)); - if let Ok(cxx) = build.cxx(target) { - let cxxflags = build.cflags(target, GitRepo::Rustc, CLang::Cxx); - build.verbose(&format!("CXX_{} = {:?}", &target.triple, cxx)); - build.verbose(&format!("CXXFLAGS_{} = {:?}", &target.triple, cxxflags)); - } - if let Some(ar) = ar { - build.verbose(&format!("AR_{} = {:?}", &target.triple, ar)); - build.ar.borrow_mut().insert(target, ar); - } - - if let Some(ranlib) = config.and_then(|c| c.ranlib.clone()) { - build.ranlib.borrow_mut().insert(target, ranlib); - } -} - -fn set_compiler( - cfg: &mut cc::Build, - compiler: Language, - target: TargetSelection, - config: Option<&Target>, - build: &Build, -) { - match &*target.triple { - // When compiling for android we may have the NDK configured in the - // config.toml in which case we look there. Otherwise the default - // compiler already takes into account the triple in question. - t if t.contains("android") => { - if let Some(ndk) = config.and_then(|c| c.ndk.as_ref()) { - cfg.compiler(ndk_compiler(compiler, &*target.triple, ndk)); - } - } - - // The default gcc version from OpenBSD may be too old, try using egcc, - // which is a gcc version from ports, if this is the case. - t if t.contains("openbsd") => { - let c = cfg.get_compiler(); - let gnu_compiler = compiler.gcc(); - if !c.path().ends_with(gnu_compiler) { - return; - } - - let output = output(c.to_command().arg("--version")); - let i = match output.find(" 4.") { - Some(i) => i, - None => return, - }; - match output[i + 3..].chars().next().unwrap() { - '0'..='6' => {} - _ => return, - } - let alternative = format!("e{gnu_compiler}"); - if Command::new(&alternative).output().is_ok() { - cfg.compiler(alternative); - } - } - - "mips-unknown-linux-musl" => { - if cfg.get_compiler().path().to_str() == Some("gcc") { - cfg.compiler("mips-linux-musl-gcc"); - } - } - "mipsel-unknown-linux-musl" => { - if cfg.get_compiler().path().to_str() == Some("gcc") { - cfg.compiler("mipsel-linux-musl-gcc"); - } - } - - t if t.contains("musl") => { - if let Some(root) = build.musl_root(target) { - let guess = root.join("bin/musl-gcc"); - if guess.exists() { - cfg.compiler(guess); - } - } - } - - _ => {} - } -} - -pub(crate) fn ndk_compiler(compiler: Language, triple: &str, ndk: &Path) -> PathBuf { - let mut triple_iter = triple.split("-"); - let triple_translated = if let Some(arch) = triple_iter.next() { - let arch_new = match arch { - "arm" | "armv7" | "armv7neon" | "thumbv7" | "thumbv7neon" => "armv7a", - other => other, - }; - std::iter::once(arch_new).chain(triple_iter).collect::>().join("-") - } else { - triple.to_string() - }; - - // API 19 is the earliest API level supported by NDK r25b but AArch64 and x86_64 support - // begins at API level 21. - let api_level = - if triple.contains("aarch64") || triple.contains("x86_64") { "21" } else { "19" }; - let compiler = format!("{}{}-{}", triple_translated, api_level, compiler.clang()); - ndk.join("bin").join(compiler) -} - -/// The target programming language for a native compiler. -pub(crate) enum Language { - /// The compiler is targeting C. - C, - /// The compiler is targeting C++. - CPlusPlus, -} - -impl Language { - /// Obtains the name of a compiler in the GCC collection. - fn gcc(self) -> &'static str { - match self { - Language::C => "gcc", - Language::CPlusPlus => "g++", - } - } - - /// Obtains the name of a compiler in the clang suite. - fn clang(self) -> &'static str { - match self { - Language::C => "clang", - Language::CPlusPlus => "clang++", - } - } -} diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs deleted file mode 100644 index 870185740..000000000 --- a/src/bootstrap/channel.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Build configuration for Rust's release channels. -//! -//! Implements the stable/beta/nightly channel distinctions by setting various -//! flags like the `unstable_features`, calculating variables like `release` and -//! `package_vers`, and otherwise indicating to the compiler what it should -//! print out as part of its version information. - -use std::fs; -use std::path::Path; -use std::process::Command; - -use crate::util::output; -use crate::util::t; -use crate::Build; - -#[derive(Clone, Default)] -pub enum GitInfo { - /// This is not a git repository. - #[default] - Absent, - /// This is a git repository. - /// If the info should be used (`omit_git_hash` is false), this will be - /// `Some`, otherwise it will be `None`. - Present(Option), - /// This is not a git repository, but the info can be fetched from the - /// `git-commit-info` file. - RecordedForTarball(Info), -} - -#[derive(Clone)] -pub struct Info { - pub commit_date: String, - pub sha: String, - pub short_sha: String, -} - -impl GitInfo { - pub fn new(omit_git_hash: bool, dir: &Path) -> GitInfo { - // See if this even begins to look like a git dir - if !dir.join(".git").exists() { - match read_commit_info_file(dir) { - Some(info) => return GitInfo::RecordedForTarball(info), - None => return GitInfo::Absent, - } - } - - // Make sure git commands work - match Command::new("git").arg("rev-parse").current_dir(dir).output() { - Ok(ref out) if out.status.success() => {} - _ => return GitInfo::Absent, - } - - // If we're ignoring the git info, we don't actually need to collect it, just make sure this - // was a git repo in the first place. - if omit_git_hash { - return GitInfo::Present(None); - } - - // Ok, let's scrape some info - let ver_date = output( - Command::new("git") - .current_dir(dir) - .arg("log") - .arg("-1") - .arg("--date=short") - .arg("--pretty=format:%cd"), - ); - let ver_hash = output(Command::new("git").current_dir(dir).arg("rev-parse").arg("HEAD")); - let short_ver_hash = output( - Command::new("git").current_dir(dir).arg("rev-parse").arg("--short=9").arg("HEAD"), - ); - GitInfo::Present(Some(Info { - commit_date: ver_date.trim().to_string(), - sha: ver_hash.trim().to_string(), - short_sha: short_ver_hash.trim().to_string(), - })) - } - - pub fn info(&self) -> Option<&Info> { - match self { - GitInfo::Absent => None, - GitInfo::Present(info) => info.as_ref(), - GitInfo::RecordedForTarball(info) => Some(info), - } - } - - pub fn sha(&self) -> Option<&str> { - self.info().map(|s| &s.sha[..]) - } - - pub fn sha_short(&self) -> Option<&str> { - self.info().map(|s| &s.short_sha[..]) - } - - pub fn commit_date(&self) -> Option<&str> { - self.info().map(|s| &s.commit_date[..]) - } - - pub fn version(&self, build: &Build, num: &str) -> String { - let mut version = build.release(num); - if let Some(ref inner) = self.info() { - version.push_str(" ("); - version.push_str(&inner.short_sha); - version.push(' '); - version.push_str(&inner.commit_date); - version.push(')'); - } - version - } - - /// Returns whether this directory has a `.git` directory which should be managed by bootstrap. - pub fn is_managed_git_subrepository(&self) -> bool { - match self { - GitInfo::Absent | GitInfo::RecordedForTarball(_) => false, - GitInfo::Present(_) => true, - } - } - - /// Returns whether this is being built from a tarball. - pub fn is_from_tarball(&self) -> bool { - match self { - GitInfo::Absent | GitInfo::Present(_) => false, - GitInfo::RecordedForTarball(_) => true, - } - } -} - -/// Read the commit information from the `git-commit-info` file given the -/// project root. -pub fn read_commit_info_file(root: &Path) -> Option { - if let Ok(contents) = fs::read_to_string(root.join("git-commit-info")) { - let mut lines = contents.lines(); - let sha = lines.next(); - let short_sha = lines.next(); - let commit_date = lines.next(); - let info = match (commit_date, sha, short_sha) { - (Some(commit_date), Some(sha), Some(short_sha)) => Info { - commit_date: commit_date.to_owned(), - sha: sha.to_owned(), - short_sha: short_sha.to_owned(), - }, - _ => panic!("the `git-commit-info` file is malformed"), - }; - Some(info) - } else { - None - } -} - -/// Write the commit information to the `git-commit-info` file given the project -/// root. -pub fn write_commit_info_file(root: &Path, info: &Info) { - let commit_info = format!("{}\n{}\n{}\n", info.sha, info.short_sha, info.commit_date); - t!(fs::write(root.join("git-commit-info"), &commit_info)); -} - -/// Write the commit hash to the `git-commit-hash` file given the project root. -pub fn write_commit_hash_file(root: &Path, sha: &str) { - t!(fs::write(root.join("git-commit-hash"), sha)); -} diff --git a/src/bootstrap/check.rs b/src/bootstrap/check.rs deleted file mode 100644 index b417abc00..000000000 --- a/src/bootstrap/check.rs +++ /dev/null @@ -1,542 +0,0 @@ -//! Implementation of compiling the compiler and standard library, in "check"-based modes. - -use crate::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::compile::{add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo}; -use crate::config::TargetSelection; -use crate::tool::{prepare_tool_cargo, SourceType}; -use crate::INTERNER; -use crate::{Compiler, Mode, Subcommand}; -use std::path::{Path, PathBuf}; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Std { - pub target: TargetSelection, - /// Whether to build only a subset of crates. - /// - /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. - /// - /// [`compile::Rustc`]: crate::compile::Rustc - crates: Interned>, -} - -/// Returns args for the subcommand itself (not for cargo) -fn args(builder: &Builder<'_>) -> Vec { - fn strings<'a>(arr: &'a [&str]) -> impl Iterator + 'a { - arr.iter().copied().map(String::from) - } - - if let Subcommand::Clippy { fix, allow, deny, warn, forbid, .. } = &builder.config.cmd { - // disable the most spammy clippy lints - let ignored_lints = vec![ - "many_single_char_names", // there are a lot in stdarch - "collapsible_if", - "type_complexity", - "missing_safety_doc", // almost 3K warnings - "too_many_arguments", - "needless_lifetimes", // people want to keep the lifetimes - "wrong_self_convention", - ]; - let mut args = vec![]; - if *fix { - #[rustfmt::skip] - args.extend(strings(&[ - "--fix", "-Zunstable-options", - // FIXME: currently, `--fix` gives an error while checking tests for libtest, - // possibly because libtest is not yet built in the sysroot. - // As a workaround, avoid checking tests and benches when passed --fix. - "--lib", "--bins", "--examples", - ])); - } - args.extend(strings(&["--", "--cap-lints", "warn"])); - args.extend(ignored_lints.iter().map(|lint| format!("-Aclippy::{}", lint))); - let mut clippy_lint_levels: Vec = Vec::new(); - allow.iter().for_each(|v| clippy_lint_levels.push(format!("-A{}", v))); - deny.iter().for_each(|v| clippy_lint_levels.push(format!("-D{}", v))); - warn.iter().for_each(|v| clippy_lint_levels.push(format!("-W{}", v))); - forbid.iter().for_each(|v| clippy_lint_levels.push(format!("-F{}", v))); - args.extend(clippy_lint_levels); - args.extend(builder.config.free_args.clone()); - args - } else { - builder.config.free_args.clone() - } -} - -fn cargo_subcommand(kind: Kind) -> &'static str { - match kind { - Kind::Check => "check", - Kind::Clippy => "clippy", - Kind::Fix => "fix", - _ => unreachable!(), - } -} - -impl Std { - pub fn new(target: TargetSelection) -> Self { - Self { target, crates: INTERNER.intern_list(vec![]) } - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("sysroot").path("library") - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.make_run_crates(Alias::Library); - run.builder.ensure(Std { target: run.target, crates }); - } - - fn run(self, builder: &Builder<'_>) { - builder.update_submodule(&Path::new("library").join("stdarch")); - - let target = self.target; - let compiler = builder.compiler(builder.top_stage, builder.config.build); - - let mut cargo = builder.cargo( - compiler, - Mode::Std, - SourceType::InTree, - target, - cargo_subcommand(builder.kind), - ); - std_cargo(builder, target, compiler.stage, &mut cargo); - if matches!(builder.config.cmd, Subcommand::Fix { .. }) { - // By default, cargo tries to fix all targets. Tell it not to fix tests until we've added `test` to the sysroot. - cargo.arg("--lib"); - } - - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_check( - format_args!("library artifacts{}", crate_description(&self.crates)), - target, - ); - run_cargo( - builder, - cargo, - args(builder), - &libstd_stamp(builder, compiler, target), - vec![], - true, - false, - ); - - // We skip populating the sysroot in non-zero stage because that'll lead - // to rlib/rmeta conflicts if std gets built during this session. - if compiler.stage == 0 { - let libdir = builder.sysroot_libdir(compiler, target); - let hostdir = builder.sysroot_libdir(compiler, compiler.host); - add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); - } - drop(_guard); - - // don't run on std twice with x.py clippy - // don't check test dependencies if we haven't built libtest - if builder.kind == Kind::Clippy || !self.crates.iter().any(|krate| krate == "test") { - return; - } - - // Then run cargo again, once we've put the rmeta files for the library - // crates into the sysroot. This is needed because e.g., core's tests - // depend on `libtest` -- Cargo presumes it will exist, but it doesn't - // since we initialize with an empty sysroot. - // - // Currently only the "libtest" tree of crates does this. - let mut cargo = builder.cargo( - compiler, - Mode::Std, - SourceType::InTree, - target, - cargo_subcommand(builder.kind), - ); - - // If we're not in stage 0, tests and examples will fail to compile - // from `core` definitions being loaded from two different `libcore` - // .rmeta and .rlib files. - if compiler.stage == 0 { - cargo.arg("--all-targets"); - } - - std_cargo(builder, target, compiler.stage, &mut cargo); - - // Explicitly pass -p for all dependencies krates -- this will force cargo - // to also check the tests/benches/examples for these crates, rather - // than just the leaf crate. - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_check("library test/bench/example targets", target); - run_cargo( - builder, - cargo, - args(builder), - &libstd_test_stamp(builder, compiler, target), - vec![], - true, - false, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - /// Whether to build only a subset of crates. - /// - /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. - /// - /// [`compile::Rustc`]: crate::compile::Rustc - crates: Interned>, -} - -impl Rustc { - pub fn new(target: TargetSelection, builder: &Builder<'_>) -> Self { - let crates = builder - .in_tree_crates("rustc-main", Some(target)) - .into_iter() - .map(|krate| krate.name.to_string()) - .collect(); - Self { target, crates: INTERNER.intern_list(crates) } - } -} - -impl Step for Rustc { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("rustc-main").path("compiler") - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.make_run_crates(Alias::Compiler); - run.builder.ensure(Rustc { target: run.target, crates }); - } - - /// Builds the compiler. - /// - /// This will build the compiler for a particular stage of the build using - /// the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - if compiler.stage != 0 { - // If we're not in stage 0, then we won't have a std from the beta - // compiler around. That means we need to make sure there's one in - // the sysroot for the compiler to find. Otherwise, we're going to - // fail when building crates that need to generate code (e.g., build - // scripts and their dependencies). - builder.ensure(crate::compile::Std::new(compiler, compiler.host)); - builder.ensure(crate::compile::Std::new(compiler, target)); - } else { - builder.ensure(Std::new(target)); - } - - let mut cargo = builder.cargo( - compiler, - Mode::Rustc, - SourceType::InTree, - target, - cargo_subcommand(builder.kind), - ); - rustc_cargo(builder, &mut cargo, target, compiler.stage); - - // For ./x.py clippy, don't run with --all-targets because - // linting tests and benchmarks can produce very noisy results - if builder.kind != Kind::Clippy { - cargo.arg("--all-targets"); - } - - // Explicitly pass -p for all compiler crates -- this will force cargo - // to also check the tests/benches/examples for these crates, rather - // than just the leaf crate. - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_check( - format_args!("compiler artifacts{}", crate_description(&self.crates)), - target, - ); - run_cargo( - builder, - cargo, - args(builder), - &librustc_stamp(builder, compiler, target), - vec![], - true, - false, - ); - - let libdir = builder.sysroot_libdir(compiler, target); - let hostdir = builder.sysroot_libdir(compiler, compiler.host); - add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target)); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CodegenBackend { - pub target: TargetSelection, - pub backend: Interned, -} - -impl Step for CodegenBackend { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) - } - - fn make_run(run: RunConfig<'_>) { - for &backend in &[INTERNER.intern_str("cranelift"), INTERNER.intern_str("gcc")] { - run.builder.ensure(CodegenBackend { target: run.target, backend }); - } - } - - fn run(self, builder: &Builder<'_>) { - // FIXME: remove once https://github.com/rust-lang/rust/issues/112393 is resolved - if builder.build.config.vendor && &self.backend == "gcc" { - println!("Skipping checking of `rustc_codegen_gcc` with vendoring enabled."); - return; - } - - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - let backend = self.backend; - - builder.ensure(Rustc::new(target, builder)); - - let mut cargo = builder.cargo( - compiler, - Mode::Codegen, - SourceType::InTree, - target, - cargo_subcommand(builder.kind), - ); - cargo - .arg("--manifest-path") - .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); - rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - let _guard = builder.msg_check(&backend, target); - - run_cargo( - builder, - cargo, - args(builder), - &codegen_backend_stamp(builder, compiler, target, backend), - vec![], - true, - false, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustAnalyzer { - pub target: TargetSelection, -} - -impl Step for RustAnalyzer { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/rust-analyzer").default_condition( - builder - .config - .tools - .as_ref() - .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzer { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - builder.ensure(Std::new(target)); - - let mut cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolStd, - target, - cargo_subcommand(builder.kind), - "src/tools/rust-analyzer", - SourceType::InTree, - &["rust-analyzer/in-rust-tree".to_owned()], - ); - - cargo.allow_features(crate::tool::RustAnalyzer::ALLOW_FEATURES); - - // For ./x.py clippy, don't check those targets because - // linting tests and benchmarks can produce very noisy results - if builder.kind != Kind::Clippy { - // can't use `--all-targets` because `--examples` doesn't work well - cargo.arg("--bins"); - cargo.arg("--tests"); - cargo.arg("--benches"); - } - - let _guard = builder.msg_check("rust-analyzer artifacts", target); - run_cargo( - builder, - cargo, - args(builder), - &stamp(builder, compiler, target), - vec![], - true, - false, - ); - - /// Cargo's output path in a given stage, compiled by a particular - /// compiler for the specified target. - fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::ToolStd, target).join(".rust-analyzer-check.stamp") - } - } -} - -macro_rules! tool_check_step { - ($name:ident, $path:literal, $($alias:literal, )* $source_type:path $(, $default:literal )?) => { - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] - pub struct $name { - pub target: TargetSelection, - } - - impl Step for $name { - type Output = (); - const ONLY_HOSTS: bool = true; - // don't ever check out-of-tree tools by default, they'll fail when toolstate is broken - const DEFAULT: bool = matches!($source_type, SourceType::InTree) $( && $default )?; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&[ $path, $($alias),* ]) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(builder.top_stage, builder.config.build); - let target = self.target; - - builder.ensure(Rustc::new(target, builder)); - - let mut cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - cargo_subcommand(builder.kind), - $path, - $source_type, - &[], - ); - - // For ./x.py clippy, don't run with --all-targets because - // linting tests and benchmarks can produce very noisy results - if builder.kind != Kind::Clippy { - cargo.arg("--all-targets"); - } - - // Enable internal lints for clippy and rustdoc - // NOTE: this doesn't enable lints for any other tools unless they explicitly add `#![warn(rustc::internal)]` - // See https://github.com/rust-lang/rust/pull/80573#issuecomment-754010776 - cargo.rustflag("-Zunstable-options"); - let _guard = builder.msg_check(&concat!(stringify!($name), " artifacts").to_lowercase(), target); - run_cargo( - builder, - cargo, - args(builder), - &stamp(builder, compiler, target), - vec![], - true, - false, - ); - - /// Cargo's output path in a given stage, compiled by a particular - /// compiler for the specified target. - fn stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - ) -> PathBuf { - builder - .cargo_out(compiler, Mode::ToolRustc, target) - .join(format!(".{}-check.stamp", stringify!($name).to_lowercase())) - } - } - } - }; -} - -tool_check_step!(Rustdoc, "src/tools/rustdoc", "src/librustdoc", SourceType::InTree); -// Clippy, miri and Rustfmt are hybrids. They are external tools, but use a git subtree instead -// of a submodule. Since the SourceType only drives the deny-warnings -// behavior, treat it as in-tree so that any new warnings in clippy will be -// rejected. -tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree); -tool_check_step!(Miri, "src/tools/miri", SourceType::InTree); -tool_check_step!(CargoMiri, "src/tools/miri/cargo-miri", SourceType::InTree); -tool_check_step!(Rls, "src/tools/rls", SourceType::InTree); -tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); -tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InTree); - -tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp") -} - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -fn libstd_test_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp") -} - -/// Cargo's output path for librustc in a given stage, compiled by a particular -/// compiler for the specified target. -fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp") -} - -/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular -/// compiler for the specified target and backend. -fn codegen_backend_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - backend: Interned, -) -> PathBuf { - builder - .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{backend}-check.stamp")) -} diff --git a/src/bootstrap/clean.rs b/src/bootstrap/clean.rs deleted file mode 100644 index 7389816b4..000000000 --- a/src/bootstrap/clean.rs +++ /dev/null @@ -1,237 +0,0 @@ -//! Implementation of `make clean` in rustbuild. -//! -//! Responsible for cleaning out a build directory of all old and stale -//! artifacts to prepare for a fresh build. Currently doesn't remove the -//! `build/cache` directory (download cache) or the `build/$target/llvm` -//! directory unless the `--all` flag is present. - -use std::fs; -use std::io::{self, ErrorKind}; -use std::path::Path; - -use crate::builder::{crate_description, Builder, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::util::t; -use crate::{Build, Compiler, Mode, Subcommand}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] -pub struct CleanAll {} - -impl Step for CleanAll { - const DEFAULT: bool = true; - type Output = (); - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CleanAll {}) - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let Subcommand::Clean { all, stage } = builder.config.cmd else { - unreachable!("wrong subcommand?") - }; - - if all && stage.is_some() { - panic!("--all and --stage can't be used at the same time for `x clean`"); - } - - clean(builder.build, all, stage) - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() // handled by DEFAULT - } -} - -macro_rules! clean_crate_tree { - ( $( $name:ident, $mode:path, $root_crate:literal);+ $(;)? ) => { $( - #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] - pub struct $name { - compiler: Compiler, - crates: Interned>, - } - - impl Step for $name { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let crates = run.builder.in_tree_crates($root_crate, None); - run.crates(crates) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let compiler = builder.compiler(builder.top_stage, run.target); - builder.ensure(Self { crates: run.cargo_crates_in_set(), compiler }); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let compiler = self.compiler; - let target = compiler.host; - let mut cargo = builder.bare_cargo(compiler, $mode, target, "clean"); - for krate in &*self.crates { - cargo.arg("-p"); - cargo.arg(krate); - } - - builder.info(&format!( - "Cleaning{} stage{} {} artifacts ({} -> {})", - crate_description(&self.crates), compiler.stage, stringify!($name).to_lowercase(), &compiler.host, target, - )); - - // NOTE: doesn't use `run_cargo` because we don't want to save a stamp file, - // and doesn't use `stream_cargo` to avoid passing `--message-format` which `clean` doesn't accept. - builder.run(&mut cargo); - } - } - )+ } -} - -clean_crate_tree! { - Rustc, Mode::Rustc, "rustc-main"; - Std, Mode::Std, "sysroot"; -} - -fn clean(build: &Build, all: bool, stage: Option) { - if build.config.dry_run() { - return; - } - - rm_rf("tmp".as_ref()); - - // Clean the entire build directory - if all { - rm_rf(&build.out); - return; - } - - // Clean the target stage artifacts - if let Some(stage) = stage { - clean_specific_stage(build, stage); - return; - } - - // Follow the default behaviour - clean_default(build); -} - -fn clean_specific_stage(build: &Build, stage: u32) { - for host in &build.hosts { - let entries = match build.out.join(host.triple).read_dir() { - Ok(iter) => iter, - Err(_) => continue, - }; - - for entry in entries { - let entry = t!(entry); - let stage_prefix = format!("stage{}", stage); - - // if current entry is not related with the target stage, continue - if !entry.file_name().to_str().unwrap_or("").contains(&stage_prefix) { - continue; - } - - let path = t!(entry.path().canonicalize()); - rm_rf(&path); - } - } -} - -fn clean_default(build: &Build) { - rm_rf(&build.out.join("tmp")); - rm_rf(&build.out.join("dist")); - rm_rf(&build.out.join("bootstrap")); - rm_rf(&build.out.join("rustfmt.stamp")); - - for host in &build.hosts { - let entries = match build.out.join(host.triple).read_dir() { - Ok(iter) => iter, - Err(_) => continue, - }; - - for entry in entries { - let entry = t!(entry); - if entry.file_name().to_str() == Some("llvm") { - continue; - } - let path = t!(entry.path().canonicalize()); - rm_rf(&path); - } - } -} - -fn rm_rf(path: &Path) { - match path.symlink_metadata() { - Err(e) => { - if e.kind() == ErrorKind::NotFound { - return; - } - panic!("failed to get metadata for file {}: {}", path.display(), e); - } - Ok(metadata) => { - if metadata.file_type().is_file() || metadata.file_type().is_symlink() { - do_op(path, "remove file", |p| { - fs::remove_file(p).or_else(|e| { - // Work around the fact that we cannot - // delete an executable while it runs on Windows. - #[cfg(windows)] - if e.kind() == std::io::ErrorKind::PermissionDenied - && p.file_name().and_then(std::ffi::OsStr::to_str) - == Some("bootstrap.exe") - { - eprintln!("warning: failed to delete '{}'.", p.display()); - return Ok(()); - } - Err(e) - }) - }); - return; - } - - for file in t!(fs::read_dir(path)) { - rm_rf(&t!(file).path()); - } - do_op(path, "remove dir", |p| { - fs::remove_dir(p).or_else(|e| { - // Check for dir not empty on Windows - // FIXME: Once `ErrorKind::DirectoryNotEmpty` is stabilized, - // match on `e.kind()` instead. - #[cfg(windows)] - if e.raw_os_error() == Some(145) { - return Ok(()); - } - - Err(e) - }) - }); - } - }; -} - -fn do_op(path: &Path, desc: &str, mut f: F) -where - F: FnMut(&Path) -> io::Result<()>, -{ - match f(path) { - Ok(()) => {} - // On windows we can't remove a readonly file, and git will often clone files as readonly. - // As a result, we have some special logic to remove readonly files on windows. - // This is also the reason that we can't use things like fs::remove_dir_all(). - Err(ref e) if cfg!(windows) && e.kind() == ErrorKind::PermissionDenied => { - let m = t!(path.symlink_metadata()); - let mut p = m.permissions(); - p.set_readonly(false); - t!(fs::set_permissions(path, p)); - f(path).unwrap_or_else(|e| { - // Delete symlinked directories on Windows - #[cfg(windows)] - if m.file_type().is_symlink() && path.is_dir() && fs::remove_dir(path).is_ok() { - return; - } - panic!("failed to {} {}: {}", desc, path.display(), e); - }); - } - Err(e) => { - panic!("failed to {} {}: {}", desc, path.display(), e); - } - } -} diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs deleted file mode 100644 index 292ccc578..000000000 --- a/src/bootstrap/compile.rs +++ /dev/null @@ -1,2015 +0,0 @@ -//! Implementation of compiling various phases of the compiler and standard -//! library. -//! -//! This module contains some of the real meat in the rustbuild build system -//! which is where Cargo is used to compile the standard library, libtest, and -//! the compiler. This module is also responsible for assembling the sysroot as it -//! goes along from the output of the previous stage. - -use std::borrow::Cow; -use std::collections::HashSet; -use std::env; -use std::ffi::OsStr; -use std::fs; -use std::io::prelude::*; -use std::io::BufReader; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::str; - -use serde_derive::Deserialize; - -use crate::builder::crate_description; -use crate::builder::Cargo; -use crate::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; -use crate::cache::{Interned, INTERNER}; -use crate::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; -use crate::dist; -use crate::llvm; -use crate::tool::SourceType; -use crate::util::get_clang_cl_resource_dir; -use crate::util::{exe, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date}; -use crate::LLVM_TOOLS; -use crate::{CLang, Compiler, DependencyType, GitRepo, Mode}; -use filetime::FileTime; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Std { - pub target: TargetSelection, - pub compiler: Compiler, - /// Whether to build only a subset of crates in the standard library. - /// - /// This shouldn't be used from other steps; see the comment on [`Rustc`]. - crates: Interned>, - /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself, - /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overriden by `builder.ensure` from other steps. - force_recompile: bool, -} - -impl Std { - pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default(), force_recompile: false } - } - - pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default(), force_recompile: true } - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // When downloading stage1, the standard library has already been copied to the sysroot, so - // there's no need to rebuild it. - let builder = run.builder; - run.crate_or_deps("sysroot") - .path("library") - .lazy_default_condition(Box::new(|| !builder.download_rustc())) - } - - fn make_run(run: RunConfig<'_>) { - // If the paths include "library", build the entire standard library. - let has_alias = - run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); - let crates = if has_alias { Default::default() } else { run.cargo_crates_in_set() }; - - run.builder.ensure(Std { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - crates, - force_recompile: false, - }); - } - - /// Builds the standard library. - /// - /// This will build the standard library for a particular stage of the build - /// using the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let compiler = self.compiler; - - // When using `download-rustc`, we already have artifacts for the host available. Don't - // recompile them. - if builder.download_rustc() && target == builder.build.build - // NOTE: the beta compiler may generate different artifacts than the downloaded compiler, so - // its artifacts can't be reused. - && compiler.stage != 0 - // This check is specific to testing std itself; see `test::Std` for more details. - && !self.force_recompile - { - cp_rustc_component_to_ci_sysroot( - builder, - compiler, - builder.config.ci_rust_std_contents(), - ); - return; - } - - if builder.config.keep_stage.contains(&compiler.stage) - || builder.config.keep_stage_std.contains(&compiler.stage) - { - builder.info("Warning: Using a potentially old libstd. This may not behave well."); - - copy_third_party_objects(builder, &compiler, target); - copy_self_contained_objects(builder, &compiler, target); - - builder.ensure(StdLink::from_std(self, compiler)); - return; - } - - builder.update_submodule(&Path::new("library").join("stdarch")); - - // Profiler information requires LLVM's compiler-rt - if builder.config.profiler { - builder.update_submodule(&Path::new("src/llvm-project")); - } - - let mut target_deps = builder.ensure(StartupObjects { compiler, target }); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(Std::new(compiler_to_use, target)); - let msg = if compiler_to_use.host == target { - format!( - "Uplifting library (stage{} -> stage{})", - compiler_to_use.stage, compiler.stage - ) - } else { - format!( - "Uplifting library (stage{}:{} -> stage{}:{})", - compiler_to_use.stage, compiler_to_use.host, compiler.stage, target - ) - }; - builder.info(&msg); - - // Even if we're not building std this stage, the new sysroot must - // still contain the third party objects needed by various targets. - copy_third_party_objects(builder, &compiler, target); - copy_self_contained_objects(builder, &compiler, target); - - builder.ensure(StdLink::from_std(self, compiler_to_use)); - return; - } - - target_deps.extend(copy_third_party_objects(builder, &compiler, target)); - target_deps.extend(copy_self_contained_objects(builder, &compiler, target)); - - let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "build"); - std_cargo(builder, target, compiler.stage, &mut cargo); - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - // See src/bootstrap/synthetic_targets.rs - if target.is_synthetic() { - cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1"); - } - - let _guard = builder.msg( - Kind::Build, - compiler.stage, - format_args!("library artifacts{}", crate_description(&self.crates)), - compiler.host, - target, - ); - run_cargo( - builder, - cargo, - vec![], - &libstd_stamp(builder, compiler, target), - target_deps, - false, - false, - ); - - builder.ensure(StdLink::from_std( - self, - builder.compiler(compiler.stage, builder.config.build), - )); - } -} - -fn copy_and_stamp( - builder: &Builder<'_>, - libdir: &Path, - sourcedir: &Path, - name: &str, - target_deps: &mut Vec<(PathBuf, DependencyType)>, - dependency_type: DependencyType, -) { - let target = libdir.join(name); - builder.copy(&sourcedir.join(name), &target); - - target_deps.push((target, dependency_type)); -} - -fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf { - let libunwind_path = builder.ensure(llvm::Libunwind { target }); - let libunwind_source = libunwind_path.join("libunwind.a"); - let libunwind_target = libdir.join("libunwind.a"); - builder.copy(&libunwind_source, &libunwind_target); - libunwind_target -} - -/// Copies third party objects needed by various targets. -fn copy_third_party_objects( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec<(PathBuf, DependencyType)> { - let mut target_deps = vec![]; - - if builder.config.sanitizers_enabled(target) && compiler.stage != 0 { - // The sanitizers are only copied in stage1 or above, - // to avoid creating dependency on LLVM. - target_deps.extend( - copy_sanitizers(builder, &compiler, target) - .into_iter() - .map(|d| (d, DependencyType::Target)), - ); - } - - if target == "x86_64-fortanix-unknown-sgx" - || builder.config.llvm_libunwind(target) == LlvmLibunwind::InTree - && (target.contains("linux") || target.contains("fuchsia")) - { - let libunwind_path = - copy_llvm_libunwind(builder, target, &builder.sysroot_libdir(*compiler, target)); - target_deps.push((libunwind_path, DependencyType::Target)); - } - - target_deps -} - -/// Copies third party objects needed by various targets for self-contained linkage. -fn copy_self_contained_objects( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec<(PathBuf, DependencyType)> { - let libdir_self_contained = builder.sysroot_libdir(*compiler, target).join("self-contained"); - t!(fs::create_dir_all(&libdir_self_contained)); - let mut target_deps = vec![]; - - // Copies the libc and CRT objects. - // - // rustc historically provides a more self-contained installation for musl targets - // not requiring the presence of a native musl toolchain. For example, it can fall back - // to using gcc from a glibc-targeting toolchain for linking. - // To do that we have to distribute musl startup objects as a part of Rust toolchain - // and link with them manually in the self-contained mode. - if target.contains("musl") && !target.contains("unikraft") { - let srcdir = builder.musl_libdir(target).unwrap_or_else(|| { - panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple) - }); - for &obj in &["libc.a", "crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] { - copy_and_stamp( - builder, - &libdir_self_contained, - &srcdir, - obj, - &mut target_deps, - DependencyType::TargetSelfContained, - ); - } - let crt_path = builder.ensure(llvm::CrtBeginEnd { target }); - for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { - let src = crt_path.join(obj); - let target = libdir_self_contained.join(obj); - builder.copy(&src, &target); - target_deps.push((target, DependencyType::TargetSelfContained)); - } - - if !target.starts_with("s390x") { - let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained); - target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); - } - } else if target.contains("-wasi") { - let srcdir = builder - .wasi_root(target) - .unwrap_or_else(|| { - panic!("Target {:?} does not have a \"wasi-root\" key", target.triple) - }) - .join("lib") - .join(target.to_string().replace("-preview1", "")); - for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { - copy_and_stamp( - builder, - &libdir_self_contained, - &srcdir, - obj, - &mut target_deps, - DependencyType::TargetSelfContained, - ); - } - } else if target.ends_with("windows-gnu") { - for obj in ["crt2.o", "dllcrt2.o"].iter() { - let src = compiler_file(builder, &builder.cc(target), target, CLang::C, obj); - let target = libdir_self_contained.join(obj); - builder.copy(&src, &target); - target_deps.push((target, DependencyType::TargetSelfContained)); - } - } - - target_deps -} - -/// Configure cargo to compile the standard library, adding appropriate env vars -/// and such. -pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) { - if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") { - cargo.env("MACOSX_DEPLOYMENT_TARGET", target); - } - - if let Some(path) = builder.config.profiler_path(target) { - cargo.env("LLVM_PROFILER_RT_LIB", path); - } - - // Determine if we're going to compile in optimized C intrinsics to - // the `compiler-builtins` crate. These intrinsics live in LLVM's - // `compiler-rt` repository, but our `src/llvm-project` submodule isn't - // always checked out, so we need to conditionally look for this. (e.g. if - // an external LLVM is used we skip the LLVM submodule checkout). - // - // Note that this shouldn't affect the correctness of `compiler-builtins`, - // but only its speed. Some intrinsics in C haven't been translated to Rust - // yet but that's pretty rare. Other intrinsics have optimized - // implementations in C which have only had slower versions ported to Rust, - // so we favor the C version where we can, but it's not critical. - // - // If `compiler-rt` is available ensure that the `c` feature of the - // `compiler-builtins` crate is enabled and it's configured to learn where - // `compiler-rt` is located. - let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt"); - let compiler_builtins_c_feature = if compiler_builtins_root.exists() { - // Note that `libprofiler_builtins/build.rs` also computes this so if - // you're changing something here please also change that. - cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root); - " compiler-builtins-c" - } else { - "" - }; - - // `libtest` uses this to know whether or not to support - // `-Zunstable-options`. - if !builder.unstable_features() { - cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); - } - - let mut features = String::new(); - - // Cranelift doesn't support `asm`. - if stage != 0 && builder.config.default_codegen_backend().unwrap_or_default() == "cranelift" { - features += " compiler-builtins-no-asm"; - } - - if builder.no_std(target) == Some(true) { - features += " compiler-builtins-mem"; - if !target.starts_with("bpf") { - features.push_str(compiler_builtins_c_feature); - } - - // for no-std targets we only compile a few no_std crates - cargo - .args(&["-p", "alloc"]) - .arg("--manifest-path") - .arg(builder.src.join("library/alloc/Cargo.toml")) - .arg("--features") - .arg(features); - } else { - features += &builder.std_features(target); - features.push_str(compiler_builtins_c_feature); - - cargo - .arg("--features") - .arg(features) - .arg("--manifest-path") - .arg(builder.src.join("library/sysroot/Cargo.toml")); - - // Help the libc crate compile by assisting it in finding various - // sysroot native libraries. - if target.contains("musl") { - if let Some(p) = builder.musl_libdir(target) { - let root = format!("native={}", p.to_str().unwrap()); - cargo.rustflag("-L").rustflag(&root); - } - } - - if target.contains("-wasi") { - if let Some(p) = builder.wasi_root(target) { - let root = format!( - "native={}/lib/{}", - p.to_str().unwrap(), - target.to_string().replace("-preview1", "") - ); - cargo.rustflag("-L").rustflag(&root); - } - } - } - - // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that - // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be - // built with bitcode so that the produced rlibs can be used for both LTO - // builds (which use bitcode) and non-LTO builds (which use object code). - // So we override the override here! - // - // But we don't bother for the stage 0 compiler because it's never used - // with LTO. - if stage >= 1 { - cargo.rustflag("-Cembed-bitcode=yes"); - } - if builder.config.rust_lto == RustcLto::Off { - cargo.rustflag("-Clto=off"); - } - - // By default, rustc does not include unwind tables unless they are required - // for a particular target. They are not required by RISC-V targets, but - // compiling the standard library with them means that users can get - // backtraces without having to recompile the standard library themselves. - // - // This choice was discussed in https://github.com/rust-lang/rust/pull/69890 - if target.contains("riscv") { - cargo.rustflag("-Cforce-unwind-tables=yes"); - } - - let html_root = - format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),); - cargo.rustflag(&html_root); - cargo.rustdocflag(&html_root); - - cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -struct StdLink { - pub compiler: Compiler, - pub target_compiler: Compiler, - pub target: TargetSelection, - /// Not actually used; only present to make sure the cache invalidation is correct. - crates: Interned>, - /// See [`Std::force_recompile`]. - force_recompile: bool, -} - -impl StdLink { - fn from_std(std: Std, host_compiler: Compiler) -> Self { - Self { - compiler: host_compiler, - target_compiler: std.compiler, - target: std.target, - crates: std.crates, - force_recompile: std.force_recompile, - } - } -} - -impl Step for StdLink { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Link all libstd rlibs/dylibs into the sysroot location. - /// - /// Links those artifacts generated by `compiler` to the `stage` compiler's - /// sysroot for the specified `host` and `target`. - /// - /// Note that this assumes that `compiler` has already generated the libstd - /// libraries for `target`, and this method will find them in the relevant - /// output directory. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target_compiler = self.target_compiler; - let target = self.target; - - // NOTE: intentionally does *not* check `target == builder.build` to avoid having to add the same check in `test::Crate`. - let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() { - // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too - let lib = builder.sysroot_libdir_relative(self.compiler); - let sysroot = builder.ensure(crate::compile::Sysroot { - compiler: self.compiler, - force_recompile: self.force_recompile, - }); - let libdir = sysroot.join(lib).join("rustlib").join(target.triple).join("lib"); - let hostdir = sysroot.join(lib).join("rustlib").join(compiler.host.triple).join("lib"); - (INTERNER.intern_path(libdir), INTERNER.intern_path(hostdir)) - } else { - let libdir = builder.sysroot_libdir(target_compiler, target); - let hostdir = builder.sysroot_libdir(target_compiler, compiler.host); - (libdir, hostdir) - }; - - add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); - - // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0` - // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta, - // and is not set to a custom path. - if compiler.stage == 0 - && builder - .build - .config - .initial_rustc - .starts_with(builder.out.join(&compiler.host.triple).join("stage0/bin")) - { - // Copy bin files from stage0/bin to stage0-sysroot/bin - let sysroot = builder.out.join(&compiler.host.triple).join("stage0-sysroot"); - - let host = compiler.host.triple; - let stage0_bin_dir = builder.out.join(&host).join("stage0/bin"); - let sysroot_bin_dir = sysroot.join("bin"); - t!(fs::create_dir_all(&sysroot_bin_dir)); - builder.cp_r(&stage0_bin_dir, &sysroot_bin_dir); - - // Copy all *.so files from stage0/lib to stage0-sysroot/lib - let stage0_lib_dir = builder.out.join(&host).join("stage0/lib"); - if let Ok(files) = fs::read_dir(&stage0_lib_dir) { - for file in files { - let file = t!(file); - let path = file.path(); - if path.is_file() && is_dylib(&file.file_name().into_string().unwrap()) { - builder.copy(&path, &sysroot.join("lib").join(path.file_name().unwrap())); - } - } - } - - // Copy codegen-backends from stage0 - let sysroot_codegen_backends = builder.sysroot_codegen_backends(compiler); - t!(fs::create_dir_all(&sysroot_codegen_backends)); - let stage0_codegen_backends = builder - .out - .join(&host) - .join("stage0/lib/rustlib") - .join(&host) - .join("codegen-backends"); - builder.cp_r(&stage0_codegen_backends, &sysroot_codegen_backends); - } - } -} - -/// Copies sanitizer runtime libraries into target libdir. -fn copy_sanitizers( - builder: &Builder<'_>, - compiler: &Compiler, - target: TargetSelection, -) -> Vec { - let runtimes: Vec = builder.ensure(llvm::Sanitizers { target }); - - if builder.config.dry_run() { - return Vec::new(); - } - - let mut target_deps = Vec::new(); - let libdir = builder.sysroot_libdir(*compiler, target); - - for runtime in &runtimes { - let dst = libdir.join(&runtime.name); - builder.copy(&runtime.path, &dst); - - // The `aarch64-apple-ios-macabi` and `x86_64-apple-ios-macabi` are also supported for - // sanitizers, but they share a sanitizer runtime with `${arch}-apple-darwin`, so we do - // not list them here to rename and sign the runtime library. - if target == "x86_64-apple-darwin" - || target == "aarch64-apple-darwin" - || target == "aarch64-apple-ios" - || target == "aarch64-apple-ios-sim" - || target == "x86_64-apple-ios" - { - // Update the library’s install name to reflect that it has been renamed. - apple_darwin_update_library_name(&dst, &format!("@rpath/{}", &runtime.name)); - // Upon renaming the install name, the code signature of the file will invalidate, - // so we will sign it again. - apple_darwin_sign_file(&dst); - } - - target_deps.push(dst); - } - - target_deps -} - -fn apple_darwin_update_library_name(library_path: &Path, new_name: &str) { - let status = Command::new("install_name_tool") - .arg("-id") - .arg(new_name) - .arg(library_path) - .status() - .expect("failed to execute `install_name_tool`"); - assert!(status.success()); -} - -fn apple_darwin_sign_file(file_path: &Path) { - let status = Command::new("codesign") - .arg("-f") // Force to rewrite the existing signature - .arg("-s") - .arg("-") - .arg(file_path) - .status() - .expect("failed to execute `codesign`"); - assert!(status.success()); -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct StartupObjects { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for StartupObjects { - type Output = Vec<(PathBuf, DependencyType)>; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("library/rtstartup") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(StartupObjects { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - }); - } - - /// Builds and prepare startup objects like rsbegin.o and rsend.o - /// - /// These are primarily used on Windows right now for linking executables/dlls. - /// They don't require any library support as they're just plain old object - /// files, so we just use the nightly snapshot compiler to always build them (as - /// no other compilers are guaranteed to be available). - fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> { - let for_compiler = self.compiler; - let target = self.target; - if !target.ends_with("windows-gnu") { - return vec![]; - } - - let mut target_deps = vec![]; - - let src_dir = &builder.src.join("library").join("rtstartup"); - let dst_dir = &builder.native_dir(target).join("rtstartup"); - let sysroot_dir = &builder.sysroot_libdir(for_compiler, target); - t!(fs::create_dir_all(dst_dir)); - - for file in &["rsbegin", "rsend"] { - let src_file = &src_dir.join(file.to_string() + ".rs"); - let dst_file = &dst_dir.join(file.to_string() + ".o"); - if !up_to_date(src_file, dst_file) { - let mut cmd = Command::new(&builder.initial_rustc); - cmd.env("RUSTC_BOOTSTRAP", "1"); - if !builder.local_rebuild { - // a local_rebuild compiler already has stage1 features - cmd.arg("--cfg").arg("bootstrap"); - } - builder.run( - cmd.arg("--target") - .arg(target.rustc_target_arg()) - .arg("--emit=obj") - .arg("-o") - .arg(dst_file) - .arg(src_file), - ); - } - - let target = sysroot_dir.join((*file).to_string() + ".o"); - builder.copy(dst_file, &target); - target_deps.push((target, DependencyType::Target)); - } - - target_deps - } -} - -fn cp_rustc_component_to_ci_sysroot( - builder: &Builder<'_>, - compiler: Compiler, - contents: Vec, -) { - let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); - let ci_rustc_dir = builder.config.ci_rustc_dir(); - - for file in contents { - let src = ci_rustc_dir.join(&file); - let dst = sysroot.join(file); - if src.is_dir() { - t!(fs::create_dir_all(dst)); - } else { - builder.copy(&src, &dst); - } - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Rustc { - pub target: TargetSelection, - pub compiler: Compiler, - /// Whether to build a subset of crates, rather than the whole compiler. - /// - /// This should only be requested by the user, not used within rustbuild itself. - /// Using it within rustbuild can lead to confusing situation where lints are replayed - /// in two different steps. - crates: Interned>, -} - -impl Rustc { - pub fn new(compiler: Compiler, target: TargetSelection) -> Self { - Self { target, compiler, crates: Default::default() } - } -} - -impl Step for Rustc { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let mut crates = run.builder.in_tree_crates("rustc-main", None); - for (i, krate) in crates.iter().enumerate() { - // We can't allow `build rustc` as an alias for this Step, because that's reserved by `Assemble`. - // Ideally Assemble would use `build compiler` instead, but that seems too confusing to be worth the breaking change. - if krate.name == "rustc-main" { - crates.swap_remove(i); - break; - } - } - run.crates(crates) - } - - fn make_run(run: RunConfig<'_>) { - let crates = run.cargo_crates_in_set(); - run.builder.ensure(Rustc { - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - target: run.target, - crates, - }); - } - - /// Builds the compiler. - /// - /// This will build the compiler for a particular stage of the build using - /// the `compiler` targeting the `target` architecture. The artifacts - /// created will also be linked into the sysroot directory. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - - // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, - // so its artifacts can't be reused. - if builder.download_rustc() && compiler.stage != 0 { - // Copy the existing artifacts instead of rebuilding them. - // NOTE: this path is only taken for tools linking to rustc-dev (including ui-fulldeps tests). - cp_rustc_component_to_ci_sysroot( - builder, - compiler, - builder.config.ci_rustc_dev_contents(), - ); - return; - } - - builder.ensure(Std::new(compiler, target)); - - if builder.config.keep_stage.contains(&compiler.stage) { - builder.info("Warning: Using a potentially old librustc. This may not behave well."); - builder.info("Warning: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); - builder.ensure(RustcLink::from_rustc(self, compiler)); - return; - } - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(Rustc::new(compiler_to_use, target)); - let msg = if compiler_to_use.host == target { - format!( - "Uplifting rustc (stage{} -> stage{})", - compiler_to_use.stage, - compiler.stage + 1 - ) - } else { - format!( - "Uplifting rustc (stage{}:{} -> stage{}:{})", - compiler_to_use.stage, - compiler_to_use.host, - compiler.stage + 1, - target - ) - }; - builder.info(&msg); - builder.ensure(RustcLink::from_rustc(self, compiler_to_use)); - return; - } - - // Ensure that build scripts and proc macros have a std / libproc_macro to link against. - builder.ensure(Std::new( - builder.compiler(self.compiler.stage, builder.config.build), - builder.config.build, - )); - - let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "build"); - rustc_cargo(builder, &mut cargo, target, compiler.stage); - - if builder.config.rust_profile_use.is_some() - && builder.config.rust_profile_generate.is_some() - { - panic!("Cannot use and generate PGO profiles at the same time"); - } - - // With LLD, we can use ICF (identical code folding) to reduce the executable size - // of librustc_driver/rustc and to improve i-cache utilization. - // - // -Wl,[link options] doesn't work on MSVC. However, /OPT:ICF (technically /OPT:REF,ICF) - // is already on by default in MSVC optimized builds, which is interpreted as --icf=all: - // https://github.com/llvm/llvm-project/blob/3329cec2f79185bafd678f310fafadba2a8c76d2/lld/COFF/Driver.cpp#L1746 - // https://github.com/rust-lang/rust/blob/f22819bcce4abaff7d1246a56eec493418f9f4ee/compiler/rustc_codegen_ssa/src/back/linker.rs#L827 - if builder.config.use_lld && !compiler.host.contains("msvc") { - cargo.rustflag("-Clink-args=-Wl,--icf=all"); - } - - let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { - if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-generate={path}")); - // Apparently necessary to avoid overflowing the counters during - // a Cargo build profile - cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); - true - } else { - false - } - } else if let Some(path) = &builder.config.rust_profile_use { - if compiler.stage == 1 { - cargo.rustflag(&format!("-Cprofile-use={path}")); - cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); - true - } else { - false - } - } else { - false - }; - if is_collecting { - // Ensure paths to Rust sources are relative, not absolute. - cargo.rustflag(&format!( - "-Cllvm-args=-static-func-strip-dirname-prefix={}", - builder.config.src.components().count() - )); - } - - // We currently don't support cross-crate LTO in stage0. This also isn't hugely necessary - // and may just be a time sink. - if compiler.stage != 0 { - match builder.config.rust_lto { - RustcLto::Thin | RustcLto::Fat => { - // Since using LTO for optimizing dylibs is currently experimental, - // we need to pass -Zdylib-lto. - cargo.rustflag("-Zdylib-lto"); - // Cargo by default passes `-Cembed-bitcode=no` and doesn't pass `-Clto` when - // compiling dylibs (and their dependencies), even when LTO is enabled for the - // crate. Therefore, we need to override `-Clto` and `-Cembed-bitcode` here. - let lto_type = match builder.config.rust_lto { - RustcLto::Thin => "thin", - RustcLto::Fat => "fat", - _ => unreachable!(), - }; - cargo.rustflag(&format!("-Clto={lto_type}")); - cargo.rustflag("-Cembed-bitcode=yes"); - } - RustcLto::ThinLocal => { /* Do nothing, this is the default */ } - RustcLto::Off => { - cargo.rustflag("-Clto=off"); - } - } - } else if builder.config.rust_lto == RustcLto::Off { - cargo.rustflag("-Clto=off"); - } - - for krate in &*self.crates { - cargo.arg("-p").arg(krate); - } - - let _guard = builder.msg_sysroot_tool( - Kind::Build, - compiler.stage, - format_args!("compiler artifacts{}", crate_description(&self.crates)), - compiler.host, - target, - ); - let stamp = librustc_stamp(builder, compiler, target); - run_cargo( - builder, - cargo, - vec![], - &stamp, - vec![], - false, - true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. - ); - - // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain - // unexpected debuginfo from dependencies, for example from the C++ standard library used in - // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with - // debuginfo (via the debuginfo level of the executables using it): strip this debuginfo - // away after the fact. - if builder.config.rust_debuginfo_level_rustc == DebuginfoLevel::None - && builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None - { - let target_root_dir = stamp.parent().unwrap(); - let rustc_driver = target_root_dir.join("librustc_driver.so"); - strip_debug(builder, target, &rustc_driver); - } - - builder.ensure(RustcLink::from_rustc( - self, - builder.compiler(compiler.stage, builder.config.build), - )); - } -} - -pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection, stage: u32) { - cargo - .arg("--features") - .arg(builder.rustc_features(builder.kind)) - .arg("--manifest-path") - .arg(builder.src.join("compiler/rustc/Cargo.toml")); - - cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); - - rustc_cargo_env(builder, cargo, target, stage); -} - -pub fn rustc_cargo_env( - builder: &Builder<'_>, - cargo: &mut Cargo, - target: TargetSelection, - stage: u32, -) { - // Set some configuration variables picked up by build scripts and - // the compiler alike - cargo - .env("CFG_RELEASE", builder.rust_release()) - .env("CFG_RELEASE_CHANNEL", &builder.config.channel) - .env("CFG_VERSION", builder.rust_version()); - - if let Some(backend) = builder.config.default_codegen_backend() { - cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); - } - - let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib")); - let target_config = builder.config.target_config.get(&target); - - cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); - - if let Some(ref ver_date) = builder.rust_info().commit_date() { - cargo.env("CFG_VER_DATE", ver_date); - } - if let Some(ref ver_hash) = builder.rust_info().sha() { - cargo.env("CFG_VER_HASH", ver_hash); - } - if !builder.unstable_features() { - cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); - } - - // Prefer the current target's own default_linker, else a globally - // specified one. - if let Some(s) = target_config.and_then(|c| c.default_linker.as_ref()) { - cargo.env("CFG_DEFAULT_LINKER", s); - } else if let Some(ref s) = builder.config.rustc_default_linker { - cargo.env("CFG_DEFAULT_LINKER", s); - } - - if builder.config.rustc_parallel { - // keep in sync with `bootstrap/lib.rs:Build::rustc_features` - // `cfg` option for rustc, `features` option for cargo, for conditional compilation - cargo.rustflag("--cfg=parallel_compiler"); - cargo.rustdocflag("--cfg=parallel_compiler"); - } - if builder.config.rust_verify_llvm_ir { - cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); - } - - // Note that this is disabled if LLVM itself is disabled or we're in a check - // build. If we are in a check build we still go ahead here presuming we've - // detected that LLVM is already built and good to go which helps prevent - // busting caches (e.g. like #71152). - if builder.config.llvm_enabled() { - let building_is_expensive = crate::llvm::prebuilt_llvm_config(builder, target).is_err(); - // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler - let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; - let should_skip_build = building_is_expensive && can_skip_build; - if !should_skip_build { - rustc_llvm_env(builder, cargo, target) - } - } -} - -/// Pass down configuration from the LLVM build into the build of -/// rustc_llvm and rustc_codegen_llvm. -fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { - let target_config = builder.config.target_config.get(&target); - - if builder.is_rust_llvm(target) { - cargo.env("LLVM_RUSTLLVM", "1"); - } - let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target }); - cargo.env("LLVM_CONFIG", &llvm_config); - if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { - cargo.env("CFG_LLVM_ROOT", s); - } - - // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script - // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by - // whitespace. - // - // For example: - // - on windows, when `clang-cl` is used with instrumentation, we need to manually add - // clang's runtime library resource directory so that the profiler runtime library can be - // found. This is to avoid the linker errors about undefined references to - // `__llvm_profile_instrument_memop` when linking `rustc_driver`. - let mut llvm_linker_flags = String::new(); - if builder.config.llvm_profile_generate && target.contains("msvc") { - if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { - // Add clang's runtime library directory to the search path - let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); - llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); - } - } - - // The config can also specify its own llvm linker flags. - if let Some(ref s) = builder.config.llvm_ldflags { - if !llvm_linker_flags.is_empty() { - llvm_linker_flags.push_str(" "); - } - llvm_linker_flags.push_str(s); - } - - // Set the linker flags via the env var that `rustc_llvm`'s build script will read. - if !llvm_linker_flags.is_empty() { - cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); - } - - // Building with a static libstdc++ is only supported on linux right now, - // not for MSVC or macOS - if builder.config.llvm_static_stdcpp - && !target.contains("freebsd") - && !target.contains("msvc") - && !target.contains("apple") - && !target.contains("solaris") - { - let file = compiler_file( - builder, - &builder.cxx(target).unwrap(), - target, - CLang::Cxx, - "libstdc++.a", - ); - cargo.env("LLVM_STATIC_STDCPP", file); - } - if builder.llvm_link_shared() { - cargo.env("LLVM_LINK_SHARED", "1"); - } - if builder.config.llvm_use_libcxx { - cargo.env("LLVM_USE_LIBCXX", "1"); - } - if builder.config.llvm_optimize && !builder.config.llvm_release_debuginfo { - cargo.env("LLVM_NDEBUG", "1"); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -struct RustcLink { - pub compiler: Compiler, - pub target_compiler: Compiler, - pub target: TargetSelection, - /// Not actually used; only present to make sure the cache invalidation is correct. - crates: Interned>, -} - -impl RustcLink { - fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self { - Self { - compiler: host_compiler, - target_compiler: rustc.compiler, - target: rustc.target, - crates: rustc.crates, - } - } -} - -impl Step for RustcLink { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Same as `std_link`, only for librustc - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target_compiler = self.target_compiler; - let target = self.target; - add_to_sysroot( - builder, - &builder.sysroot_libdir(target_compiler, target), - &builder.sysroot_libdir(target_compiler, compiler.host), - &librustc_stamp(builder, compiler, target), - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CodegenBackend { - pub target: TargetSelection, - pub compiler: Compiler, - pub backend: Interned, -} - -fn needs_codegen_config(run: &RunConfig<'_>) -> bool { - let mut needs_codegen_cfg = false; - for path_set in &run.paths { - needs_codegen_cfg = match path_set { - PathSet::Set(set) => set.iter().any(|p| is_codegen_cfg_needed(p, run)), - PathSet::Suite(suite) => is_codegen_cfg_needed(&suite, run), - } - } - needs_codegen_cfg -} - -pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; - -fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { - if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) { - let mut needs_codegen_backend_config = true; - for &backend in &run.builder.config.rust_codegen_backends { - if path - .path - .to_str() - .unwrap() - .ends_with(&(CODEGEN_BACKEND_PREFIX.to_owned() + &backend)) - { - needs_codegen_backend_config = false; - } - } - if needs_codegen_backend_config { - run.builder.info( - "Warning: no codegen-backends config matched the requested path to build a codegen backend. \ - Help: add backend to codegen-backends in config.toml.", - ); - return true; - } - } - - return false; -} - -impl Step for CodegenBackend { - type Output = (); - const ONLY_HOSTS: bool = true; - // Only the backends specified in the `codegen-backends` entry of `config.toml` are built. - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) - } - - fn make_run(run: RunConfig<'_>) { - if needs_codegen_config(&run) { - return; - } - - for &backend in &run.builder.config.rust_codegen_backends { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - run.builder.ensure(CodegenBackend { - target: run.target, - compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), - backend, - }); - } - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - let backend = self.backend; - - builder.ensure(Rustc::new(compiler, target)); - - if builder.config.keep_stage.contains(&compiler.stage) { - builder.info( - "Warning: Using a potentially old codegen backend. \ - This may not behave well.", - ); - // Codegen backends are linked separately from this step today, so we don't do - // anything here. - return; - } - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - if compiler_to_use != compiler { - builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend }); - return; - } - - let out_dir = builder.cargo_out(compiler, Mode::Codegen, target); - - let mut cargo = builder.cargo(compiler, Mode::Codegen, SourceType::InTree, target, "build"); - cargo - .arg("--manifest-path") - .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); - rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - let tmp_stamp = out_dir.join(".tmp.stamp"); - - let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target); - let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false); - if builder.config.dry_run() { - return; - } - let mut files = files.into_iter().filter(|f| { - let filename = f.file_name().unwrap().to_str().unwrap(); - is_dylib(filename) && filename.contains("rustc_codegen_") - }); - let codegen_backend = match files.next() { - Some(f) => f, - None => panic!("no dylibs built for codegen backend?"), - }; - if let Some(f) = files.next() { - panic!( - "codegen backend built two dylibs:\n{}\n{}", - codegen_backend.display(), - f.display() - ); - } - let stamp = codegen_backend_stamp(builder, compiler, target, backend); - let codegen_backend = codegen_backend.to_str().unwrap(); - t!(fs::write(&stamp, &codegen_backend)); - } -} - -/// Creates the `codegen-backends` folder for a compiler that's about to be -/// assembled as a complete compiler. -/// -/// This will take the codegen artifacts produced by `compiler` and link them -/// into an appropriate location for `target_compiler` to be a functional -/// compiler. -fn copy_codegen_backends_to_sysroot( - builder: &Builder<'_>, - compiler: Compiler, - target_compiler: Compiler, -) { - let target = target_compiler.host; - - // Note that this step is different than all the other `*Link` steps in - // that it's not assembling a bunch of libraries but rather is primarily - // moving the codegen backend into place. The codegen backend of rustc is - // not linked into the main compiler by default but is rather dynamically - // selected at runtime for inclusion. - // - // Here we're looking for the output dylib of the `CodegenBackend` step and - // we're copying that into the `codegen-backends` folder. - let dst = builder.sysroot_codegen_backends(target_compiler); - t!(fs::create_dir_all(&dst), dst); - - if builder.config.dry_run() { - return; - } - - for backend in builder.config.rust_codegen_backends.iter() { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - let stamp = codegen_backend_stamp(builder, compiler, target, *backend); - let dylib = t!(fs::read_to_string(&stamp)); - let file = Path::new(&dylib); - let filename = file.file_name().unwrap().to_str().unwrap(); - // change `librustc_codegen_cranelift-xxxxxx.so` to - // `librustc_codegen_cranelift-release.so` - let target_filename = { - let dash = filename.find('-').unwrap(); - let dot = filename.find('.').unwrap(); - format!("{}-{}{}", &filename[..dash], builder.rust_release(), &filename[dot..]) - }; - builder.copy(&file, &dst.join(target_filename)); - } -} - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp") -} - -/// Cargo's output path for librustc in a given stage, compiled by a particular -/// compiler for the specified target. -pub fn librustc_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> PathBuf { - builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp") -} - -/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular -/// compiler for the specified target and backend. -fn codegen_backend_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - backend: Interned, -) -> PathBuf { - builder - .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{backend}.stamp")) -} - -pub fn compiler_file( - builder: &Builder<'_>, - compiler: &Path, - target: TargetSelection, - c: CLang, - file: &str, -) -> PathBuf { - if builder.config.dry_run() { - return PathBuf::new(); - } - let mut cmd = Command::new(compiler); - cmd.args(builder.cflags(target, GitRepo::Rustc, c)); - cmd.arg(format!("-print-file-name={file}")); - let out = output(&mut cmd); - PathBuf::from(out.trim()) -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Sysroot { - pub compiler: Compiler, - /// See [`Std::force_recompile`]. - force_recompile: bool, -} - -impl Sysroot { - pub(crate) fn new(compiler: Compiler) -> Self { - Sysroot { compiler, force_recompile: false } - } -} - -impl Step for Sysroot { - type Output = Interned; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Returns the sysroot for the `compiler` specified that *this build system - /// generates*. - /// - /// That is, the sysroot for the stage0 compiler is not what the compiler - /// thinks it is by default, but it's the same as the default for stages - /// 1-3. - fn run(self, builder: &Builder<'_>) -> Interned { - let compiler = self.compiler; - let host_dir = builder.out.join(&compiler.host.triple); - - let sysroot_dir = |stage| { - if stage == 0 { - host_dir.join("stage0-sysroot") - } else if self.force_recompile && stage == compiler.stage { - host_dir.join(format!("stage{stage}-test-sysroot")) - } else if builder.download_rustc() && compiler.stage != builder.top_stage { - host_dir.join("ci-rustc-sysroot") - } else { - host_dir.join(format!("stage{}", stage)) - } - }; - let sysroot = sysroot_dir(compiler.stage); - - builder.verbose(&format!("Removing sysroot {} to avoid caching bugs", sysroot.display())); - let _ = fs::remove_dir_all(&sysroot); - t!(fs::create_dir_all(&sysroot)); - - // In some cases(see https://github.com/rust-lang/rust/issues/109314), when the stage0 - // compiler relies on more recent version of LLVM than the beta compiler, it may not - // be able to locate the correct LLVM in the sysroot. This situation typically occurs - // when we upgrade LLVM version while the beta compiler continues to use an older version. - // - // Make sure to add the correct version of LLVM into the stage0 sysroot. - if compiler.stage == 0 { - dist::maybe_install_llvm_target(builder, compiler.host, &sysroot); - } - - // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.download_rustc() && compiler.stage != 0 { - assert_eq!( - builder.config.build, compiler.host, - "Cross-compiling is not yet supported with `download-rustc`", - ); - - // #102002, cleanup old toolchain folders when using download-rustc so people don't use them by accident. - for stage in 0..=2 { - if stage != compiler.stage { - let dir = sysroot_dir(stage); - if !dir.ends_with("ci-rustc-sysroot") { - let _ = fs::remove_dir_all(dir); - } - } - } - - // Copy the compiler into the correct sysroot. - // NOTE(#108767): We intentionally don't copy `rustc-dev` artifacts until they're requested with `builder.ensure(Rustc)`. - // This fixes an issue where we'd have multiple copies of libc in the sysroot with no way to tell which to load. - // There are a few quirks of bootstrap that interact to make this reliable: - // 1. The order `Step`s are run is hard-coded in `builder.rs` and not configurable. This - // avoids e.g. reordering `test::UiFulldeps` before `test::Ui` and causing the latter to - // fail because of duplicate metadata. - // 2. The sysroot is deleted and recreated between each invocation, so running `x test - // ui-fulldeps && x test ui` can't cause failures. - let mut filtered_files = Vec::new(); - let mut add_filtered_files = |suffix, contents| { - for path in contents { - let path = Path::new(&path); - if path.parent().map_or(false, |parent| parent.ends_with(&suffix)) { - filtered_files.push(path.file_name().unwrap().to_owned()); - } - } - }; - let suffix = format!("lib/rustlib/{}/lib", compiler.host); - add_filtered_files(suffix.as_str(), builder.config.ci_rustc_dev_contents()); - // NOTE: we can't copy std eagerly because `stage2-test-sysroot` needs to have only the - // newly compiled std, not the downloaded std. - add_filtered_files("lib", builder.config.ci_rust_std_contents()); - - let filtered_extensions = [ - OsStr::new("rmeta"), - OsStr::new("rlib"), - // FIXME: this is wrong when compiler.host != build, but we don't support that today - OsStr::new(std::env::consts::DLL_EXTENSION), - ]; - let ci_rustc_dir = builder.config.ci_rustc_dir(); - builder.cp_filtered(&ci_rustc_dir, &sysroot, &|path| { - if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { - return true; - } - if !path.parent().map_or(true, |p| p.ends_with(&suffix)) { - return true; - } - if !filtered_files.iter().all(|f| f != path.file_name().unwrap()) { - builder.verbose_than(1, &format!("ignoring {}", path.display())); - false - } else { - true - } - }); - } - - // Symlink the source root into the same location inside the sysroot, - // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), - // so that any tools relying on `rust-src` also work for local builds, - // and also for translating the virtual `/rustc/$hash` back to the real - // directory (for running tests with `rust.remap-debuginfo = true`). - let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); - t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); - let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); - if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { - eprintln!( - "warning: creating symbolic link `{}` to `{}` failed with {}", - sysroot_lib_rustlib_src_rust.display(), - builder.src.display(), - e, - ); - if builder.config.rust_remap_debuginfo { - eprintln!( - "warning: some `tests/ui` tests will fail when lacking `{}`", - sysroot_lib_rustlib_src_rust.display(), - ); - } - } - // Same for the rustc-src component. - let sysroot_lib_rustlib_rustcsrc = sysroot.join("lib/rustlib/rustc-src"); - t!(fs::create_dir_all(&sysroot_lib_rustlib_rustcsrc)); - let sysroot_lib_rustlib_rustcsrc_rust = sysroot_lib_rustlib_rustcsrc.join("rust"); - if let Err(e) = - symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_rustcsrc_rust) - { - eprintln!( - "warning: creating symbolic link `{}` to `{}` failed with {}", - sysroot_lib_rustlib_rustcsrc_rust.display(), - builder.src.display(), - e, - ); - } - - INTERNER.intern_path(sysroot) - } -} - -#[derive(Debug, Copy, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] -pub struct Assemble { - /// The compiler which we will produce in this step. Assemble itself will - /// take care of ensuring that the necessary prerequisites to do so exist, - /// that is, this target can be a stage2 compiler and Assemble will build - /// previous stages for you. - pub target_compiler: Compiler, -} - -impl Step for Assemble { - type Output = Compiler; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("compiler/rustc").path("compiler") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Assemble { - target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target), - }); - } - - /// Prepare a new compiler from the artifacts in `stage` - /// - /// This will assemble a compiler in `build/$host/stage$stage`. The compiler - /// must have been previously produced by the `stage - 1` builder.build - /// compiler. - fn run(self, builder: &Builder<'_>) -> Compiler { - let target_compiler = self.target_compiler; - - if target_compiler.stage == 0 { - assert_eq!( - builder.config.build, target_compiler.host, - "Cannot obtain compiler for non-native build triple at stage 0" - ); - // The stage 0 compiler for the build triple is always pre-built. - return target_compiler; - } - - // Get the compiler that we'll use to bootstrap ourselves. - // - // Note that this is where the recursive nature of the bootstrap - // happens, as this will request the previous stage's compiler on - // downwards to stage 0. - // - // Also note that we're building a compiler for the host platform. We - // only assume that we can run `build` artifacts, which means that to - // produce some other architecture compiler we need to start from - // `build` to get there. - // - // FIXME: It may be faster if we build just a stage 1 compiler and then - // use that to bootstrap this compiler forward. - let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); - - // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. - if builder.download_rustc() { - let sysroot = - builder.ensure(Sysroot { compiler: target_compiler, force_recompile: false }); - // Ensure that `libLLVM.so` ends up in the newly created target directory, - // so that tools using `rustc_private` can use it. - dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); - // Lower stages use `ci-rustc-sysroot`, not stageN - if target_compiler.stage == builder.top_stage { - builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage=target_compiler.stage)); - } - return target_compiler; - } - - // Build the libraries for this compiler to link to (i.e., the libraries - // it uses at runtime). NOTE: Crates the target compiler compiles don't - // link to these. (FIXME: Is that correct? It seems to be correct most - // of the time but I think we do link to these for stage2/bin compilers - // when not performing a full bootstrap). - builder.ensure(Rustc::new(build_compiler, target_compiler.host)); - - // FIXME: For now patch over problems noted in #90244 by early returning here, even though - // we've not properly assembled the target sysroot. A full fix is pending further investigation, - // for now full bootstrap usage is rare enough that this is OK. - if target_compiler.stage >= 3 && !builder.config.full_bootstrap { - return target_compiler; - } - - for &backend in builder.config.rust_codegen_backends.iter() { - if backend == "llvm" { - continue; // Already built as part of rustc - } - - builder.ensure(CodegenBackend { - compiler: build_compiler, - target: target_compiler.host, - backend, - }); - } - - let lld_install = if builder.config.lld_enabled { - Some(builder.ensure(llvm::Lld { target: target_compiler.host })) - } else { - None - }; - - let stage = target_compiler.stage; - let host = target_compiler.host; - let (host_info, dir_name) = if build_compiler.host == host { - ("".into(), "host".into()) - } else { - (format!(" ({host})"), host.to_string()) - }; - // NOTE: "Creating a sysroot" is somewhat inconsistent with our internal terminology, since - // sysroots can temporarily be empty until we put the compiler inside. However, - // `ensure(Sysroot)` isn't really something that's user facing, so there shouldn't be any - // ambiguity. - let msg = format!( - "Creating a sysroot for stage{stage} compiler{host_info} (use `rustup toolchain link 'name' build/{dir_name}/stage{stage}`)" - ); - builder.info(&msg); - - // Link in all dylibs to the libdir - let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); - let proc_macros = builder - .read_stamp_file(&stamp) - .into_iter() - .filter_map(|(path, dependency_type)| { - if dependency_type == DependencyType::Host { - Some(path.file_name().unwrap().to_owned().into_string().unwrap()) - } else { - None - } - }) - .collect::>(); - - let sysroot = builder.sysroot(target_compiler); - let rustc_libdir = builder.rustc_libdir(target_compiler); - t!(fs::create_dir_all(&rustc_libdir)); - let src_libdir = builder.sysroot_libdir(build_compiler, host); - for f in builder.read_dir(&src_libdir) { - let filename = f.file_name().into_string().unwrap(); - if (is_dylib(&filename) || is_debug_info(&filename)) && !proc_macros.contains(&filename) - { - builder.copy(&f.path(), &rustc_libdir.join(&filename)); - } - } - - copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); - - // We prepend this bin directory to the user PATH when linking Rust binaries. To - // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. - let libdir = builder.sysroot_libdir(target_compiler, target_compiler.host); - let libdir_bin = libdir.parent().unwrap().join("bin"); - t!(fs::create_dir_all(&libdir_bin)); - if let Some(lld_install) = lld_install { - let src_exe = exe("lld", target_compiler.host); - let dst_exe = exe("rust-lld", target_compiler.host); - builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); - // for `-Z gcc-ld=lld` - let gcc_ld_dir = libdir_bin.join("gcc-ld"); - t!(fs::create_dir(&gcc_ld_dir)); - let lld_wrapper_exe = builder.ensure(crate::tool::LldWrapper { - compiler: build_compiler, - target: target_compiler.host, - }); - for name in crate::LLD_FILE_NAMES { - builder.copy(&lld_wrapper_exe, &gcc_ld_dir.join(exe(name, target_compiler.host))); - } - } - - if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { - let llvm::LlvmResult { llvm_config, .. } = - builder.ensure(llvm::Llvm { target: target_compiler.host }); - if !builder.config.dry_run() { - let llvm_bin_dir = output(Command::new(llvm_config).arg("--bindir")); - let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); - - // Since we've already built the LLVM tools, install them to the sysroot. - // This is the equivalent of installing the `llvm-tools-preview` component via - // rustup, and lets developers use a locally built toolchain to - // build projects that expect llvm tools to be present in the sysroot - // (e.g. the `bootimage` crate). - for tool in LLVM_TOOLS { - let tool_exe = exe(tool, target_compiler.host); - let src_path = llvm_bin_dir.join(&tool_exe); - // When using `download-ci-llvm`, some of the tools - // may not exist, so skip trying to copy them. - if src_path.exists() { - builder.copy(&src_path, &libdir_bin.join(&tool_exe)); - } - } - } - } - - // Ensure that `libLLVM.so` ends up in the newly build compiler directory, - // so that it can be found when the newly built `rustc` is run. - dist::maybe_install_llvm_runtime(builder, target_compiler.host, &sysroot); - dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); - - // Link the compiler binary itself into place - let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host); - let rustc = out_dir.join(exe("rustc-main", host)); - let bindir = sysroot.join("bin"); - t!(fs::create_dir_all(&bindir)); - let compiler = builder.rustc(target_compiler); - builder.copy(&rustc, &compiler); - - target_compiler - } -} - -/// Link some files into a rustc sysroot. -/// -/// For a particular stage this will link the file listed in `stamp` into the -/// `sysroot_dst` provided. -pub fn add_to_sysroot( - builder: &Builder<'_>, - sysroot_dst: &Path, - sysroot_host_dst: &Path, - stamp: &Path, -) { - let self_contained_dst = &sysroot_dst.join("self-contained"); - t!(fs::create_dir_all(&sysroot_dst)); - t!(fs::create_dir_all(&sysroot_host_dst)); - t!(fs::create_dir_all(&self_contained_dst)); - for (path, dependency_type) in builder.read_stamp_file(stamp) { - let dst = match dependency_type { - DependencyType::Host => sysroot_host_dst, - DependencyType::Target => sysroot_dst, - DependencyType::TargetSelfContained => self_contained_dst, - }; - builder.copy(&path, &dst.join(path.file_name().unwrap())); - } -} - -pub fn run_cargo( - builder: &Builder<'_>, - cargo: Cargo, - tail_args: Vec, - stamp: &Path, - additional_target_deps: Vec<(PathBuf, DependencyType)>, - is_check: bool, - rlib_only_metadata: bool, -) -> Vec { - if builder.config.dry_run() { - return Vec::new(); - } - - // `target_root_dir` looks like $dir/$target/release - let target_root_dir = stamp.parent().unwrap(); - // `target_deps_dir` looks like $dir/$target/release/deps - let target_deps_dir = target_root_dir.join("deps"); - // `host_root_dir` looks like $dir/release - let host_root_dir = target_root_dir - .parent() - .unwrap() // chop off `release` - .parent() - .unwrap() // chop off `$target` - .join(target_root_dir.file_name().unwrap()); - - // Spawn Cargo slurping up its JSON output. We'll start building up the - // `deps` array of all files it generated along with a `toplevel` array of - // files we need to probe for later. - let mut deps = Vec::new(); - let mut toplevel = Vec::new(); - let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| { - let (filenames, crate_types) = match msg { - CargoMessage::CompilerArtifact { - filenames, - target: CargoTarget { crate_types }, - .. - } => (filenames, crate_types), - _ => return, - }; - for filename in filenames { - // Skip files like executables - let mut keep = false; - if filename.ends_with(".lib") - || filename.ends_with(".a") - || is_debug_info(&filename) - || is_dylib(&filename) - { - // Always keep native libraries, rust dylibs and debuginfo - keep = true; - } - if is_check && filename.ends_with(".rmeta") { - // During check builds we need to keep crate metadata - keep = true; - } else if rlib_only_metadata { - if filename.contains("jemalloc_sys") - || filename.contains("rustc_smir") - || filename.contains("stable_mir") - { - // jemalloc_sys and rustc_smir are not linked into librustc_driver.so, - // so we need to distribute them as rlib to be able to use them. - keep |= filename.ends_with(".rlib"); - } else { - // Distribute the rest of the rustc crates as rmeta files only to reduce - // the tarball sizes by about 50%. The object files are linked into - // librustc_driver.so, so it is still possible to link against them. - keep |= filename.ends_with(".rmeta"); - } - } else { - // In all other cases keep all rlibs - keep |= filename.ends_with(".rlib"); - } - - if !keep { - continue; - } - - let filename = Path::new(&*filename); - - // If this was an output file in the "host dir" we don't actually - // worry about it, it's not relevant for us - if filename.starts_with(&host_root_dir) { - // Unless it's a proc macro used in the compiler - if crate_types.iter().any(|t| t == "proc-macro") { - deps.push((filename.to_path_buf(), DependencyType::Host)); - } - continue; - } - - // If this was output in the `deps` dir then this is a precise file - // name (hash included) so we start tracking it. - if filename.starts_with(&target_deps_dir) { - deps.push((filename.to_path_buf(), DependencyType::Target)); - continue; - } - - // Otherwise this was a "top level artifact" which right now doesn't - // have a hash in the name, but there's a version of this file in - // the `deps` folder which *does* have a hash in the name. That's - // the one we'll want to we'll probe for it later. - // - // We do not use `Path::file_stem` or `Path::extension` here, - // because some generated files may have multiple extensions e.g. - // `std-.dll.lib` on Windows. The aforementioned methods only - // split the file name by the last extension (`.lib`) while we need - // to split by all extensions (`.dll.lib`). - let expected_len = t!(filename.metadata()).len(); - let filename = filename.file_name().unwrap().to_str().unwrap(); - let mut parts = filename.splitn(2, '.'); - let file_stem = parts.next().unwrap().to_owned(); - let extension = parts.next().unwrap().to_owned(); - - toplevel.push((file_stem, extension, expected_len)); - } - }); - - if !ok { - crate::exit!(1); - } - - // Ok now we need to actually find all the files listed in `toplevel`. We've - // got a list of prefix/extensions and we basically just need to find the - // most recent file in the `deps` folder corresponding to each one. - let contents = t!(target_deps_dir.read_dir()) - .map(|e| t!(e)) - .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata()))) - .collect::>(); - for (prefix, extension, expected_len) in toplevel { - let candidates = contents.iter().filter(|&&(_, ref filename, ref meta)| { - meta.len() == expected_len - && filename - .strip_prefix(&prefix[..]) - .map(|s| s.starts_with('-') && s.ends_with(&extension[..])) - .unwrap_or(false) - }); - let max = candidates.max_by_key(|&&(_, _, ref metadata)| { - metadata.modified().expect("mtime should be available on all relevant OSes") - }); - let path_to_add = match max { - Some(triple) => triple.0.to_str().unwrap(), - None => panic!("no output generated for {prefix:?} {extension:?}"), - }; - if is_dylib(path_to_add) { - let candidate = format!("{path_to_add}.lib"); - let candidate = PathBuf::from(candidate); - if candidate.exists() { - deps.push((candidate, DependencyType::Target)); - } - } - deps.push((path_to_add.into(), DependencyType::Target)); - } - - deps.extend(additional_target_deps); - deps.sort(); - let mut new_contents = Vec::new(); - for (dep, dependency_type) in deps.iter() { - new_contents.extend(match *dependency_type { - DependencyType::Host => b"h", - DependencyType::Target => b"t", - DependencyType::TargetSelfContained => b"s", - }); - new_contents.extend(dep.to_str().unwrap().as_bytes()); - new_contents.extend(b"\0"); - } - t!(fs::write(&stamp, &new_contents)); - deps.into_iter().map(|(d, _)| d).collect() -} - -pub fn stream_cargo( - builder: &Builder<'_>, - cargo: Cargo, - tail_args: Vec, - cb: &mut dyn FnMut(CargoMessage<'_>), -) -> bool { - let mut cargo = Command::from(cargo); - if builder.config.dry_run() { - return true; - } - // Instruct Cargo to give us json messages on stdout, critically leaving - // stderr as piped so we can get those pretty colors. - let mut message_format = if builder.config.json_output { - String::from("json") - } else { - String::from("json-render-diagnostics") - }; - if let Some(s) = &builder.config.rustc_error_format { - message_format.push_str(",json-diagnostic-"); - message_format.push_str(s); - } - cargo.arg("--message-format").arg(message_format).stdout(Stdio::piped()); - - for arg in tail_args { - cargo.arg(arg); - } - - builder.verbose(&format!("running: {cargo:?}")); - let mut child = match cargo.spawn() { - Ok(child) => child, - Err(e) => panic!("failed to execute command: {cargo:?}\nerror: {e}"), - }; - - // Spawn Cargo slurping up its JSON output. We'll start building up the - // `deps` array of all files it generated along with a `toplevel` array of - // files we need to probe for later. - let stdout = BufReader::new(child.stdout.take().unwrap()); - for line in stdout.lines() { - let line = t!(line); - match serde_json::from_str::>(&line) { - Ok(msg) => { - if builder.config.json_output { - // Forward JSON to stdout. - println!("{line}"); - } - cb(msg) - } - // If this was informational, just print it out and continue - Err(_) => println!("{line}"), - } - } - - // Make sure Cargo actually succeeded after we read all of its stdout. - let status = t!(child.wait()); - if builder.is_verbose() && !status.success() { - eprintln!( - "command did not execute successfully: {cargo:?}\n\ - expected success, got: {status}" - ); - } - status.success() -} - -#[derive(Deserialize)] -pub struct CargoTarget<'a> { - crate_types: Vec>, -} - -#[derive(Deserialize)] -#[serde(tag = "reason", rename_all = "kebab-case")] -pub enum CargoMessage<'a> { - CompilerArtifact { - package_id: Cow<'a, str>, - features: Vec>, - filenames: Vec>, - target: CargoTarget<'a>, - }, - BuildScriptExecuted { - package_id: Cow<'a, str>, - }, - BuildFinished { - success: bool, - }, -} - -pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) { - // FIXME: to make things simpler for now, limit this to the host and target where we know - // `strip -g` is both available and will fix the issue, i.e. on a x64 linux host that is not - // cross-compiling. Expand this to other appropriate targets in the future. - if target != "x86_64-unknown-linux-gnu" || target != builder.config.build || !path.exists() { - return; - } - - let previous_mtime = FileTime::from_last_modification_time(&path.metadata().unwrap()); - // Note: `output` will propagate any errors here. - output(Command::new("strip").arg("--strip-debug").arg(path)); - - // After running `strip`, we have to set the file modification time to what it was before, - // otherwise we risk Cargo invalidating its fingerprint and rebuilding the world next time - // bootstrap is invoked. - // - // An example of this is if we run this on librustc_driver.so. In the first invocation: - // - Cargo will build librustc_driver.so (mtime of 1) - // - Cargo will build rustc-main (mtime of 2) - // - Bootstrap will strip librustc_driver.so (changing the mtime to 3). - // - // In the second invocation of bootstrap, Cargo will see that the mtime of librustc_driver.so - // is greater than the mtime of rustc-main, and will rebuild rustc-main. That will then cause - // everything else (standard library, future stages...) to be rebuilt. - t!(filetime::set_file_mtime(path, previous_mtime)); -} diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs deleted file mode 100644 index 836328f94..000000000 --- a/src/bootstrap/config.rs +++ /dev/null @@ -1,2104 +0,0 @@ -//! Serialized configuration of a build. -//! -//! This module implements parsing `config.toml` configuration files to tweak -//! how the build runs. - -#[cfg(test)] -mod tests; - -use std::cell::{Cell, RefCell}; -use std::cmp; -use std::collections::{HashMap, HashSet}; -use std::env; -use std::fmt::{self, Display}; -use std::fs; -use std::io::IsTerminal; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::str::FromStr; - -use crate::cache::{Interned, INTERNER}; -use crate::cc_detect::{ndk_compiler, Language}; -use crate::channel::{self, GitInfo}; -use crate::compile::CODEGEN_BACKEND_PREFIX; -pub use crate::flags::Subcommand; -use crate::flags::{Color, Flags, Warnings}; -use crate::util::{exe, output, t}; -use build_helper::exit; -use once_cell::sync::OnceCell; -use semver::Version; -use serde::{Deserialize, Deserializer}; -use serde_derive::Deserialize; - -macro_rules! check_ci_llvm { - ($name:expr) => { - assert!( - $name.is_none(), - "setting {} is incompatible with download-ci-llvm.", - stringify!($name) - ); - }; -} - -#[derive(Clone, Default)] -pub enum DryRun { - /// This isn't a dry run. - #[default] - Disabled, - /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. - SelfCheck, - /// This is a dry run enabled by the `--dry-run` flag. - UserSelected, -} - -#[derive(Copy, Clone, Default, PartialEq, Eq)] -pub enum DebuginfoLevel { - #[default] - None, - LineTablesOnly, - Limited, - Full, -} - -// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only -// deserializes i64, and derive() only generates visit_u64 -impl<'de> Deserialize<'de> for DebuginfoLevel { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - use serde::de::Error; - - Ok(match Deserialize::deserialize(deserializer)? { - StringOrInt::String("none") | StringOrInt::Int(0) => DebuginfoLevel::None, - StringOrInt::String("line-tables-only") => DebuginfoLevel::LineTablesOnly, - StringOrInt::String("limited") | StringOrInt::Int(1) => DebuginfoLevel::Limited, - StringOrInt::String("full") | StringOrInt::Int(2) => DebuginfoLevel::Full, - StringOrInt::Int(n) => { - let other = serde::de::Unexpected::Signed(n); - return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); - } - StringOrInt::String(s) => { - let other = serde::de::Unexpected::Str(s); - return Err(D::Error::invalid_value( - other, - &"expected none, line-tables-only, limited, or full", - )); - } - }) - } -} - -/// Suitable for passing to `-C debuginfo` -impl Display for DebuginfoLevel { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - use DebuginfoLevel::*; - f.write_str(match self { - None => "0", - LineTablesOnly => "line-tables-only", - Limited => "1", - Full => "2", - }) - } -} - -/// Global configuration for the entire build and/or bootstrap. -/// -/// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. -/// -/// Note that this structure is not decoded directly into, but rather it is -/// filled out from the decoded forms of the structs below. For documentation -/// each field, see the corresponding fields in -/// `config.example.toml`. -#[derive(Default, Clone)] -pub struct Config { - pub changelog_seen: Option, - pub ccache: Option, - /// Call Build::ninja() instead of this. - pub ninja_in_file: bool, - pub verbose: usize, - pub submodules: Option, - pub compiler_docs: bool, - pub library_docs_private_items: bool, - pub docs_minification: bool, - pub docs: bool, - pub locked_deps: bool, - pub vendor: bool, - pub target_config: HashMap, - pub full_bootstrap: bool, - pub extended: bool, - pub tools: Option>, - pub sanitizers: bool, - pub profiler: bool, - pub omit_git_hash: bool, - pub skip: Vec, - pub include_default_paths: bool, - pub rustc_error_format: Option, - pub json_output: bool, - pub test_compare_mode: bool, - pub color: Color, - pub patch_binaries_for_nix: Option, - pub stage0_metadata: Stage0Metadata, - - pub stdout_is_tty: bool, - pub stderr_is_tty: bool, - - pub on_fail: Option, - pub stage: u32, - pub keep_stage: Vec, - pub keep_stage_std: Vec, - pub src: PathBuf, - /// defaults to `config.toml` - pub config: Option, - pub jobs: Option, - pub cmd: Subcommand, - pub incremental: bool, - pub dry_run: DryRun, - /// Arguments appearing after `--` to be forwarded to tools, - /// e.g. `--fix-broken` or test arguments. - pub free_args: Vec, - - /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. - #[cfg(not(test))] - download_rustc_commit: Option, - #[cfg(test)] - pub download_rustc_commit: Option, - - pub deny_warnings: bool, - pub backtrace_on_ice: bool, - - // llvm codegen options - pub llvm_assertions: bool, - pub llvm_tests: bool, - pub llvm_plugins: bool, - pub llvm_optimize: bool, - pub llvm_thin_lto: bool, - pub llvm_release_debuginfo: bool, - pub llvm_static_stdcpp: bool, - /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. - #[cfg(not(test))] - llvm_link_shared: Cell>, - #[cfg(test)] - pub llvm_link_shared: Cell>, - pub llvm_clang_cl: Option, - pub llvm_targets: Option, - pub llvm_experimental_targets: Option, - pub llvm_link_jobs: Option, - pub llvm_version_suffix: Option, - pub llvm_use_linker: Option, - pub llvm_allow_old_toolchain: bool, - pub llvm_polly: bool, - pub llvm_clang: bool, - pub llvm_enable_warnings: bool, - pub llvm_from_ci: bool, - pub llvm_build_config: HashMap, - - pub use_lld: bool, - pub lld_enabled: bool, - pub llvm_tools_enabled: bool, - - pub llvm_cflags: Option, - pub llvm_cxxflags: Option, - pub llvm_ldflags: Option, - pub llvm_use_libcxx: bool, - - // rust codegen options - pub rust_optimize: RustOptimize, - pub rust_codegen_units: Option, - pub rust_codegen_units_std: Option, - pub rust_debug_assertions: bool, - pub rust_debug_assertions_std: bool, - pub rust_overflow_checks: bool, - pub rust_overflow_checks_std: bool, - pub rust_debug_logging: bool, - pub rust_debuginfo_level_rustc: DebuginfoLevel, - pub rust_debuginfo_level_std: DebuginfoLevel, - pub rust_debuginfo_level_tools: DebuginfoLevel, - pub rust_debuginfo_level_tests: DebuginfoLevel, - pub rust_split_debuginfo: SplitDebuginfo, - pub rust_rpath: bool, - pub rustc_parallel: bool, - pub rustc_default_linker: Option, - pub rust_optimize_tests: bool, - pub rust_dist_src: bool, - pub rust_codegen_backends: Vec>, - pub rust_verify_llvm_ir: bool, - pub rust_thin_lto_import_instr_limit: Option, - pub rust_remap_debuginfo: bool, - pub rust_new_symbol_mangling: Option, - pub rust_profile_use: Option, - pub rust_profile_generate: Option, - pub rust_lto: RustcLto, - pub rust_validate_mir_opts: Option, - pub llvm_profile_use: Option, - pub llvm_profile_generate: bool, - pub llvm_libunwind_default: Option, - - pub reproducible_artifacts: Vec, - - pub build: TargetSelection, - pub hosts: Vec, - pub targets: Vec, - pub local_rebuild: bool, - pub jemalloc: bool, - pub control_flow_guard: bool, - - // dist misc - pub dist_sign_folder: Option, - pub dist_upload_addr: Option, - pub dist_compression_formats: Option>, - pub dist_compression_profile: String, - pub dist_include_mingw_linker: bool, - - // libstd features - pub backtrace: bool, // support for RUST_BACKTRACE - - // misc - pub low_priority: bool, - pub channel: String, - pub description: Option, - pub verbose_tests: bool, - pub save_toolstates: Option, - pub print_step_timings: bool, - pub print_step_rusage: bool, - pub missing_tools: bool, - - // Fallback musl-root for all targets - pub musl_root: Option, - pub prefix: Option, - pub sysconfdir: Option, - pub datadir: Option, - pub docdir: Option, - pub bindir: PathBuf, - pub libdir: Option, - pub mandir: Option, - pub codegen_tests: bool, - pub nodejs: Option, - pub npm: Option, - pub gdb: Option, - pub python: Option, - pub reuse: Option, - pub cargo_native_static: bool, - pub configure_args: Vec, - pub out: PathBuf, - pub rust_info: channel::GitInfo, - - // These are either the stage0 downloaded binaries or the locally installed ones. - pub initial_cargo: PathBuf, - pub initial_rustc: PathBuf, - - #[cfg(not(test))] - initial_rustfmt: RefCell, - #[cfg(test)] - pub initial_rustfmt: RefCell, - - pub paths: Vec, -} - -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Metadata { - pub compiler: CompilerMetadata, - pub config: Stage0Config, - pub checksums_sha256: HashMap, - pub rustfmt: Option, -} -#[derive(Default, Deserialize, Clone)] -pub struct CompilerMetadata { - pub date: String, - pub version: String, -} - -#[derive(Default, Deserialize, Clone)] -pub struct Stage0Config { - pub dist_server: String, - pub artifacts_server: String, - pub artifacts_with_llvm_assertions_server: String, - pub git_merge_commit_email: String, - pub nightly_branch: String, -} -#[derive(Default, Deserialize, Clone)] -pub struct RustfmtMetadata { - pub date: String, - pub version: String, -} - -#[derive(Clone, Debug, Default)] -pub enum RustfmtState { - SystemToolchain(PathBuf), - Downloaded(PathBuf), - Unavailable, - #[default] - LazyEvaluated, -} - -#[derive(Debug, Default, Clone, Copy, PartialEq)] -pub enum LlvmLibunwind { - #[default] - No, - InTree, - System, -} - -impl FromStr for LlvmLibunwind { - type Err = String; - - fn from_str(value: &str) -> Result { - match value { - "no" => Ok(Self::No), - "in-tree" => Ok(Self::InTree), - "system" => Ok(Self::System), - invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), - } - } -} - -#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub enum SplitDebuginfo { - Packed, - Unpacked, - #[default] - Off, -} - -impl std::str::FromStr for SplitDebuginfo { - type Err = (); - - fn from_str(s: &str) -> Result { - match s { - "packed" => Ok(SplitDebuginfo::Packed), - "unpacked" => Ok(SplitDebuginfo::Unpacked), - "off" => Ok(SplitDebuginfo::Off), - _ => Err(()), - } - } -} - -impl SplitDebuginfo { - /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for - /// `rust.split-debuginfo` in `config.example.toml`. - fn default_for_platform(target: &str) -> Self { - if target.contains("apple") { - SplitDebuginfo::Unpacked - } else if target.contains("windows") { - SplitDebuginfo::Packed - } else { - SplitDebuginfo::Off - } - } -} - -/// LTO mode used for compiling rustc itself. -#[derive(Default, Clone, PartialEq, Debug)] -pub enum RustcLto { - Off, - #[default] - ThinLocal, - Thin, - Fat, -} - -impl std::str::FromStr for RustcLto { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "thin-local" => Ok(RustcLto::ThinLocal), - "thin" => Ok(RustcLto::Thin), - "fat" => Ok(RustcLto::Fat), - "off" => Ok(RustcLto::Off), - _ => Err(format!("Invalid value for rustc LTO: {s}")), - } - } -} - -#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct TargetSelection { - pub triple: Interned, - file: Option>, - synthetic: bool, -} - -/// Newtype over `Vec` so we can implement custom parsing logic -#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] -pub struct TargetSelectionList(Vec); - -pub fn target_selection_list(s: &str) -> Result { - Ok(TargetSelectionList( - s.split(",").filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), - )) -} - -impl TargetSelection { - pub fn from_user(selection: &str) -> Self { - let path = Path::new(selection); - - let (triple, file) = if path.exists() { - let triple = path - .file_stem() - .expect("Target specification file has no file stem") - .to_str() - .expect("Target specification file stem is not UTF-8"); - - (triple, Some(selection)) - } else { - (selection, None) - }; - - let triple = INTERNER.intern_str(triple); - let file = file.map(|f| INTERNER.intern_str(f)); - - Self { triple, file, synthetic: false } - } - - pub fn create_synthetic(triple: &str, file: &str) -> Self { - Self { - triple: INTERNER.intern_str(triple), - file: Some(INTERNER.intern_str(file)), - synthetic: true, - } - } - - pub fn rustc_target_arg(&self) -> &str { - self.file.as_ref().unwrap_or(&self.triple) - } - - pub fn contains(&self, needle: &str) -> bool { - self.triple.contains(needle) - } - - pub fn starts_with(&self, needle: &str) -> bool { - self.triple.starts_with(needle) - } - - pub fn ends_with(&self, needle: &str) -> bool { - self.triple.ends_with(needle) - } - - // See src/bootstrap/synthetic_targets.rs - pub fn is_synthetic(&self) -> bool { - self.synthetic - } -} - -impl fmt::Display for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}", self.triple)?; - if let Some(file) = self.file { - write!(f, "({file})")?; - } - Ok(()) - } -} - -impl fmt::Debug for TargetSelection { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{self}") - } -} - -impl PartialEq<&str> for TargetSelection { - fn eq(&self, other: &&str) -> bool { - self.triple == *other - } -} - -/// Per-target configuration stored in the global configuration structure. -#[derive(Default, Clone)] -pub struct Target { - /// Some(path to llvm-config) if using an external LLVM. - pub llvm_config: Option, - pub llvm_has_rust_patches: Option, - /// Some(path to FileCheck) if one was specified. - pub llvm_filecheck: Option, - pub llvm_libunwind: Option, - pub cc: Option, - pub cxx: Option, - pub ar: Option, - pub ranlib: Option, - pub default_linker: Option, - pub linker: Option, - pub ndk: Option, - pub sanitizers: Option, - pub profiler: Option, - pub rpath: Option, - pub crt_static: Option, - pub musl_root: Option, - pub musl_libdir: Option, - pub wasi_root: Option, - pub qemu_rootfs: Option, - pub no_std: bool, -} - -impl Target { - pub fn from_triple(triple: &str) -> Self { - let mut target: Self = Default::default(); - if triple.contains("-none") || triple.contains("nvptx") || triple.contains("switch") { - target.no_std = true; - } - target - } -} -/// Structure of the `config.toml` file that configuration is read from. -/// -/// This structure uses `Decodable` to automatically decode a TOML configuration -/// file into this format, and then this is traversed and written into the above -/// `Config` structure. -#[derive(Deserialize, Default)] -#[serde(deny_unknown_fields, rename_all = "kebab-case")] -struct TomlConfig { - changelog_seen: Option, - build: Option, - install: Option, - llvm: Option, - rust: Option, - target: Option>, - dist: Option, - profile: Option, -} - -/// Describes how to handle conflicts in merging two [`TomlConfig`] -#[derive(Copy, Clone, Debug)] -enum ReplaceOpt { - /// Silently ignore a duplicated value - IgnoreDuplicate, - /// Override the current value, even if it's `Some` - Override, - /// Exit with an error on duplicate values - ErrorOnDuplicate, -} - -trait Merge { - fn merge(&mut self, other: Self, replace: ReplaceOpt); -} - -impl Merge for TomlConfig { - fn merge( - &mut self, - TomlConfig { build, install, llvm, rust, dist, target, profile: _, changelog_seen }: Self, - replace: ReplaceOpt, - ) { - fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { - if let Some(new) = y { - if let Some(original) = x { - original.merge(new, replace); - } else { - *x = Some(new); - } - } - } - self.changelog_seen.merge(changelog_seen, replace); - do_merge(&mut self.build, build, replace); - do_merge(&mut self.install, install, replace); - do_merge(&mut self.llvm, llvm, replace); - do_merge(&mut self.rust, rust, replace); - do_merge(&mut self.dist, dist, replace); - assert!(target.is_none(), "merging target-specific config is not currently supported"); - } -} - -// We are using a decl macro instead of a derive proc macro here to reduce the compile time of -// rustbuild. -macro_rules! define_config { - ($(#[$attr:meta])* struct $name:ident { - $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* - }) => { - $(#[$attr])* - struct $name { - $($field: Option<$field_ty>,)* - } - - impl Merge for $name { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { - $( - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.$field.is_none() { - self.$field = other.$field; - } - }, - ReplaceOpt::Override => { - if other.$field.is_some() { - self.$field = other.$field; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.$field.is_some() { - if self.$field.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option: `{}`", stringify!($field)); - exit!(2); - } - } else { - self.$field = other.$field; - } - } - } - } - )* - } - } - - // The following is a trimmed version of what serde_derive generates. All parts not relevant - // for toml deserialization have been removed. This reduces the binary size and improves - // compile time of rustbuild. - impl<'de> Deserialize<'de> for $name { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - struct Field; - impl<'de> serde::de::Visitor<'de> for Field { - type Value = $name; - fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(concat!("struct ", stringify!($name))) - } - - #[inline] - fn visit_map(self, mut map: A) -> Result - where - A: serde::de::MapAccess<'de>, - { - $(let mut $field: Option<$field_ty> = None;)* - while let Some(key) = - match serde::de::MapAccess::next_key::(&mut map) { - Ok(val) => val, - Err(err) => { - return Err(err); - } - } - { - match &*key { - $($field_key => { - if $field.is_some() { - return Err(::duplicate_field( - $field_key, - )); - } - $field = match serde::de::MapAccess::next_value::<$field_ty>( - &mut map, - ) { - Ok(val) => Some(val), - Err(err) => { - return Err(err); - } - }; - })* - key => { - return Err(serde::de::Error::unknown_field(key, FIELDS)); - } - } - } - Ok($name { $($field),* }) - } - } - const FIELDS: &'static [&'static str] = &[ - $($field_key,)* - ]; - Deserializer::deserialize_struct( - deserializer, - stringify!($name), - FIELDS, - Field, - ) - } - } - } -} - -impl Merge for Option { - fn merge(&mut self, other: Self, replace: ReplaceOpt) { - match replace { - ReplaceOpt::IgnoreDuplicate => { - if self.is_none() { - *self = other; - } - } - ReplaceOpt::Override => { - if other.is_some() { - *self = other; - } - } - ReplaceOpt::ErrorOnDuplicate => { - if other.is_some() { - if self.is_some() { - if cfg!(test) { - panic!("overriding existing option") - } else { - eprintln!("overriding existing option"); - exit!(2); - } - } else { - *self = other; - } - } - } - } - } -} - -define_config! { - /// TOML representation of various global build decisions. - #[derive(Default)] - struct Build { - build: Option = "build", - host: Option> = "host", - target: Option> = "target", - build_dir: Option = "build-dir", - cargo: Option = "cargo", - rustc: Option = "rustc", - rustfmt: Option = "rustfmt", - docs: Option = "docs", - compiler_docs: Option = "compiler-docs", - library_docs_private_items: Option = "library-docs-private-items", - docs_minification: Option = "docs-minification", - submodules: Option = "submodules", - gdb: Option = "gdb", - nodejs: Option = "nodejs", - npm: Option = "npm", - python: Option = "python", - reuse: Option = "reuse", - locked_deps: Option = "locked-deps", - vendor: Option = "vendor", - full_bootstrap: Option = "full-bootstrap", - extended: Option = "extended", - tools: Option> = "tools", - verbose: Option = "verbose", - sanitizers: Option = "sanitizers", - profiler: Option = "profiler", - cargo_native_static: Option = "cargo-native-static", - low_priority: Option = "low-priority", - configure_args: Option> = "configure-args", - local_rebuild: Option = "local-rebuild", - print_step_timings: Option = "print-step-timings", - print_step_rusage: Option = "print-step-rusage", - check_stage: Option = "check-stage", - doc_stage: Option = "doc-stage", - build_stage: Option = "build-stage", - test_stage: Option = "test-stage", - install_stage: Option = "install-stage", - dist_stage: Option = "dist-stage", - bench_stage: Option = "bench-stage", - patch_binaries_for_nix: Option = "patch-binaries-for-nix", - // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally - metrics: Option = "metrics", - } -} - -define_config! { - /// TOML representation of various global install decisions. - struct Install { - prefix: Option = "prefix", - sysconfdir: Option = "sysconfdir", - docdir: Option = "docdir", - bindir: Option = "bindir", - libdir: Option = "libdir", - mandir: Option = "mandir", - datadir: Option = "datadir", - } -} - -define_config! { - /// TOML representation of how the LLVM build is configured. - struct Llvm { - optimize: Option = "optimize", - thin_lto: Option = "thin-lto", - release_debuginfo: Option = "release-debuginfo", - assertions: Option = "assertions", - tests: Option = "tests", - plugins: Option = "plugins", - ccache: Option = "ccache", - static_libstdcpp: Option = "static-libstdcpp", - ninja: Option = "ninja", - targets: Option = "targets", - experimental_targets: Option = "experimental-targets", - link_jobs: Option = "link-jobs", - link_shared: Option = "link-shared", - version_suffix: Option = "version-suffix", - clang_cl: Option = "clang-cl", - cflags: Option = "cflags", - cxxflags: Option = "cxxflags", - ldflags: Option = "ldflags", - use_libcxx: Option = "use-libcxx", - use_linker: Option = "use-linker", - allow_old_toolchain: Option = "allow-old-toolchain", - polly: Option = "polly", - clang: Option = "clang", - enable_warnings: Option = "enable-warnings", - download_ci_llvm: Option = "download-ci-llvm", - build_config: Option> = "build-config", - } -} - -define_config! { - struct Dist { - sign_folder: Option = "sign-folder", - gpg_password_file: Option = "gpg-password-file", - upload_addr: Option = "upload-addr", - src_tarball: Option = "src-tarball", - missing_tools: Option = "missing-tools", - compression_formats: Option> = "compression-formats", - compression_profile: Option = "compression-profile", - include_mingw_linker: Option = "include-mingw-linker", - } -} - -#[derive(Clone, Debug, Deserialize)] -#[serde(untagged)] -pub enum StringOrBool { - String(String), - Bool(bool), -} - -impl Default for StringOrBool { - fn default() -> StringOrBool { - StringOrBool::Bool(false) - } -} - -impl StringOrBool { - fn is_string_or_true(&self) -> bool { - matches!(self, Self::String(_) | Self::Bool(true)) - } -} - -#[derive(Clone, Debug, PartialEq, Eq)] -pub enum RustOptimize { - String(String), - Int(u8), - Bool(bool), -} - -impl Default for RustOptimize { - fn default() -> RustOptimize { - RustOptimize::Bool(false) - } -} - -impl<'de> Deserialize<'de> for RustOptimize { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_any(OptimizeVisitor) - } -} - -struct OptimizeVisitor; - -impl<'de> serde::de::Visitor<'de> for OptimizeVisitor { - type Value = RustOptimize; - - fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) - } - - fn visit_str(self, value: &str) -> Result - where - E: serde::de::Error, - { - if ["s", "z"].iter().find(|x| **x == value).is_some() { - Ok(RustOptimize::String(value.to_string())) - } else { - Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) - } - } - - fn visit_i64(self, value: i64) -> Result - where - E: serde::de::Error, - { - if matches!(value, 0..=3) { - Ok(RustOptimize::Int(value as u8)) - } else { - Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) - } - } - - fn visit_bool(self, value: bool) -> Result - where - E: serde::de::Error, - { - Ok(RustOptimize::Bool(value)) - } -} - -fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { - format!( - r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# - ) -} - -impl RustOptimize { - pub(crate) fn is_release(&self) -> bool { - match &self { - RustOptimize::Bool(true) | RustOptimize::String(_) => true, - RustOptimize::Int(i) => *i > 0, - RustOptimize::Bool(false) => false, - } - } - - pub(crate) fn get_opt_level(&self) -> Option { - match &self { - RustOptimize::String(s) => Some(s.clone()), - RustOptimize::Int(i) => Some(i.to_string()), - RustOptimize::Bool(_) => None, - } - } -} - -#[derive(Deserialize)] -#[serde(untagged)] -enum StringOrInt<'a> { - String(&'a str), - Int(i64), -} -define_config! { - /// TOML representation of how the Rust build is configured. - struct Rust { - optimize: Option = "optimize", - debug: Option = "debug", - codegen_units: Option = "codegen-units", - codegen_units_std: Option = "codegen-units-std", - debug_assertions: Option = "debug-assertions", - debug_assertions_std: Option = "debug-assertions-std", - overflow_checks: Option = "overflow-checks", - overflow_checks_std: Option = "overflow-checks-std", - debug_logging: Option = "debug-logging", - debuginfo_level: Option = "debuginfo-level", - debuginfo_level_rustc: Option = "debuginfo-level-rustc", - debuginfo_level_std: Option = "debuginfo-level-std", - debuginfo_level_tools: Option = "debuginfo-level-tools", - debuginfo_level_tests: Option = "debuginfo-level-tests", - split_debuginfo: Option = "split-debuginfo", - run_dsymutil: Option = "run-dsymutil", - backtrace: Option = "backtrace", - incremental: Option = "incremental", - parallel_compiler: Option = "parallel-compiler", - default_linker: Option = "default-linker", - channel: Option = "channel", - description: Option = "description", - musl_root: Option = "musl-root", - rpath: Option = "rpath", - verbose_tests: Option = "verbose-tests", - optimize_tests: Option = "optimize-tests", - codegen_tests: Option = "codegen-tests", - omit_git_hash: Option = "omit-git-hash", - dist_src: Option = "dist-src", - save_toolstates: Option = "save-toolstates", - codegen_backends: Option> = "codegen-backends", - lld: Option = "lld", - use_lld: Option = "use-lld", - llvm_tools: Option = "llvm-tools", - deny_warnings: Option = "deny-warnings", - backtrace_on_ice: Option = "backtrace-on-ice", - verify_llvm_ir: Option = "verify-llvm-ir", - thin_lto_import_instr_limit: Option = "thin-lto-import-instr-limit", - remap_debuginfo: Option = "remap-debuginfo", - jemalloc: Option = "jemalloc", - test_compare_mode: Option = "test-compare-mode", - llvm_libunwind: Option = "llvm-libunwind", - control_flow_guard: Option = "control-flow-guard", - new_symbol_mangling: Option = "new-symbol-mangling", - profile_generate: Option = "profile-generate", - profile_use: Option = "profile-use", - // ignored; this is set from an env var set by bootstrap.py - download_rustc: Option = "download-rustc", - lto: Option = "lto", - validate_mir_opts: Option = "validate-mir-opts", - } -} - -define_config! { - /// TOML representation of how each build target is configured. - struct TomlTarget { - cc: Option = "cc", - cxx: Option = "cxx", - ar: Option = "ar", - ranlib: Option = "ranlib", - default_linker: Option = "default-linker", - linker: Option = "linker", - llvm_config: Option = "llvm-config", - llvm_has_rust_patches: Option = "llvm-has-rust-patches", - llvm_filecheck: Option = "llvm-filecheck", - llvm_libunwind: Option = "llvm-libunwind", - android_ndk: Option = "android-ndk", - sanitizers: Option = "sanitizers", - profiler: Option = "profiler", - rpath: Option = "rpath", - crt_static: Option = "crt-static", - musl_root: Option = "musl-root", - musl_libdir: Option = "musl-libdir", - wasi_root: Option = "wasi-root", - qemu_rootfs: Option = "qemu-rootfs", - no_std: Option = "no-std", - } -} - -impl Config { - pub fn default_opts() -> Config { - let mut config = Config::default(); - config.llvm_optimize = true; - config.ninja_in_file = true; - config.llvm_static_stdcpp = false; - config.backtrace = true; - config.rust_optimize = RustOptimize::Bool(true); - config.rust_optimize_tests = true; - config.submodules = None; - config.docs = true; - config.docs_minification = true; - config.rust_rpath = true; - config.channel = "dev".to_string(); - config.codegen_tests = true; - config.rust_dist_src = true; - config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")]; - config.deny_warnings = true; - config.bindir = "bin".into(); - config.dist_include_mingw_linker = true; - config.dist_compression_profile = "fast".into(); - - config.stdout_is_tty = std::io::stdout().is_terminal(); - config.stderr_is_tty = std::io::stderr().is_terminal(); - - // set by build.rs - config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); - - let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); - // Undo `src/bootstrap` - config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); - config.out = PathBuf::from("build"); - - config - } - - pub fn parse(args: &[String]) -> Config { - #[cfg(test)] - fn get_toml(_: &Path) -> TomlConfig { - TomlConfig::default() - } - - #[cfg(not(test))] - fn get_toml(file: &Path) -> TomlConfig { - let contents = - t!(fs::read_to_string(file), format!("config file {} not found", file.display())); - // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of - // TomlConfig and sub types to be monomorphized 5x by toml. - toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - .unwrap_or_else(|err| { - eprintln!("failed to parse TOML configuration '{}': {err}", file.display()); - exit!(2); - }) - } - Self::parse_inner(args, get_toml) - } - - fn parse_inner(args: &[String], get_toml: impl Fn(&Path) -> TomlConfig) -> Config { - let mut flags = Flags::parse(&args); - let mut config = Config::default_opts(); - - // Set flags. - config.paths = std::mem::take(&mut flags.paths); - config.skip = flags.skip.into_iter().chain(flags.exclude).collect(); - config.include_default_paths = flags.include_default_paths; - config.rustc_error_format = flags.rustc_error_format; - config.json_output = flags.json_output; - config.on_fail = flags.on_fail; - config.jobs = Some(threads_from_config(flags.jobs as u32)); - config.cmd = flags.cmd; - config.incremental = flags.incremental; - config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; - config.keep_stage = flags.keep_stage; - config.keep_stage_std = flags.keep_stage_std; - config.color = flags.color; - config.free_args = std::mem::take(&mut flags.free_args); - config.llvm_profile_use = flags.llvm_profile_use; - config.llvm_profile_generate = flags.llvm_profile_generate; - - // Infer the rest of the configuration. - - // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, - // running on a completely machine from where it was compiled. - let mut cmd = Command::new("git"); - // NOTE: we cannot support running from outside the repository because the only path we have available - // is set at compile time, which can be wrong if bootstrap was downloaded from source. - // We still support running outside the repository if we find we aren't in a git directory. - cmd.arg("rev-parse").arg("--show-toplevel"); - // Discard stderr because we expect this to fail when building from a tarball. - let output = cmd - .stderr(std::process::Stdio::null()) - .output() - .ok() - .and_then(|output| if output.status.success() { Some(output) } else { None }); - if let Some(output) = output { - let git_root = String::from_utf8(output.stdout).unwrap(); - // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. - let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); - let s = git_root.to_str().unwrap(); - - // Bootstrap is quite bad at handling /? in front of paths - let src = match s.strip_prefix("\\\\?\\") { - Some(p) => PathBuf::from(p), - None => PathBuf::from(git_root), - }; - // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, - // for example, the build directory is inside of another unrelated git directory. - // In that case keep the original `CARGO_MANIFEST_DIR` handling. - // - // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside - // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. - if src.join("src").join("stage0.json").exists() { - config.src = src; - } - } else { - // We're building from a tarball, not git sources. - // We don't support pre-downloaded bootstrap in this case. - } - - if cfg!(test) { - // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. - config.out = Path::new( - &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), - ) - .parent() - .unwrap() - .to_path_buf(); - } - - let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); - - config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); - - // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. - let toml_path = flags - .config - .clone() - .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); - let using_default_path = toml_path.is_none(); - let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); - if using_default_path && !toml_path.exists() { - toml_path = config.src.join(toml_path); - } - - // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, - // but not if `config.toml` hasn't been created. - let mut toml = if !using_default_path || toml_path.exists() { - config.config = Some(toml_path.clone()); - get_toml(&toml_path) - } else { - config.config = None; - TomlConfig::default() - }; - - if let Some(include) = &toml.profile { - // Allows creating alias for profile names, allowing - // profiles to be renamed while maintaining back compatibility - // Keep in sync with `profile_aliases` in bootstrap.py - let profile_aliases = HashMap::from([("user", "dist")]); - let include = match profile_aliases.get(include.as_str()) { - Some(alias) => alias, - None => include.as_str(), - }; - let mut include_path = config.src.clone(); - include_path.push("src"); - include_path.push("bootstrap"); - include_path.push("defaults"); - include_path.push(format!("config.{include}.toml")); - let included_toml = get_toml(&include_path); - toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate); - } - - let mut override_toml = TomlConfig::default(); - for option in flags.set.iter() { - fn get_table(option: &str) -> Result { - toml::from_str(&option) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - } - - let mut err = match get_table(option) { - Ok(v) => { - override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); - continue; - } - Err(e) => e, - }; - // We want to be able to set string values without quotes, - // like in `configure.py`. Try adding quotes around the right hand side - if let Some((key, value)) = option.split_once("=") { - if !value.contains('"') { - match get_table(&format!(r#"{key}="{value}""#)) { - Ok(v) => { - override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); - continue; - } - Err(e) => err = e, - } - } - } - eprintln!("failed to parse override `{option}`: `{err}"); - exit!(2) - } - toml.merge(override_toml, ReplaceOpt::Override); - - config.changelog_seen = toml.changelog_seen; - - let build = toml.build.unwrap_or_default(); - if let Some(file_build) = build.build { - config.build = TargetSelection::from_user(&file_build); - }; - - set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from))); - // NOTE: Bootstrap spawns various commands with different working directories. - // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. - if !config.out.is_absolute() { - // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. - config.out = crate::util::absolute(&config.out); - } - - config.initial_rustc = if let Some(rustc) = build.rustc { - // FIXME(#115065): re-enable this check - // config.check_build_rustc_version(&rustc); - PathBuf::from(rustc) - } else { - config.download_beta_toolchain(); - config.out.join(config.build.triple).join("stage0/bin/rustc") - }; - - config.initial_cargo = build - .cargo - .map(|cargo| { - t!(PathBuf::from(cargo).canonicalize(), "`initial_cargo` not found on disk") - }) - .unwrap_or_else(|| config.out.join(config.build.triple).join("stage0/bin/cargo")); - - // NOTE: it's important this comes *after* we set `initial_rustc` just above. - if config.dry_run() { - let dir = config.out.join("tmp-dry-run"); - t!(fs::create_dir_all(&dir)); - config.out = dir; - } - - config.hosts = if let Some(TargetSelectionList(arg_host)) = flags.host { - arg_host - } else if let Some(file_host) = build.host { - file_host.iter().map(|h| TargetSelection::from_user(h)).collect() - } else { - vec![config.build] - }; - config.targets = if let Some(TargetSelectionList(arg_target)) = flags.target { - arg_target - } else if let Some(file_target) = build.target { - file_target.iter().map(|h| TargetSelection::from_user(h)).collect() - } else { - // If target is *not* configured, then default to the host - // toolchains. - config.hosts.clone() - }; - - config.nodejs = build.nodejs.map(PathBuf::from); - config.npm = build.npm.map(PathBuf::from); - config.gdb = build.gdb.map(PathBuf::from); - config.python = build.python.map(PathBuf::from); - config.reuse = build.reuse.map(PathBuf::from); - config.submodules = build.submodules; - set(&mut config.low_priority, build.low_priority); - set(&mut config.compiler_docs, build.compiler_docs); - set(&mut config.library_docs_private_items, build.library_docs_private_items); - set(&mut config.docs_minification, build.docs_minification); - set(&mut config.docs, build.docs); - set(&mut config.locked_deps, build.locked_deps); - set(&mut config.vendor, build.vendor); - set(&mut config.full_bootstrap, build.full_bootstrap); - set(&mut config.extended, build.extended); - config.tools = build.tools; - set(&mut config.verbose, build.verbose); - set(&mut config.sanitizers, build.sanitizers); - set(&mut config.profiler, build.profiler); - set(&mut config.cargo_native_static, build.cargo_native_static); - set(&mut config.configure_args, build.configure_args); - set(&mut config.local_rebuild, build.local_rebuild); - set(&mut config.print_step_timings, build.print_step_timings); - set(&mut config.print_step_rusage, build.print_step_rusage); - config.patch_binaries_for_nix = build.patch_binaries_for_nix; - - config.verbose = cmp::max(config.verbose, flags.verbose as usize); - - if let Some(install) = toml.install { - config.prefix = install.prefix.map(PathBuf::from); - config.sysconfdir = install.sysconfdir.map(PathBuf::from); - config.datadir = install.datadir.map(PathBuf::from); - config.docdir = install.docdir.map(PathBuf::from); - set(&mut config.bindir, install.bindir.map(PathBuf::from)); - config.libdir = install.libdir.map(PathBuf::from); - config.mandir = install.mandir.map(PathBuf::from); - } - - // Store off these values as options because if they're not provided - // we'll infer default values for them later - let mut llvm_assertions = None; - let mut llvm_tests = None; - let mut llvm_plugins = None; - let mut debug = None; - let mut debug_assertions = None; - let mut debug_assertions_std = None; - let mut overflow_checks = None; - let mut overflow_checks_std = None; - let mut debug_logging = None; - let mut debuginfo_level = None; - let mut debuginfo_level_rustc = None; - let mut debuginfo_level_std = None; - let mut debuginfo_level_tools = None; - let mut debuginfo_level_tests = None; - let mut optimize = None; - let mut omit_git_hash = None; - - if let Some(rust) = toml.rust { - set(&mut config.channel, rust.channel); - - config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); - // This list is incomplete, please help by expanding it! - if config.download_rustc_commit.is_some() { - // We need the channel used by the downloaded compiler to match the one we set for rustdoc; - // otherwise rustdoc-ui tests break. - let ci_channel = t!(fs::read_to_string(config.src.join("src/ci/channel"))); - let ci_channel = ci_channel.trim_end(); - if config.channel != ci_channel - && !(config.channel == "dev" && ci_channel == "nightly") - { - panic!( - "setting rust.channel={} is incompatible with download-rustc", - config.channel - ); - } - } - - debug = rust.debug; - debug_assertions = rust.debug_assertions; - debug_assertions_std = rust.debug_assertions_std; - overflow_checks = rust.overflow_checks; - overflow_checks_std = rust.overflow_checks_std; - debug_logging = rust.debug_logging; - debuginfo_level = rust.debuginfo_level; - debuginfo_level_rustc = rust.debuginfo_level_rustc; - debuginfo_level_std = rust.debuginfo_level_std; - debuginfo_level_tools = rust.debuginfo_level_tools; - debuginfo_level_tests = rust.debuginfo_level_tests; - - config.rust_split_debuginfo = rust - .split_debuginfo - .as_deref() - .map(SplitDebuginfo::from_str) - .map(|v| v.expect("invalid value for rust.split_debuginfo")) - .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple)); - optimize = rust.optimize; - omit_git_hash = rust.omit_git_hash; - config.rust_new_symbol_mangling = rust.new_symbol_mangling; - set(&mut config.rust_optimize_tests, rust.optimize_tests); - set(&mut config.codegen_tests, rust.codegen_tests); - set(&mut config.rust_rpath, rust.rpath); - set(&mut config.jemalloc, rust.jemalloc); - set(&mut config.test_compare_mode, rust.test_compare_mode); - set(&mut config.backtrace, rust.backtrace); - config.description = rust.description; - set(&mut config.rust_dist_src, rust.dist_src); - set(&mut config.verbose_tests, rust.verbose_tests); - // in the case "false" is set explicitly, do not overwrite the command line args - if let Some(true) = rust.incremental { - config.incremental = true; - } - set(&mut config.use_lld, rust.use_lld); - set(&mut config.lld_enabled, rust.lld); - set(&mut config.llvm_tools_enabled, rust.llvm_tools); - config.rustc_parallel = rust.parallel_compiler.unwrap_or(false); - config.rustc_default_linker = rust.default_linker; - config.musl_root = rust.musl_root.map(PathBuf::from); - config.save_toolstates = rust.save_toolstates.map(PathBuf::from); - set( - &mut config.deny_warnings, - match flags.warnings { - Warnings::Deny => Some(true), - Warnings::Warn => Some(false), - Warnings::Default => rust.deny_warnings, - }, - ); - set(&mut config.backtrace_on_ice, rust.backtrace_on_ice); - set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir); - config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit; - set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo); - set(&mut config.control_flow_guard, rust.control_flow_guard); - config.llvm_libunwind_default = rust - .llvm_libunwind - .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); - - if let Some(ref backends) = rust.codegen_backends { - let available_backends = vec!["llvm", "cranelift", "gcc"]; - - config.rust_codegen_backends = backends.iter().map(|s| { - if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { - if available_backends.contains(&backend) { - panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); - } else { - println!("help: '{s}' for 'rust.codegen-backends' might fail. \ - Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ - In this case, it would be referred to as '{backend}'."); - } - } - - INTERNER.intern_str(s) - }).collect(); - } - - config.rust_codegen_units = rust.codegen_units.map(threads_from_config); - config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); - config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); - config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); - config.rust_lto = rust - .lto - .as_deref() - .map(|value| RustcLto::from_str(value).unwrap()) - .unwrap_or_default(); - config.rust_validate_mir_opts = rust.validate_mir_opts; - } else { - config.rust_profile_use = flags.rust_profile_use; - config.rust_profile_generate = flags.rust_profile_generate; - } - - config.reproducible_artifacts = flags.reproducible_artifact; - - // rust_info must be set before is_ci_llvm_available() is called. - let default = config.channel == "dev"; - config.omit_git_hash = omit_git_hash.unwrap_or(default); - config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); - - if let Some(llvm) = toml.llvm { - match llvm.ccache { - Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), - Some(StringOrBool::Bool(true)) => { - config.ccache = Some("ccache".to_string()); - } - Some(StringOrBool::Bool(false)) | None => {} - } - set(&mut config.ninja_in_file, llvm.ninja); - llvm_assertions = llvm.assertions; - llvm_tests = llvm.tests; - llvm_plugins = llvm.plugins; - set(&mut config.llvm_optimize, llvm.optimize); - set(&mut config.llvm_thin_lto, llvm.thin_lto); - set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo); - set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp); - if let Some(v) = llvm.link_shared { - config.llvm_link_shared.set(Some(v)); - } - config.llvm_targets = llvm.targets.clone(); - config.llvm_experimental_targets = llvm.experimental_targets.clone(); - config.llvm_link_jobs = llvm.link_jobs; - config.llvm_version_suffix = llvm.version_suffix.clone(); - config.llvm_clang_cl = llvm.clang_cl.clone(); - - config.llvm_cflags = llvm.cflags.clone(); - config.llvm_cxxflags = llvm.cxxflags.clone(); - config.llvm_ldflags = llvm.ldflags.clone(); - set(&mut config.llvm_use_libcxx, llvm.use_libcxx); - config.llvm_use_linker = llvm.use_linker.clone(); - config.llvm_allow_old_toolchain = llvm.allow_old_toolchain.unwrap_or(false); - config.llvm_polly = llvm.polly.unwrap_or(false); - config.llvm_clang = llvm.clang.unwrap_or(false); - config.llvm_enable_warnings = llvm.enable_warnings.unwrap_or(false); - config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default()); - - let asserts = llvm_assertions.unwrap_or(false); - config.llvm_from_ci = match llvm.download_ci_llvm { - Some(StringOrBool::String(s)) => { - assert_eq!(s, "if-available", "unknown option `{s}` for download-ci-llvm"); - crate::llvm::is_ci_llvm_available(&config, asserts) - } - Some(StringOrBool::Bool(b)) => b, - None => { - config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, asserts) - } - }; - - if config.llvm_from_ci { - // None of the LLVM options, except assertions, are supported - // when using downloaded LLVM. We could just ignore these but - // that's potentially confusing, so force them to not be - // explicitly set. The defaults and CI defaults don't - // necessarily match but forcing people to match (somewhat - // arbitrary) CI configuration locally seems bad/hard. - check_ci_llvm!(llvm.optimize); - check_ci_llvm!(llvm.thin_lto); - check_ci_llvm!(llvm.release_debuginfo); - // CI-built LLVM can be either dynamic or static. We won't know until we download it. - check_ci_llvm!(llvm.link_shared); - check_ci_llvm!(llvm.static_libstdcpp); - check_ci_llvm!(llvm.targets); - check_ci_llvm!(llvm.experimental_targets); - check_ci_llvm!(llvm.link_jobs); - check_ci_llvm!(llvm.clang_cl); - check_ci_llvm!(llvm.version_suffix); - check_ci_llvm!(llvm.cflags); - check_ci_llvm!(llvm.cxxflags); - check_ci_llvm!(llvm.ldflags); - check_ci_llvm!(llvm.use_libcxx); - check_ci_llvm!(llvm.use_linker); - check_ci_llvm!(llvm.allow_old_toolchain); - check_ci_llvm!(llvm.polly); - check_ci_llvm!(llvm.clang); - check_ci_llvm!(llvm.build_config); - check_ci_llvm!(llvm.plugins); - } - - // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above. - if config.llvm_thin_lto && llvm.link_shared.is_none() { - // If we're building with ThinLTO on, by default we want to link - // to LLVM shared, to avoid re-doing ThinLTO (which happens in - // the link step) with each stage. - config.llvm_link_shared.set(Some(true)); - } - } else { - config.llvm_from_ci = - config.channel == "dev" && crate::llvm::is_ci_llvm_available(&config, false); - } - - if let Some(t) = toml.target { - for (triple, cfg) in t { - let mut target = Target::from_triple(&triple); - - if let Some(ref s) = cfg.llvm_config { - if config.download_rustc_commit.is_some() && triple == &*config.build.triple { - panic!( - "setting llvm_config for the host is incompatible with download-rustc" - ); - } - target.llvm_config = Some(config.src.join(s)); - } - target.llvm_has_rust_patches = cfg.llvm_has_rust_patches; - if let Some(ref s) = cfg.llvm_filecheck { - target.llvm_filecheck = Some(config.src.join(s)); - } - target.llvm_libunwind = cfg - .llvm_libunwind - .as_ref() - .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); - if let Some(ref s) = cfg.android_ndk { - target.ndk = Some(config.src.join(s)); - } - if let Some(s) = cfg.no_std { - target.no_std = s; - } - target.cc = cfg.cc.map(PathBuf::from).or_else(|| { - target.ndk.as_ref().map(|ndk| ndk_compiler(Language::C, &triple, ndk)) - }); - target.cxx = cfg.cxx.map(PathBuf::from).or_else(|| { - target.ndk.as_ref().map(|ndk| ndk_compiler(Language::CPlusPlus, &triple, ndk)) - }); - target.ar = cfg.ar.map(PathBuf::from); - target.ranlib = cfg.ranlib.map(PathBuf::from); - target.linker = cfg.linker.map(PathBuf::from); - target.crt_static = cfg.crt_static; - target.musl_root = cfg.musl_root.map(PathBuf::from); - target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); - target.wasi_root = cfg.wasi_root.map(PathBuf::from); - target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); - target.sanitizers = cfg.sanitizers; - target.profiler = cfg.profiler; - target.rpath = cfg.rpath; - - config.target_config.insert(TargetSelection::from_user(&triple), target); - } - } - - if config.llvm_from_ci { - let triple = &config.build.triple; - let ci_llvm_bin = config.ci_llvm_root().join("bin"); - let build_target = config - .target_config - .entry(config.build) - .or_insert_with(|| Target::from_triple(&triple)); - - check_ci_llvm!(build_target.llvm_config); - check_ci_llvm!(build_target.llvm_filecheck); - build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); - build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); - } - - if let Some(t) = toml.dist { - config.dist_sign_folder = t.sign_folder.map(PathBuf::from); - config.dist_upload_addr = t.upload_addr; - config.dist_compression_formats = t.compression_formats; - set(&mut config.dist_compression_profile, t.compression_profile); - set(&mut config.rust_dist_src, t.src_tarball); - set(&mut config.missing_tools, t.missing_tools); - set(&mut config.dist_include_mingw_linker, t.include_mingw_linker) - } - - if let Some(r) = build.rustfmt { - *config.initial_rustfmt.borrow_mut() = if r.exists() { - RustfmtState::SystemToolchain(r) - } else { - RustfmtState::Unavailable - }; - } - - // Now that we've reached the end of our configuration, infer the - // default values for all options that we haven't otherwise stored yet. - - config.llvm_assertions = llvm_assertions.unwrap_or(false); - config.llvm_tests = llvm_tests.unwrap_or(false); - config.llvm_plugins = llvm_plugins.unwrap_or(false); - config.rust_optimize = optimize.unwrap_or(RustOptimize::Bool(true)); - - let default = debug == Some(true); - config.rust_debug_assertions = debug_assertions.unwrap_or(default); - config.rust_debug_assertions_std = - debug_assertions_std.unwrap_or(config.rust_debug_assertions); - config.rust_overflow_checks = overflow_checks.unwrap_or(default); - config.rust_overflow_checks_std = - overflow_checks_std.unwrap_or(config.rust_overflow_checks); - - config.rust_debug_logging = debug_logging.unwrap_or(config.rust_debug_assertions); - - let with_defaults = |debuginfo_level_specific: Option<_>| { - debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { - DebuginfoLevel::Limited - } else { - DebuginfoLevel::None - }) - }; - config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); - config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); - config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); - config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(DebuginfoLevel::None); - - let download_rustc = config.download_rustc_commit.is_some(); - // See https://github.com/rust-lang/compiler-team/issues/326 - config.stage = match config.cmd { - Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0), - // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. - Subcommand::Doc { .. } => { - flags.stage.or(build.doc_stage).unwrap_or(if download_rustc { 2 } else { 0 }) - } - Subcommand::Build { .. } => { - flags.stage.or(build.build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) - } - Subcommand::Test { .. } => { - flags.stage.or(build.test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) - } - Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2), - Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2), - Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2), - // These are all bootstrap tools, which don't depend on the compiler. - // The stage we pass shouldn't matter, but use 0 just in case. - Subcommand::Clean { .. } - | Subcommand::Clippy { .. } - | Subcommand::Fix { .. } - | Subcommand::Run { .. } - | Subcommand::Setup { .. } - | Subcommand::Format { .. } - | Subcommand::Suggest { .. } => flags.stage.unwrap_or(0), - }; - - // CI should always run stage 2 builds, unless it specifically states otherwise - #[cfg(not(test))] - if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None { - match config.cmd { - Subcommand::Test { .. } - | Subcommand::Doc { .. } - | Subcommand::Build { .. } - | Subcommand::Bench { .. } - | Subcommand::Dist { .. } - | Subcommand::Install { .. } => { - assert_eq!( - config.stage, 2, - "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", - config.stage, - ); - } - Subcommand::Clean { .. } - | Subcommand::Check { .. } - | Subcommand::Clippy { .. } - | Subcommand::Fix { .. } - | Subcommand::Run { .. } - | Subcommand::Setup { .. } - | Subcommand::Format { .. } - | Subcommand::Suggest { .. } => {} - } - } - - config - } - - pub(crate) fn dry_run(&self) -> bool { - match self.dry_run { - DryRun::Disabled => false, - DryRun::SelfCheck | DryRun::UserSelected => true, - } - } - - /// Runs a command, printing out nice contextual information if it fails. - /// Exits if the command failed to execute at all, otherwise returns its - /// `status.success()`. - #[deprecated = "use `Builder::try_run` instead where possible"] - pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> { - if self.dry_run() { - return Ok(()); - } - self.verbose(&format!("running: {cmd:?}")); - build_helper::util::try_run(cmd, self.is_verbose()) - } - - /// A git invocation which runs inside the source directory. - /// - /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. - pub(crate) fn git(&self) -> Command { - let mut git = Command::new("git"); - git.current_dir(&self.src); - git - } - - pub(crate) fn test_args(&self) -> Vec<&str> { - let mut test_args = match self.cmd { - Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { - test_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - test_args.extend(self.free_args.iter().map(|s| s.as_str())); - test_args - } - - pub(crate) fn args(&self) -> Vec<&str> { - let mut args = match self.cmd { - Subcommand::Run { ref args, .. } => { - args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - }; - args.extend(self.free_args.iter().map(|s| s.as_str())); - args - } - - /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. - /// Return the version it would have used for the given commit. - pub(crate) fn artifact_version_part(&self, commit: &str) -> String { - let (channel, version) = if self.rust_info.is_managed_git_subrepository() { - let mut channel = self.git(); - channel.arg("show").arg(format!("{commit}:src/ci/channel")); - let channel = output(&mut channel); - let mut version = self.git(); - version.arg("show").arg(format!("{commit}:src/version")); - let version = output(&mut version); - (channel.trim().to_owned(), version.trim().to_owned()) - } else { - let channel = fs::read_to_string(self.src.join("src/ci/channel")); - let version = fs::read_to_string(self.src.join("src/version")); - match (channel, version) { - (Ok(channel), Ok(version)) => { - (channel.trim().to_owned(), version.trim().to_owned()) - } - (channel, version) => { - let src = self.src.display(); - eprintln!("error: failed to determine artifact channel and/or version"); - eprintln!( - "help: consider using a git checkout or ensure these files are readable" - ); - if let Err(channel) = channel { - eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); - } - if let Err(version) = version { - eprintln!("reading {src}/src/version failed: {version:?}"); - } - panic!(); - } - } - }; - - match channel.as_str() { - "stable" => version, - "beta" => channel, - "nightly" => channel, - other => unreachable!("{:?} is not recognized as a valid channel", other), - } - } - - /// Try to find the relative path of `bindir`, otherwise return it in full. - pub fn bindir_relative(&self) -> &Path { - let bindir = &self.bindir; - if bindir.is_absolute() { - // Try to make it relative to the prefix. - if let Some(prefix) = &self.prefix { - if let Ok(stripped) = bindir.strip_prefix(prefix) { - return stripped; - } - } - } - bindir - } - - /// Try to find the relative path of `libdir`. - pub fn libdir_relative(&self) -> Option<&Path> { - let libdir = self.libdir.as_ref()?; - if libdir.is_relative() { - Some(libdir) - } else { - // Try to make it relative to the prefix. - libdir.strip_prefix(self.prefix.as_ref()?).ok() - } - } - - /// The absolute path to the downloaded LLVM artifacts. - pub(crate) fn ci_llvm_root(&self) -> PathBuf { - assert!(self.llvm_from_ci); - self.out.join(&*self.build.triple).join("ci-llvm") - } - - /// Directory where the extracted `rustc-dev` component is stored. - pub(crate) fn ci_rustc_dir(&self) -> PathBuf { - assert!(self.download_rustc()); - self.out.join(self.build.triple).join("ci-rustc") - } - - /// Determine whether llvm should be linked dynamically. - /// - /// If `false`, llvm should be linked statically. - /// This is computed on demand since LLVM might have to first be downloaded from CI. - pub(crate) fn llvm_link_shared(&self) -> bool { - let mut opt = self.llvm_link_shared.get(); - if opt.is_none() && self.dry_run() { - // just assume static for now - dynamic linking isn't supported on all platforms - return false; - } - - let llvm_link_shared = *opt.get_or_insert_with(|| { - if self.llvm_from_ci { - self.maybe_download_ci_llvm(); - let ci_llvm = self.ci_llvm_root(); - let link_type = t!( - std::fs::read_to_string(ci_llvm.join("link-type.txt")), - format!("CI llvm missing: {}", ci_llvm.display()) - ); - link_type == "dynamic" - } else { - // unclear how thought-through this default is, but it maintains compatibility with - // previous behavior - false - } - }); - self.llvm_link_shared.set(opt); - llvm_link_shared - } - - /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source. - pub(crate) fn download_rustc(&self) -> bool { - self.download_rustc_commit().is_some() - } - - pub(crate) fn download_rustc_commit(&self) -> Option<&str> { - static DOWNLOAD_RUSTC: OnceCell> = OnceCell::new(); - if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() { - // avoid trying to actually download the commit - return self.download_rustc_commit.as_deref(); - } - - DOWNLOAD_RUSTC - .get_or_init(|| match &self.download_rustc_commit { - None => None, - Some(commit) => { - self.download_ci_rustc(commit); - Some(commit.clone()) - } - }) - .as_deref() - } - - pub(crate) fn initial_rustfmt(&self) -> Option { - match &mut *self.initial_rustfmt.borrow_mut() { - RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), - RustfmtState::Unavailable => None, - r @ RustfmtState::LazyEvaluated => { - if self.dry_run() { - return Some(PathBuf::new()); - } - let path = self.maybe_download_rustfmt(); - *r = if let Some(p) = &path { - RustfmtState::Downloaded(p.clone()) - } else { - RustfmtState::Unavailable - }; - path - } - } - } - - pub fn verbose(&self, msg: &str) { - if self.verbose > 0 { - println!("{msg}"); - } - } - - pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).map(|t| t.sanitizers).flatten().unwrap_or(self.sanitizers) - } - - pub fn any_sanitizers_enabled(&self) -> bool { - self.target_config.values().any(|t| t.sanitizers == Some(true)) || self.sanitizers - } - - pub fn profiler_path(&self, target: TargetSelection) -> Option<&str> { - match self.target_config.get(&target)?.profiler.as_ref()? { - StringOrBool::String(s) => Some(s), - StringOrBool::Bool(_) => None, - } - } - - pub fn profiler_enabled(&self, target: TargetSelection) -> bool { - self.target_config - .get(&target) - .and_then(|t| t.profiler.as_ref()) - .map(StringOrBool::is_string_or_true) - .unwrap_or(self.profiler) - } - - pub fn any_profiler_enabled(&self) -> bool { - self.target_config.values().any(|t| matches!(&t.profiler, Some(p) if p.is_string_or_true())) - || self.profiler - } - - pub fn rpath_enabled(&self, target: TargetSelection) -> bool { - self.target_config.get(&target).map(|t| t.rpath).flatten().unwrap_or(self.rust_rpath) - } - - pub fn llvm_enabled(&self) -> bool { - self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) - } - - pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { - self.target_config - .get(&target) - .and_then(|t| t.llvm_libunwind) - .or(self.llvm_libunwind_default) - .unwrap_or(if target.contains("fuchsia") { - LlvmLibunwind::InTree - } else { - LlvmLibunwind::No - }) - } - - pub fn submodules(&self, rust_info: &GitInfo) -> bool { - self.submodules.unwrap_or(rust_info.is_managed_git_subrepository()) - } - - pub fn default_codegen_backend(&self) -> Option> { - self.rust_codegen_backends.get(0).cloned() - } - - pub fn check_build_rustc_version(&self, rustc_path: &str) { - if self.dry_run() { - return; - } - - // check rustc version is same or lower with 1 apart from the building one - let mut cmd = Command::new(rustc_path); - cmd.arg("--version"); - let rustc_output = output(&mut cmd) - .lines() - .next() - .unwrap() - .split(' ') - .nth(1) - .unwrap() - .split('-') - .next() - .unwrap() - .to_owned(); - let rustc_version = Version::parse(&rustc_output.trim()).unwrap(); - let source_version = - Version::parse(&fs::read_to_string(self.src.join("src/version")).unwrap().trim()) - .unwrap(); - if !(source_version == rustc_version - || (source_version.major == rustc_version.major - && (source_version.minor == rustc_version.minor - || source_version.minor == rustc_version.minor + 1))) - { - let prev_version = format!("{}.{}.x", source_version.major, source_version.minor - 1); - eprintln!( - "Unexpected rustc version: {rustc_version}, we should use {prev_version}/{source_version} to build source with {source_version}" - ); - exit!(1); - } - } - - /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. - fn download_ci_rustc_commit(&self, download_rustc: Option) -> Option { - // If `download-rustc` is not set, default to rebuilding. - let if_unchanged = match download_rustc { - None | Some(StringOrBool::Bool(false)) => return None, - Some(StringOrBool::Bool(true)) => false, - Some(StringOrBool::String(s)) if s == "if-unchanged" => true, - Some(StringOrBool::String(other)) => { - panic!("unrecognized option for download-rustc: {other}") - } - }; - - // Handle running from a directory other than the top level - let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); - let top_level = top_level.trim_end(); - let compiler = format!("{top_level}/compiler/"); - let library = format!("{top_level}/library/"); - - // Look for a version to compare to based on the current commit. - // Only commits merged by bors will have CI artifacts. - let merge_base = output( - self.git() - .arg("rev-list") - .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) - .args(&["-n1", "--first-parent", "HEAD"]), - ); - let commit = merge_base.trim_end(); - if commit.is_empty() { - println!("error: could not find commit hash for downloading rustc"); - println!("help: maybe your repository history is too shallow?"); - println!("help: consider disabling `download-rustc`"); - println!("help: or fetch enough history to include one upstream commit"); - crate::exit!(1); - } - - // Warn if there were changes to the compiler or standard library since the ancestor commit. - let has_changes = !t!(self - .git() - .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) - .status()) - .success(); - if has_changes { - if if_unchanged { - if self.verbose > 0 { - println!( - "warning: saw changes to compiler/ or library/ since {commit}; \ - ignoring `download-rustc`" - ); - } - return None; - } - println!( - "warning: `download-rustc` is enabled, but there are changes to \ - compiler/ or library/" - ); - } - - Some(commit.to_string()) - } -} - -fn set(field: &mut T, val: Option) { - if let Some(v) = val { - *field = v; - } -} - -fn threads_from_config(v: u32) -> u32 { - match v { - 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, - n => n, - } -} diff --git a/src/bootstrap/config/tests.rs b/src/bootstrap/config/tests.rs deleted file mode 100644 index aac76cdcb..000000000 --- a/src/bootstrap/config/tests.rs +++ /dev/null @@ -1,198 +0,0 @@ -use crate::config::TomlConfig; - -use super::{Config, Flags}; -use clap::CommandFactory; -use serde::Deserialize; -use std::{env, path::Path}; - -fn parse(config: &str) -> Config { - Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], |&_| { - toml::from_str(config).unwrap() - }) -} - -#[test] -fn download_ci_llvm() { - if crate::llvm::is_ci_llvm_modified(&parse("")) { - eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); - return; - } - - let parse_llvm = |s| parse(s).llvm_from_ci; - let if_available = parse_llvm("llvm.download-ci-llvm = \"if-available\""); - - assert!(parse_llvm("llvm.download-ci-llvm = true")); - assert!(!parse_llvm("llvm.download-ci-llvm = false")); - assert_eq!(parse_llvm(""), if_available); - assert_eq!(parse_llvm("rust.channel = \"dev\""), if_available); - assert!(!parse_llvm("rust.channel = \"stable\"")); - assert!(parse_llvm("build.build = \"x86_64-unknown-linux-gnu\"")); - assert!(parse_llvm( - "llvm.assertions = true \r\n build.build = \"x86_64-unknown-linux-gnu\" \r\n llvm.download-ci-llvm = \"if-available\"" - )); - assert!(!parse_llvm( - "llvm.assertions = true \r\n build.build = \"aarch64-apple-darwin\" \r\n llvm.download-ci-llvm = \"if-available\"" - )); -} - -// FIXME(onur-ozkan): extend scope of the test -// refs: -// - https://github.com/rust-lang/rust/issues/109120 -// - https://github.com/rust-lang/rust/pull/109162#issuecomment-1496782487 -#[test] -fn detect_src_and_out() { - fn test(cfg: Config, build_dir: Option<&str>) { - // This will bring absolute form of `src/bootstrap` path - let current_dir = std::env::current_dir().unwrap(); - - // get `src` by moving into project root path - let expected_src = current_dir.ancestors().nth(2).unwrap(); - assert_eq!(&cfg.src, expected_src); - - // Sanity check for `src` - let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); - let expected_src = manifest_dir.ancestors().nth(2).unwrap(); - assert_eq!(&cfg.src, expected_src); - - // test if build-dir was manually given in config.toml - if let Some(custom_build_dir) = build_dir { - assert_eq!(&cfg.out, Path::new(custom_build_dir)); - } - // test the native bootstrap way - else { - // This should bring output path of bootstrap in absolute form - let cargo_target_dir = env::var_os("CARGO_TARGET_DIR").expect( - "CARGO_TARGET_DIR must been provided for the test environment from bootstrap", - ); - - // Move to `build` from `build/bootstrap` - let expected_out = Path::new(&cargo_target_dir).parent().unwrap(); - assert_eq!(&cfg.out, expected_out); - - let args: Vec = env::args().collect(); - - // Another test for `out` as a sanity check - // - // This will bring something similar to: - // `{build-dir}/bootstrap/debug/deps/bootstrap-c7ee91d5661e2804` - // `{build-dir}` can be anywhere, not just in the rust project directory. - let dep = Path::new(args.first().unwrap()); - let expected_out = dep.ancestors().nth(4).unwrap(); - - assert_eq!(&cfg.out, expected_out); - } - } - - test(parse(""), None); - - { - let build_dir = if cfg!(windows) { Some("C:\\tmp") } else { Some("/tmp") }; - test(parse("build.build-dir = \"/tmp\""), build_dir); - } -} - -#[test] -fn clap_verify() { - Flags::command().debug_assert(); -} - -#[test] -fn override_toml() { - let config = Config::parse_inner( - &[ - "check".to_owned(), - "--config=/does/not/exist".to_owned(), - "--set=changelog-seen=1".to_owned(), - "--set=rust.lto=fat".to_owned(), - "--set=rust.deny-warnings=false".to_owned(), - "--set=build.gdb=\"bar\"".to_owned(), - "--set=build.tools=[\"cargo\"]".to_owned(), - "--set=llvm.build-config={\"foo\" = \"bar\"}".to_owned(), - ], - |&_| { - toml::from_str( - r#" -changelog-seen = 0 -[rust] -lto = "off" -deny-warnings = true - -[build] -gdb = "foo" -tools = [] - -[llvm] -download-ci-llvm = false -build-config = {} - "#, - ) - .unwrap() - }, - ); - assert_eq!(config.changelog_seen, Some(1), "setting top-level value"); - assert_eq!( - config.rust_lto, - crate::config::RustcLto::Fat, - "setting string value without quotes" - ); - assert_eq!(config.gdb, Some("bar".into()), "setting string value with quotes"); - assert!(!config.deny_warnings, "setting boolean value"); - assert_eq!( - config.tools, - Some(["cargo".to_string()].into_iter().collect()), - "setting list value" - ); - assert_eq!( - config.llvm_build_config, - [("foo".to_string(), "bar".to_string())].into_iter().collect(), - "setting dictionary value" - ); -} - -#[test] -#[should_panic] -fn override_toml_duplicate() { - Config::parse_inner( - &[ - "check".to_owned(), - "--config=/does/not/exist".to_owned(), - "--set=changelog-seen=1".to_owned(), - "--set=changelog-seen=2".to_owned(), - ], - |&_| toml::from_str("changelog-seen = 0").unwrap(), - ); -} - -#[test] -fn profile_user_dist() { - fn get_toml(file: &Path) -> TomlConfig { - let contents = if file.ends_with("config.toml") { - "profile = \"user\"".to_owned() - } else { - assert!(file.ends_with("config.dist.toml")); - std::fs::read_to_string(dbg!(file)).unwrap() - }; - toml::from_str(&contents) - .and_then(|table: toml::Value| TomlConfig::deserialize(table)) - .unwrap() - } - Config::parse_inner(&["check".to_owned()], get_toml); -} - -#[test] -fn rust_optimize() { - assert!(parse("").rust_optimize.is_release()); - assert!(!parse("rust.optimize = false").rust_optimize.is_release()); - assert!(parse("rust.optimize = true").rust_optimize.is_release()); - assert!(!parse("rust.optimize = 0").rust_optimize.is_release()); - assert!(parse("rust.optimize = 1").rust_optimize.is_release()); - assert!(parse("rust.optimize = \"s\"").rust_optimize.is_release()); - assert_eq!(parse("rust.optimize = 1").rust_optimize.get_opt_level(), Some("1".to_string())); - assert_eq!(parse("rust.optimize = \"s\"").rust_optimize.get_opt_level(), Some("s".to_string())); -} - -#[test] -#[should_panic] -fn invalid_rust_optimize() { - parse("rust.optimize = \"a\""); -} diff --git a/src/bootstrap/configure.py b/src/bootstrap/configure.py index f469dbea6..544a42d9a 100755 --- a/src/bootstrap/configure.py +++ b/src/bootstrap/configure.py @@ -59,6 +59,7 @@ o("missing-tools", "dist.missing-tools", "allow failures when building tools") o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++") o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard") o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains") +o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0") v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags") v("llvm-cxxflags", "llvm.cxxflags", "build LLVM with these extra compiler flags") @@ -97,20 +98,7 @@ v("llvm-root", None, "set LLVM root") v("llvm-config", None, "set path to llvm-config") v("llvm-filecheck", None, "set path to LLVM's FileCheck utility") v("python", "build.python", "set path to python") -v("android-cross-path", "target.arm-linux-androideabi.android-ndk", - "Android NDK standalone path (deprecated)") -v("i686-linux-android-ndk", "target.i686-linux-android.android-ndk", - "i686-linux-android NDK standalone path") -v("arm-linux-androideabi-ndk", "target.arm-linux-androideabi.android-ndk", - "arm-linux-androideabi NDK standalone path") -v("armv7-linux-androideabi-ndk", "target.armv7-linux-androideabi.android-ndk", - "armv7-linux-androideabi NDK standalone path") -v("thumbv7neon-linux-androideabi-ndk", "target.thumbv7neon-linux-androideabi.android-ndk", - "thumbv7neon-linux-androideabi NDK standalone path") -v("aarch64-linux-android-ndk", "target.aarch64-linux-android.android-ndk", - "aarch64-linux-android NDK standalone path") -v("x86_64-linux-android-ndk", "target.x86_64-linux-android.android-ndk", - "x86_64-linux-android NDK standalone path") +v("android-ndk", "build.android-ndk", "set path to Android NDK") v("musl-root", "target.x86_64-unknown-linux-musl.musl-root", "MUSL root installation directory (deprecated)") v("musl-root-x86_64", "target.x86_64-unknown-linux-musl.musl-root", @@ -265,7 +253,7 @@ def parse_args(args): if not found: unknown_args.append(arg) - # Note: here and a few other places, we use [-1] to apply the *last* value + # NOTE: here and a few other places, we use [-1] to apply the *last* value # passed. But if option-checking is enabled, then the known_args loop will # also assert that options are only passed once. option_checking = ('option-checking' not in known_args @@ -489,7 +477,7 @@ def configure_section(lines, config): # These are used by rpm, but aren't accepted by x.py. # Give a warning that they're ignored, but not a hard error. if key in ["infodir", "localstatedir"]: - print("warning: {} will be ignored".format(key)) + print("WARNING: {} will be ignored".format(key)) else: raise RuntimeError("failed to find config line for {}".format(key)) diff --git a/src/bootstrap/defaults/config.codegen.toml b/src/bootstrap/defaults/config.codegen.toml index 113df88d7..7c33ce958 100644 --- a/src/bootstrap/defaults/config.codegen.toml +++ b/src/bootstrap/defaults/config.codegen.toml @@ -10,7 +10,7 @@ assertions = true # enable warnings during the llvm compilation enable-warnings = true # build llvm from source -download-ci-llvm = false +download-ci-llvm = "if-unchanged" [rust] # This enables `RUSTC_LOG=debug`, avoiding confusing situations diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs deleted file mode 100644 index 32da4ac29..000000000 --- a/src/bootstrap/dist.rs +++ /dev/null @@ -1,2274 +0,0 @@ -//! Implementation of the various distribution aspects of the compiler. -//! -//! This module is responsible for creating tarballs of the standard library, -//! compiler, and documentation. This ends up being what we distribute to -//! everyone as well. -//! -//! No tarball is actually created literally in this file, but rather we shell -//! out to `rust-installer` still. This may one day be replaced with bits and -//! pieces of `rustup.rs`! - -use std::collections::HashSet; -use std::env; -use std::ffi::OsStr; -use std::fs; -use std::io::Write; -use std::path::{Path, PathBuf}; -use std::process::Command; - -use object::read::archive::ArchiveFile; -use object::BinaryFormat; - -use crate::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::{Interned, INTERNER}; -use crate::channel; -use crate::compile; -use crate::config::TargetSelection; -use crate::doc::DocumentationFormat; -use crate::llvm; -use crate::tarball::{GeneratedTarball, OverlayKind, Tarball}; -use crate::tool::{self, Tool}; -use crate::util::{exe, is_dylib, output, t, timeit}; -use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; - -pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { - format!("{}-{}", component, builder.rust_package_vers()) -} - -pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf { - builder.out.join("dist") -} - -pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { - builder.out.join("tmp/dist") -} - -fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { - if !builder.config.extended { - return false; - } - builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool)) -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Docs { - pub host: TargetSelection, -} - -impl Step for Docs { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.docs; - run.alias("rust-docs").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Docs { host: run.target }); - } - - /// Builds the `rust-docs` installer component. - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - builder.default_doc(&[]); - - let dest = "share/doc/rust/html"; - - let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); - tarball.set_product_name("Rust Documentation"); - tarball.add_bulk_dir(&builder.doc_out(host), dest); - tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct JsonDocs { - pub host: TargetSelection, -} - -impl Step for JsonDocs { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.docs; - run.alias("rust-docs-json").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(JsonDocs { host: run.target }); - } - - /// Builds the `rust-docs-json` installer component. - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - builder.ensure(crate::doc::Std::new( - builder.top_stage, - host, - builder, - DocumentationFormat::JSON, - )); - - let dest = "share/doc/rust/json"; - - let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple); - tarball.set_product_name("Rust Documentation In JSON Format"); - tarball.is_preview(true); - tarball.add_bulk_dir(&builder.json_doc_out(host), dest); - Some(tarball.generate()) - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustcDocs { - pub host: TargetSelection, -} - -impl Step for RustcDocs { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.alias("rustc-docs").default_condition(builder.config.compiler_docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcDocs { host: run.target }); - } - - /// Builds the `rustc-docs` installer component. - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - builder.default_doc(&[]); - - let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); - tarball.set_product_name("Rustc Documentation"); - tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); - Some(tarball.generate()) - } -} - -fn find_files(files: &[&str], path: &[PathBuf]) -> Vec { - let mut found = Vec::with_capacity(files.len()); - - for file in files { - let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists()); - - if let Some(file_path) = file_path { - found.push(file_path); - } else { - panic!("Could not find '{file}' in {path:?}"); - } - } - - found -} - -fn make_win_dist( - rust_root: &Path, - plat_root: &Path, - target: TargetSelection, - builder: &Builder<'_>, -) { - if builder.config.dry_run() { - return; - } - - //Ask gcc where it keeps its stuff - let mut cmd = Command::new(builder.cc(target)); - cmd.arg("-print-search-dirs"); - let gcc_out = output(&mut cmd); - - let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect(); - let mut lib_path = Vec::new(); - - for line in gcc_out.lines() { - let idx = line.find(':').unwrap(); - let key = &line[..idx]; - let trim_chars: &[_] = &[' ', '=']; - let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars)); - - if key == "programs" { - bin_path.extend(value); - } else if key == "libraries" { - lib_path.extend(value); - } - } - - let compiler = if target == "i686-pc-windows-gnu" { - "i686-w64-mingw32-gcc.exe" - } else if target == "x86_64-pc-windows-gnu" { - "x86_64-w64-mingw32-gcc.exe" - } else { - "gcc.exe" - }; - let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"]; - let mut rustc_dlls = vec!["libwinpthread-1.dll"]; - if target.starts_with("i686-") { - rustc_dlls.push("libgcc_s_dw2-1.dll"); - } else { - rustc_dlls.push("libgcc_s_seh-1.dll"); - } - - // Libraries necessary to link the windows-gnu toolchains. - // System libraries will be preferred if they are available (see #67429). - let target_libs = [ - //MinGW libs - "libgcc.a", - "libgcc_eh.a", - "libgcc_s.a", - "libm.a", - "libmingw32.a", - "libmingwex.a", - "libstdc++.a", - "libiconv.a", - "libmoldname.a", - "libpthread.a", - //Windows import libs - //This should contain only the set of libraries necessary to link the standard library. - "libadvapi32.a", - "libbcrypt.a", - "libcomctl32.a", - "libcomdlg32.a", - "libcredui.a", - "libcrypt32.a", - "libdbghelp.a", - "libgdi32.a", - "libimagehlp.a", - "libiphlpapi.a", - "libkernel32.a", - "libmsimg32.a", - "libmsvcrt.a", - "libntdll.a", - "libodbc32.a", - "libole32.a", - "liboleaut32.a", - "libopengl32.a", - "libpsapi.a", - "librpcrt4.a", - "libsecur32.a", - "libsetupapi.a", - "libshell32.a", - "libsynchronization.a", - "libuser32.a", - "libuserenv.a", - "libuuid.a", - "libwinhttp.a", - "libwinmm.a", - "libwinspool.a", - "libws2_32.a", - "libwsock32.a", - ]; - - //Find mingw artifacts we want to bundle - let target_tools = find_files(&target_tools, &bin_path); - let rustc_dlls = find_files(&rustc_dlls, &bin_path); - let target_libs = find_files(&target_libs, &lib_path); - - // Copy runtime dlls next to rustc.exe - let dist_bin_dir = rust_root.join("bin/"); - fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed"); - for src in rustc_dlls { - builder.copy_to_folder(&src, &dist_bin_dir); - } - - //Copy platform tools to platform-specific bin directory - let target_bin_dir = plat_root - .join("lib") - .join("rustlib") - .join(target.triple) - .join("bin") - .join("self-contained"); - fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed"); - for src in target_tools { - builder.copy_to_folder(&src, &target_bin_dir); - } - - // Warn windows-gnu users that the bundled GCC cannot compile C files - builder.create( - &target_bin_dir.join("GCC-WARNING.txt"), - "gcc.exe contained in this folder cannot be used for compiling C files - it is only \ - used as a linker. In order to be able to compile projects containing C code use \ - the GCC provided by MinGW or Cygwin.", - ); - - //Copy platform libs to platform-specific lib directory - let target_lib_dir = plat_root - .join("lib") - .join("rustlib") - .join(target.triple) - .join("lib") - .join("self-contained"); - fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed"); - for src in target_libs { - builder.copy_to_folder(&src, &target_lib_dir); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Mingw { - pub host: TargetSelection, -} - -impl Step for Mingw { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-mingw") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Mingw { host: run.target }); - } - - /// Builds the `rust-mingw` installer component. - /// - /// This contains all the bits and pieces to run the MinGW Windows targets - /// without any extra installed software (e.g., we bundle gcc, libraries, etc). - fn run(self, builder: &Builder<'_>) -> Option { - let host = self.host; - if !host.ends_with("pc-windows-gnu") || !builder.config.dist_include_mingw_linker { - return None; - } - - let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); - tarball.set_product_name("Rust MinGW"); - - // The first argument is a "temporary directory" which is just - // thrown away (this contains the runtime DLLs included in the rustc package - // above) and the second argument is where to place all the MinGW components - // (which is what we want). - make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Rustc { - pub compiler: Compiler, -} - -impl Step for Rustc { - type Output = GeneratedTarball; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rustc") - } - - fn make_run(run: RunConfig<'_>) { - run.builder - .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); - } - - /// Creates the `rustc` installer component. - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - let compiler = self.compiler; - let host = self.compiler.host; - - let tarball = Tarball::new(builder, "rustc", &host.triple); - - // Prepare the rustc "image", what will actually end up getting installed - prepare_image(builder, compiler, tarball.image_dir()); - - // On MinGW we've got a few runtime DLL dependencies that we need to - // include. The first argument to this script is where to put these DLLs - // (the image we're creating), and the second argument is a junk directory - // to ignore all other MinGW stuff the script creates. - // - // On 32-bit MinGW we're always including a DLL which needs some extra - // licenses to distribute. On 64-bit MinGW we don't actually distribute - // anything requiring us to distribute a license, but it's likely the - // install will *also* include the rust-mingw package, which also needs - // licenses, so to be safe we just include it here in all MinGW packages. - if host.ends_with("pc-windows-gnu") && builder.config.dist_include_mingw_linker { - make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); - tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); - } - - return tarball.generate(); - - fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { - let host = compiler.host; - let src = builder.sysroot(compiler); - - // Copy rustc/rustdoc binaries - t!(fs::create_dir_all(image.join("bin"))); - builder.cp_r(&src.join("bin"), &image.join("bin")); - - if builder - .config - .tools - .as_ref() - .map_or(true, |tools| tools.iter().any(|tool| tool == "rustdoc")) - { - let rustdoc = builder.rustdoc(compiler); - builder.install(&rustdoc, &image.join("bin"), 0o755); - } - - if let Some(ra_proc_macro_srv) = builder.ensure_if_default( - tool::RustAnalyzerProcMacroSrv { - compiler: builder.compiler_for( - compiler.stage, - builder.config.build, - compiler.host, - ), - target: compiler.host, - }, - builder.kind, - ) { - builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); - } - - let libdir_relative = builder.libdir_relative(compiler); - - // Copy runtime DLLs needed by the compiler - if libdir_relative.to_str() != Some("bin") { - let libdir = builder.rustc_libdir(compiler); - for entry in builder.read_dir(&libdir) { - let name = entry.file_name(); - if let Some(s) = name.to_str() { - if is_dylib(s) { - // Don't use custom libdir here because ^lib/ will be resolved again - // with installer - builder.install(&entry.path(), &image.join("lib"), 0o644); - } - } - } - } - - // Copy over the codegen backends - let backends_src = builder.sysroot_codegen_backends(compiler); - let backends_rel = backends_src - .strip_prefix(&src) - .unwrap() - .strip_prefix(builder.sysroot_libdir_relative(compiler)) - .unwrap(); - // Don't use custom libdir here because ^lib/ will be resolved again with installer - let backends_dst = image.join("lib").join(&backends_rel); - - t!(fs::create_dir_all(&backends_dst)); - builder.cp_r(&backends_src, &backends_dst); - - // Copy libLLVM.so to the lib dir as well, if needed. While not - // technically needed by rustc itself it's needed by lots of other - // components like the llvm tools and LLD. LLD is included below and - // tools/LLDB come later, so let's just throw it in the rustc - // component for now. - maybe_install_llvm_runtime(builder, host, image); - - let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin"); - t!(fs::create_dir_all(&dst_dir)); - - // Copy over lld if it's there - if builder.config.lld_enabled { - let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin"); - let rust_lld = exe("rust-lld", compiler.host); - builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); - // for `-Z gcc-ld=lld` - let gcc_lld_src_dir = src_dir.join("gcc-ld"); - let gcc_lld_dst_dir = dst_dir.join("gcc-ld"); - t!(fs::create_dir(&gcc_lld_dst_dir)); - for name in crate::LLD_FILE_NAMES { - let exe_name = exe(name, compiler.host); - builder - .copy(&gcc_lld_src_dir.join(&exe_name), &gcc_lld_dst_dir.join(&exe_name)); - } - } - - // Man pages - t!(fs::create_dir_all(image.join("share/man/man1"))); - let man_src = builder.src.join("src/doc/man"); - let man_dst = image.join("share/man/man1"); - - // don't use our `bootstrap::util::{copy, cp_r}`, because those try - // to hardlink, and we don't want to edit the source templates - for file_entry in builder.read_dir(&man_src) { - let page_src = file_entry.path(); - let page_dst = man_dst.join(file_entry.file_name()); - let src_text = t!(std::fs::read_to_string(&page_src)); - let new_text = src_text.replace("", &builder.version); - t!(std::fs::write(&page_dst, &new_text)); - t!(fs::copy(&page_src, &page_dst)); - } - - // Debugger scripts - builder - .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host }); - - // Misc license info - let cp = |file: &str| { - builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644); - }; - cp("COPYRIGHT"); - cp("LICENSE-APACHE"); - cp("LICENSE-MIT"); - cp("README.md"); - } - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct DebuggerScripts { - pub sysroot: Interned, - pub host: TargetSelection, -} - -impl Step for DebuggerScripts { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Copies debugger scripts for `target` into the `sysroot` specified. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let sysroot = self.sysroot; - let dst = sysroot.join("lib/rustlib/etc"); - t!(fs::create_dir_all(&dst)); - let cp_debugger_script = |file: &str| { - builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644); - }; - if host.contains("windows-msvc") { - // windbg debugger scripts - builder.install( - &builder.src.join("src/etc/rust-windbg.cmd"), - &sysroot.join("bin"), - 0o755, - ); - - cp_debugger_script("natvis/intrinsic.natvis"); - cp_debugger_script("natvis/liballoc.natvis"); - cp_debugger_script("natvis/libcore.natvis"); - cp_debugger_script("natvis/libstd.natvis"); - } else { - cp_debugger_script("rust_types.py"); - - // gdb debugger scripts - builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755); - builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755); - - cp_debugger_script("gdb_load_rust_pretty_printers.py"); - cp_debugger_script("gdb_lookup.py"); - cp_debugger_script("gdb_providers.py"); - - // lldb debugger scripts - builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755); - - cp_debugger_script("lldb_lookup.py"); - cp_debugger_script("lldb_providers.py"); - cp_debugger_script("lldb_commands") - } - } -} - -fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { - // The only true set of target libraries came from the build triple, so - // let's reduce redundant work by only producing archives from that host. - if compiler.host != builder.config.build { - builder.info("\tskipping, not a build host"); - true - } else { - false - } -} - -/// Check that all objects in rlibs for UEFI targets are COFF. This -/// ensures that the C compiler isn't producing ELF objects, which would -/// not link correctly with the COFF objects. -fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp: &Path) { - if !target.ends_with("-uefi") { - return; - } - - for (path, _) in builder.read_stamp_file(stamp) { - if path.extension() != Some(OsStr::new("rlib")) { - continue; - } - - let data = t!(fs::read(&path)); - let data = data.as_slice(); - let archive = t!(ArchiveFile::parse(data)); - for member in archive.members() { - let member = t!(member); - let member_data = t!(member.data(data)); - - let is_coff = match object::File::parse(member_data) { - Ok(member_file) => member_file.format() == BinaryFormat::Coff, - Err(_) => false, - }; - - if !is_coff { - let member_name = String::from_utf8_lossy(member.name()); - panic!("member {} in {} is not COFF", member_name, path.display()); - } - } - } -} - -/// Copy stamped files into an image's `target/lib` directory. -fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) { - let dst = image.join("lib/rustlib").join(target.triple).join("lib"); - let self_contained_dst = dst.join("self-contained"); - t!(fs::create_dir_all(&dst)); - t!(fs::create_dir_all(&self_contained_dst)); - for (path, dependency_type) in builder.read_stamp_file(stamp) { - if dependency_type == DependencyType::TargetSelfContained { - builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap())); - } else if dependency_type == DependencyType::Target || builder.config.build == target { - builder.copy(&path, &dst.join(path.file_name().unwrap())); - } - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Std { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Std { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-std") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Std { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - if skip_host_target_lib(builder, compiler) { - return None; - } - - builder.ensure(compile::Std::new(compiler, target)); - - let mut tarball = Tarball::new(builder, "rust-std", &target.triple); - tarball.include_target_in_component_name(true); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - let stamp = compile::libstd_stamp(builder, compiler_to_use, target); - verify_uefi_rlib_format(builder, target, &stamp); - copy_target_libs(builder, target, &tarball.image_dir(), &stamp); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustcDev { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustcDev { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rustc-dev") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcDev { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - if skip_host_target_lib(builder, compiler) { - return None; - } - - builder.ensure(compile::Rustc::new(compiler, target)); - - let tarball = Tarball::new(builder, "rustc-dev", &target.triple); - - let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - let stamp = compile::librustc_stamp(builder, compiler_to_use, target); - copy_target_libs(builder, target, tarball.image_dir(), &stamp); - - let src_files = &["Cargo.lock"]; - // This is the reduced set of paths which will become the rustc-dev component - // (essentially the compiler crates and all of their path dependencies). - copy_src_dirs( - builder, - &builder.src, - &["compiler"], - &[], - &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), - ); - for file in src_files { - tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); - } - - Some(tarball.generate()) - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Analysis { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Analysis { - type Output = Option; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "analysis"); - run.alias("rust-analysis").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Analysis { - // Find the actual compiler (handling the full bootstrap option) which - // produced the save-analysis data because that data isn't copied - // through the sysroot uplifting. - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - /// Creates a tarball of (degenerate) save-analysis metadata, if available. - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - if compiler.host != builder.config.build { - return None; - } - - let src = builder - .stage_out(compiler, Mode::Std) - .join(target.triple) - .join(builder.cargo_dir()) - .join("deps") - .join("save-analysis"); - - // Write a file indicating that this component has been removed. - t!(std::fs::create_dir_all(&src)); - let mut removed = src.clone(); - removed.push("removed.json"); - let mut f = t!(std::fs::File::create(removed)); - t!(write!(f, r#"{{ "warning": "The `rust-analysis` component has been removed." }}"#)); - - let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); - tarball.include_target_in_component_name(true); - tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); - Some(tarball.generate()) - } -} - -/// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to -/// `dst_dir`. -fn copy_src_dirs( - builder: &Builder<'_>, - base: &Path, - src_dirs: &[&str], - exclude_dirs: &[&str], - dst_dir: &Path, -) { - fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool { - let spath = match path.to_str() { - Some(path) => path, - None => return false, - }; - if spath.ends_with('~') || spath.ends_with(".pyc") { - return false; - } - - const LLVM_PROJECTS: &[&str] = &[ - "llvm-project/clang", - "llvm-project\\clang", - "llvm-project/libunwind", - "llvm-project\\libunwind", - "llvm-project/lld", - "llvm-project\\lld", - "llvm-project/lldb", - "llvm-project\\lldb", - "llvm-project/llvm", - "llvm-project\\llvm", - "llvm-project/compiler-rt", - "llvm-project\\compiler-rt", - "llvm-project/cmake", - "llvm-project\\cmake", - "llvm-project/runtimes", - "llvm-project\\runtimes", - ]; - if spath.contains("llvm-project") - && !spath.ends_with("llvm-project") - && !LLVM_PROJECTS.iter().any(|path| spath.contains(path)) - { - return false; - } - - const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"]; - if LLVM_TEST.iter().any(|path| spath.contains(path)) - && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s")) - { - return false; - } - - let full_path = Path::new(dir).join(path); - if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) { - return false; - } - - let excludes = [ - "CVS", - "RCS", - "SCCS", - ".git", - ".gitignore", - ".gitmodules", - ".gitattributes", - ".cvsignore", - ".svn", - ".arch-ids", - "{arch}", - "=RELEASE-ID", - "=meta-update", - "=update", - ".bzr", - ".bzrignore", - ".bzrtags", - ".hg", - ".hgignore", - ".hgrags", - "_darcs", - ]; - !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s)) - } - - // Copy the directories using our filter - for item in src_dirs { - let dst = &dst_dir.join(item); - t!(fs::create_dir_all(dst)); - builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path)); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Src; - -impl Step for Src { - /// The output path of the src installer tarball - type Output = GeneratedTarball; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-src") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Src); - } - - /// Creates the `rust-src` installer component - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - if !builder.config.dry_run() { - builder.update_submodule(&Path::new("src/llvm-project")); - } - - let tarball = Tarball::new_targetless(builder, "rust-src"); - - // A lot of tools expect the rust-src component to be entirely in this directory, so if you - // change that (e.g. by adding another directory `lib/rustlib/src/foo` or - // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions - // and fix them... - // - // NOTE: if you update the paths here, you also should update the "virtual" path - // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` - let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); - - let src_files = ["Cargo.lock"]; - // This is the reduced set of paths which will become the rust-src component - // (essentially libstd and all of its path dependencies). - copy_src_dirs( - builder, - &builder.src, - &["library", "src/llvm-project/libunwind"], - &[ - // not needed and contains symlinks which rustup currently - // chokes on when unpacking. - "library/backtrace/crates", - // these are 30MB combined and aren't necessary for building - // the standard library. - "library/stdarch/Cargo.toml", - "library/stdarch/crates/stdarch-verify", - "library/stdarch/crates/intrinsic-test", - ], - &dst_src, - ); - for file in src_files.iter() { - builder.copy(&builder.src.join(file), &dst_src.join(file)); - } - - tarball.generate() - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct PlainSourceTarball; - -impl Step for PlainSourceTarball { - /// Produces the location of the tarball generated - type Output = GeneratedTarball; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.alias("rustc-src").default_condition(builder.config.rust_dist_src) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(PlainSourceTarball); - } - - /// Creates the plain source tarball - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which - // means neither rustup nor rustup-toolchain-install-master know how to download it. - // It also contains symbolic links, unlike other any other dist tarball. - // It's used for distros building rustc from source in a pre-vendored environment. - let mut tarball = Tarball::new(builder, "rustc", "src"); - tarball.permit_symlinks(true); - let plain_dst_src = tarball.image_dir(); - - // This is the set of root paths which will become part of the source package - let src_files = [ - "COPYRIGHT", - "LICENSE-APACHE", - "LICENSE-MIT", - "CONTRIBUTING.md", - "README.md", - "RELEASES.md", - "configure", - "x.py", - "config.example.toml", - "Cargo.toml", - "Cargo.lock", - ".gitmodules", - ]; - let src_dirs = ["src", "compiler", "library", "tests"]; - - copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src); - - // Copy the files normally - for item in &src_files { - builder.copy(&builder.src.join(item), &plain_dst_src.join(item)); - } - - // Create the version file - builder.create(&plain_dst_src.join("version"), &builder.rust_version()); - if let Some(info) = builder.rust_info().info() { - channel::write_commit_hash_file(&plain_dst_src, &info.sha); - channel::write_commit_info_file(&plain_dst_src, info); - } - - // If we're building from git sources, we need to vendor a complete distribution. - if builder.rust_info().is_managed_git_subrepository() { - // Ensure we have the submodules checked out. - builder.update_submodule(Path::new("src/tools/cargo")); - builder.update_submodule(Path::new("src/tools/rust-analyzer")); - - // Vendor all Cargo dependencies - let mut cmd = Command::new(&builder.initial_cargo); - cmd.arg("vendor") - .arg("--sync") - .arg(builder.src.join("./src/tools/cargo/Cargo.toml")) - .arg("--sync") - .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml")) - .arg("--sync") - .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml")) - .arg("--sync") - .arg(builder.src.join("./src/bootstrap/Cargo.toml")) - // Will read the libstd Cargo.toml - // which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .current_dir(&plain_dst_src); - - let config = if !builder.config.dry_run() { - t!(String::from_utf8(t!(cmd.output()).stdout)) - } else { - String::new() - }; - - let cargo_config_dir = plain_dst_src.join(".cargo"); - builder.create_dir(&cargo_config_dir); - builder.create(&cargo_config_dir.join("config.toml"), &config); - } - - tarball.bare() - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Cargo { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Cargo { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "cargo"); - run.alias("cargo").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let cargo = builder.ensure(tool::Cargo { compiler, target }); - let src = builder.src.join("src/tools/cargo"); - let etc = src.join("src/etc"); - - // Prepare the image directory - let mut tarball = Tarball::new(builder, "cargo", &target.triple); - tarball.set_overlay(OverlayKind::Cargo); - - tarball.add_file(&cargo, "bin", 0o755); - tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); - tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); - tarball.add_dir(etc.join("man"), "share/man/man1"); - tarball.add_legal_and_readme_to("share/doc/cargo"); - - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Rls { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Rls { - type Output = Option; - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "rls"); - run.alias("rls").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rls { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let rls = builder - .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) - .expect("rls expected to build"); - - let mut tarball = Tarball::new(builder, "rls", &target.triple); - tarball.set_overlay(OverlayKind::RLS); - tarball.is_preview(true); - tarball.add_file(rls, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rls"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustAnalyzer { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustAnalyzer { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "rust-analyzer"); - run.alias("rust-analyzer").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzer { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let rust_analyzer = builder - .ensure(tool::RustAnalyzer { compiler, target }) - .expect("rust-analyzer always builds"); - - let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); - tarball.set_overlay(OverlayKind::RustAnalyzer); - tarball.is_preview(true); - tarball.add_file(rust_analyzer, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Clippy { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Clippy { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "clippy"); - run.alias("clippy").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Clippy { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - // Prepare the image directory - // We expect clippy to build, because we've exited this step above if tool - // state for clippy isn't testing. - let clippy = builder - .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() }) - .expect("clippy expected to build - essential tool"); - let cargoclippy = builder - .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) - .expect("clippy expected to build - essential tool"); - - let mut tarball = Tarball::new(builder, "clippy", &target.triple); - tarball.set_overlay(OverlayKind::Clippy); - tarball.is_preview(true); - tarball.add_file(clippy, "bin", 0o755); - tarball.add_file(cargoclippy, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/clippy"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Miri { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Miri { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "miri"); - run.alias("miri").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - // This prevents miri from being built for "dist" or "install" - // on the stable/beta channels. It is a nightly-only tool and should - // not be included. - if !builder.build.unstable_features() { - return None; - } - let compiler = self.compiler; - let target = self.target; - - let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?; - let cargomiri = - builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?; - - let mut tarball = Tarball::new(builder, "miri", &target.triple); - tarball.set_overlay(OverlayKind::Miri); - tarball.is_preview(true); - tarball.add_file(miri, "bin", 0o755); - tarball.add_file(cargomiri, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/miri"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Rustfmt { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Rustfmt { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "rustfmt"); - run.alias("rustfmt").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustfmt { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - let rustfmt = builder - .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }) - .expect("rustfmt expected to build - essential tool"); - let cargofmt = builder - .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() }) - .expect("cargo fmt expected to build - essential tool"); - let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); - tarball.set_overlay(OverlayKind::Rustfmt); - tarball.is_preview(true); - tarball.add_file(rustfmt, "bin", 0o755); - tarball.add_file(cargofmt, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rustfmt"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustDemangler { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustDemangler { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // While other tools use `should_build_extended_tool` to decide whether to be run by - // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like - // the other ones or if `profiler = true`. Because we don't know the target at this stage - // we run the step by default when only `extended = true`, and decide whether to actually - // run it or not later. - let default = run.builder.config.extended; - run.alias("rust-demangler").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustDemangler { - compiler: run.builder.compiler_for( - run.builder.top_stage, - run.builder.config.build, - run.target, - ), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - - // Only build this extended tool if explicitly included in `tools`, or if `profiler = true` - let condition = should_build_extended_tool(builder, "rust-demangler") - || builder.config.profiler_enabled(target); - if builder.config.extended && !condition { - return None; - } - - let rust_demangler = builder - .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }) - .expect("rust-demangler expected to build - in-tree tool"); - - // Prepare the image directory - let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple); - tarball.set_overlay(OverlayKind::RustDemangler); - tarball.is_preview(true); - tarball.add_file(&rust_demangler, "bin", 0o755); - tarball.add_legal_and_readme_to("share/doc/rust-demangler"); - Some(tarball.generate()) - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Extended { - stage: u32, - host: TargetSelection, - target: TargetSelection, -} - -impl Step for Extended { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.alias("extended").default_condition(builder.config.extended) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Extended { - stage: run.builder.top_stage, - host: run.builder.config.build, - target: run.target, - }); - } - - /// Creates a combined installer for the specified target in the provided stage. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let stage = self.stage; - let compiler = builder.compiler_for(self.stage, self.host, self.target); - - builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target)); - - let mut tarballs = Vec::new(); - let mut built_tools = HashSet::new(); - macro_rules! add_component { - ($name:expr => $step:expr) => { - if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) { - tarballs.push(tarball); - built_tools.insert($name); - } - }; - } - - // When rust-std package split from rustc, we needed to ensure that during - // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering - // the std files during uninstall. To do this ensure that rustc comes - // before rust-std in the list below. - tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) })); - tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std")); - - if target.ends_with("windows-gnu") { - tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw")); - } - - add_component!("rust-docs" => Docs { host: target }); - add_component!("rust-json-docs" => JsonDocs { host: target }); - add_component!("rust-demangler"=> RustDemangler { compiler, target }); - add_component!("cargo" => Cargo { compiler, target }); - add_component!("rustfmt" => Rustfmt { compiler, target }); - add_component!("rls" => Rls { compiler, target }); - add_component!("rust-analyzer" => RustAnalyzer { compiler, target }); - add_component!("llvm-components" => LlvmTools { target }); - add_component!("clippy" => Clippy { compiler, target }); - add_component!("miri" => Miri { compiler, target }); - add_component!("analysis" => Analysis { compiler, target }); - - let etc = builder.src.join("src/etc/installer"); - - // Avoid producing tarballs during a dry run. - if builder.config.dry_run() { - return; - } - - let tarball = Tarball::new(builder, "rust", &target.triple); - let generated = tarball.combine(&tarballs); - - let tmp = tmpdir(builder).join("combined-tarball"); - let work = generated.work_dir(); - - let mut license = String::new(); - license += &builder.read(&builder.src.join("COPYRIGHT")); - license += &builder.read(&builder.src.join("LICENSE-APACHE")); - license += &builder.read(&builder.src.join("LICENSE-MIT")); - license.push('\n'); - license.push('\n'); - - let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18"; - let mut rtf = rtf.to_string(); - rtf.push('\n'); - for line in license.lines() { - rtf.push_str(line); - rtf.push_str("\\line "); - } - rtf.push('}'); - - fn filter(contents: &str, marker: &str) -> String { - let start = format!("tool-{marker}-start"); - let end = format!("tool-{marker}-end"); - let mut lines = Vec::new(); - let mut omitted = false; - for line in contents.lines() { - if line.contains(&start) { - omitted = true; - } else if line.contains(&end) { - omitted = false; - } else if !omitted { - lines.push(line); - } - } - - lines.join("\n") - } - - let xform = |p: &Path| { - let mut contents = t!(fs::read_to_string(p)); - for tool in &["rust-demangler", "miri", "rust-docs"] { - if !built_tools.contains(tool) { - contents = filter(&contents, tool); - } - } - let ret = tmp.join(p.file_name().unwrap()); - t!(fs::write(&ret, &contents)); - ret - }; - - if target.contains("apple-darwin") { - builder.info("building pkg installer"); - let pkg = tmp.join("pkg"); - let _ = fs::remove_dir_all(&pkg); - - let pkgbuild = |component: &str| { - let mut cmd = Command::new("pkgbuild"); - cmd.arg("--identifier") - .arg(format!("org.rust-lang.{}", component)) - .arg("--scripts") - .arg(pkg.join(component)) - .arg("--nopayload") - .arg(pkg.join(component).with_extension("pkg")); - builder.run(&mut cmd); - }; - - let prepare = |name: &str| { - builder.create_dir(&pkg.join(name)); - builder.cp_r( - &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)), - &pkg.join(name), - ); - builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755); - pkgbuild(name); - }; - prepare("rustc"); - prepare("cargo"); - prepare("rust-std"); - prepare("rust-analysis"); - prepare("clippy"); - prepare("rust-analyzer"); - for tool in &["rust-docs", "rust-demangler", "miri"] { - if built_tools.contains(tool) { - prepare(tool); - } - } - // create an 'uninstall' package - builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755); - pkgbuild("uninstall"); - - builder.create_dir(&pkg.join("res")); - builder.create(&pkg.join("res/LICENSE.txt"), &license); - builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644); - let mut cmd = Command::new("productbuild"); - cmd.arg("--distribution") - .arg(xform(&etc.join("pkg/Distribution.xml"))) - .arg("--resources") - .arg(pkg.join("res")) - .arg(distdir(builder).join(format!( - "{}-{}.pkg", - pkgname(builder, "rust"), - target.triple - ))) - .arg("--package-path") - .arg(&pkg); - let _time = timeit(builder); - builder.run(&mut cmd); - } - - if target.contains("windows") { - let exe = tmp.join("exe"); - let _ = fs::remove_dir_all(&exe); - - let prepare = |name: &str| { - builder.create_dir(&exe.join(name)); - let dir = if name == "rust-std" || name == "rust-analysis" { - format!("{}-{}", name, target.triple) - } else if name == "rust-analyzer" { - "rust-analyzer-preview".to_string() - } else if name == "clippy" { - "clippy-preview".to_string() - } else if name == "rust-demangler" { - "rust-demangler-preview".to_string() - } else if name == "miri" { - "miri-preview".to_string() - } else { - name.to_string() - }; - builder.cp_r( - &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir), - &exe.join(name), - ); - builder.remove(&exe.join(name).join("manifest.in")); - }; - prepare("rustc"); - prepare("cargo"); - prepare("rust-analysis"); - prepare("rust-std"); - for tool in &["clippy", "rust-analyzer", "rust-docs", "rust-demangler", "miri"] { - if built_tools.contains(tool) { - prepare(tool); - } - } - if target.ends_with("windows-gnu") { - prepare("rust-mingw"); - } - - builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644); - - // Generate msi installer - let wix_path = env::var_os("WIX") - .expect("`WIX` environment variable must be set for generating MSI installer(s)."); - let wix = PathBuf::from(wix_path); - let heat = wix.join("bin/heat.exe"); - let candle = wix.join("bin/candle.exe"); - let light = wix.join("bin/light.exe"); - - let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"]; - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rustc") - .args(&heat_flags) - .arg("-cg") - .arg("RustcGroup") - .arg("-dr") - .arg("Rustc") - .arg("-var") - .arg("var.RustcDir") - .arg("-out") - .arg(exe.join("RustcGroup.wxs")), - ); - if built_tools.contains("rust-docs") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-docs") - .args(&heat_flags) - .arg("-cg") - .arg("DocsGroup") - .arg("-dr") - .arg("Docs") - .arg("-var") - .arg("var.DocsDir") - .arg("-out") - .arg(exe.join("DocsGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/squash-components.xsl")), - ); - } - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("cargo") - .args(&heat_flags) - .arg("-cg") - .arg("CargoGroup") - .arg("-dr") - .arg("Cargo") - .arg("-var") - .arg("var.CargoDir") - .arg("-out") - .arg(exe.join("CargoGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-std") - .args(&heat_flags) - .arg("-cg") - .arg("StdGroup") - .arg("-dr") - .arg("Std") - .arg("-var") - .arg("var.StdDir") - .arg("-out") - .arg(exe.join("StdGroup.wxs")), - ); - if built_tools.contains("rust-analyzer") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-analyzer") - .args(&heat_flags) - .arg("-cg") - .arg("RustAnalyzerGroup") - .arg("-dr") - .arg("RustAnalyzer") - .arg("-var") - .arg("var.RustAnalyzerDir") - .arg("-out") - .arg(exe.join("RustAnalyzerGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - } - if built_tools.contains("clippy") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("clippy") - .args(&heat_flags) - .arg("-cg") - .arg("ClippyGroup") - .arg("-dr") - .arg("Clippy") - .arg("-var") - .arg("var.ClippyDir") - .arg("-out") - .arg(exe.join("ClippyGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - } - if built_tools.contains("rust-demangler") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-demangler") - .args(&heat_flags) - .arg("-cg") - .arg("RustDemanglerGroup") - .arg("-dr") - .arg("RustDemangler") - .arg("-var") - .arg("var.RustDemanglerDir") - .arg("-out") - .arg(exe.join("RustDemanglerGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - } - if built_tools.contains("miri") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("miri") - .args(&heat_flags) - .arg("-cg") - .arg("MiriGroup") - .arg("-dr") - .arg("Miri") - .arg("-var") - .arg("var.MiriDir") - .arg("-out") - .arg(exe.join("MiriGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - } - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-analysis") - .args(&heat_flags) - .arg("-cg") - .arg("AnalysisGroup") - .arg("-dr") - .arg("Analysis") - .arg("-var") - .arg("var.AnalysisDir") - .arg("-out") - .arg(exe.join("AnalysisGroup.wxs")) - .arg("-t") - .arg(etc.join("msi/remove-duplicates.xsl")), - ); - if target.ends_with("windows-gnu") { - builder.run( - Command::new(&heat) - .current_dir(&exe) - .arg("dir") - .arg("rust-mingw") - .args(&heat_flags) - .arg("-cg") - .arg("GccGroup") - .arg("-dr") - .arg("Gcc") - .arg("-var") - .arg("var.GccDir") - .arg("-out") - .arg(exe.join("GccGroup.wxs")), - ); - } - - let candle = |input: &Path| { - let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj"); - let arch = if target.contains("x86_64") { "x64" } else { "x86" }; - let mut cmd = Command::new(&candle); - cmd.current_dir(&exe) - .arg("-nologo") - .arg("-dRustcDir=rustc") - .arg("-dCargoDir=cargo") - .arg("-dStdDir=rust-std") - .arg("-dAnalysisDir=rust-analysis") - .arg("-arch") - .arg(&arch) - .arg("-out") - .arg(&output) - .arg(&input); - add_env(builder, &mut cmd, target); - - if built_tools.contains("clippy") { - cmd.arg("-dClippyDir=clippy"); - } - if built_tools.contains("rust-docs") { - cmd.arg("-dDocsDir=rust-docs"); - } - if built_tools.contains("rust-demangler") { - cmd.arg("-dRustDemanglerDir=rust-demangler"); - } - if built_tools.contains("rust-analyzer") { - cmd.arg("-dRustAnalyzerDir=rust-analyzer"); - } - if built_tools.contains("miri") { - cmd.arg("-dMiriDir=miri"); - } - if target.ends_with("windows-gnu") { - cmd.arg("-dGccDir=rust-mingw"); - } - builder.run(&mut cmd); - }; - candle(&xform(&etc.join("msi/rust.wxs"))); - candle(&etc.join("msi/ui.wxs")); - candle(&etc.join("msi/rustwelcomedlg.wxs")); - candle("RustcGroup.wxs".as_ref()); - if built_tools.contains("rust-docs") { - candle("DocsGroup.wxs".as_ref()); - } - candle("CargoGroup.wxs".as_ref()); - candle("StdGroup.wxs".as_ref()); - if built_tools.contains("clippy") { - candle("ClippyGroup.wxs".as_ref()); - } - if built_tools.contains("miri") { - candle("MiriGroup.wxs".as_ref()); - } - if built_tools.contains("rust-demangler") { - candle("RustDemanglerGroup.wxs".as_ref()); - } - if built_tools.contains("rust-analyzer") { - candle("RustAnalyzerGroup.wxs".as_ref()); - } - candle("AnalysisGroup.wxs".as_ref()); - - if target.ends_with("windows-gnu") { - candle("GccGroup.wxs".as_ref()); - } - - builder.create(&exe.join("LICENSE.rtf"), &rtf); - builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644); - builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644); - - builder.info(&format!("building `msi` installer with {light:?}")); - let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple); - let mut cmd = Command::new(&light); - cmd.arg("-nologo") - .arg("-ext") - .arg("WixUIExtension") - .arg("-ext") - .arg("WixUtilExtension") - .arg("-out") - .arg(exe.join(&filename)) - .arg("rust.wixobj") - .arg("ui.wixobj") - .arg("rustwelcomedlg.wixobj") - .arg("RustcGroup.wixobj") - .arg("CargoGroup.wixobj") - .arg("StdGroup.wixobj") - .arg("AnalysisGroup.wixobj") - .current_dir(&exe); - - if built_tools.contains("clippy") { - cmd.arg("ClippyGroup.wixobj"); - } - if built_tools.contains("miri") { - cmd.arg("MiriGroup.wixobj"); - } - if built_tools.contains("rust-analyzer") { - cmd.arg("RustAnalyzerGroup.wixobj"); - } - if built_tools.contains("rust-demangler") { - cmd.arg("RustDemanglerGroup.wixobj"); - } - if built_tools.contains("rust-docs") { - cmd.arg("DocsGroup.wixobj"); - } - - if target.ends_with("windows-gnu") { - cmd.arg("GccGroup.wixobj"); - } - // ICE57 wrongly complains about the shortcuts - cmd.arg("-sice:ICE57"); - - let _time = timeit(builder); - builder.run(&mut cmd); - - if !builder.config.dry_run() { - t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename))); - } - } - } -} - -fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) { - let mut parts = builder.version.split('.'); - cmd.env("CFG_RELEASE_INFO", builder.rust_version()) - .env("CFG_RELEASE_NUM", &builder.version) - .env("CFG_RELEASE", builder.rust_release()) - .env("CFG_VER_MAJOR", parts.next().unwrap()) - .env("CFG_VER_MINOR", parts.next().unwrap()) - .env("CFG_VER_PATCH", parts.next().unwrap()) - .env("CFG_VER_BUILD", "0") // just needed to build - .env("CFG_PACKAGE_VERS", builder.rust_package_vers()) - .env("CFG_PACKAGE_NAME", pkgname(builder, "rust")) - .env("CFG_BUILD", target.triple) - .env("CFG_CHANNEL", &builder.config.channel); - - if target.contains("windows-gnullvm") { - cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM"); - } else if target.contains("windows-gnu") { - cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU"); - } else { - cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC"); - } -} - -fn install_llvm_file(builder: &Builder<'_>, source: &Path, destination: &Path) { - if builder.config.dry_run() { - return; - } - - builder.install(&source, destination, 0o644); -} - -/// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking. -/// -/// Returns whether the files were actually copied. -fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool { - if let Some(config) = builder.config.target_config.get(&target) { - if config.llvm_config.is_some() && !builder.config.llvm_from_ci { - // If the LLVM was externally provided, then we don't currently copy - // artifacts into the sysroot. This is not necessarily the right - // choice (in particular, it will require the LLVM dylib to be in - // the linker's load path at runtime), but the common use case for - // external LLVMs is distribution provided LLVMs, and in that case - // they're usually in the standard search path (e.g., /usr/lib) and - // copying them here is going to cause problems as we may end up - // with the wrong files and isn't what distributions want. - // - // This behavior may be revisited in the future though. - // - // If the LLVM is coming from ourselves (just from CI) though, we - // still want to install it, as it otherwise won't be available. - return false; - } - } - - // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib - // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely - // clear why this is the case, though. llvm-config will emit the versioned - // paths and we don't want those in the sysroot (as we're expecting - // unversioned paths). - if target.contains("apple-darwin") && builder.llvm_link_shared() { - let src_libdir = builder.llvm_out(target).join("lib"); - let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); - if llvm_dylib_path.exists() { - builder.install(&llvm_dylib_path, dst_libdir, 0o644); - } - !builder.config.dry_run() - } else if let Ok(llvm::LlvmResult { llvm_config, .. }) = - llvm::prebuilt_llvm_config(builder, target) - { - let mut cmd = Command::new(llvm_config); - cmd.arg("--libfiles"); - builder.verbose(&format!("running {cmd:?}")); - let files = if builder.config.dry_run() { "".into() } else { output(&mut cmd) }; - let build_llvm_out = &builder.llvm_out(builder.config.build); - let target_llvm_out = &builder.llvm_out(target); - for file in files.trim_end().split(' ') { - // If we're not using a custom LLVM, make sure we package for the target. - let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) { - target_llvm_out.join(relative_path) - } else { - PathBuf::from(file) - }; - install_llvm_file(builder, &file, dst_libdir); - } - !builder.config.dry_run() - } else { - false - } -} - -/// Maybe add libLLVM.so to the target lib-dir for linking. -pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { - let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib"); - // We do not need to copy LLVM files into the sysroot if it is not - // dynamically linked; it is already included into librustc_llvm - // statically. - if builder.llvm_link_shared() { - maybe_install_llvm(builder, target, &dst_libdir); - } -} - -/// Maybe add libLLVM.so to the runtime lib-dir for rustc itself. -pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { - let dst_libdir = - sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target })); - // We do not need to copy LLVM files into the sysroot if it is not - // dynamically linked; it is already included into librustc_llvm - // statically. - if builder.llvm_link_shared() { - maybe_install_llvm(builder, target, &dst_libdir); - } -} - -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct LlvmTools { - pub target: TargetSelection, -} - -impl Step for LlvmTools { - type Output = Option; - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = should_build_extended_tool(&run.builder, "llvm-tools"); - // FIXME: allow using the names of the tools themselves? - run.alias("llvm-tools").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(LlvmTools { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let target = self.target; - - /* run only if llvm-config isn't used */ - if let Some(config) = builder.config.target_config.get(&target) { - if let Some(ref _s) = config.llvm_config { - builder.info(&format!("Skipping LlvmTools ({target}): external LLVM")); - return None; - } - } - - builder.ensure(crate::llvm::Llvm { target }); - - let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); - tarball.set_overlay(OverlayKind::LLVM); - tarball.is_preview(true); - - // Prepare the image directory - let src_bindir = builder.llvm_out(target).join("bin"); - let dst_bindir = format!("lib/rustlib/{}/bin", target.triple); - for tool in LLVM_TOOLS { - let exe = src_bindir.join(exe(tool, target)); - tarball.add_file(&exe, &dst_bindir, 0o755); - } - - // Copy libLLVM.so to the target lib dir as well, so the RPATH like - // `$ORIGIN/../lib` can find it. It may also be used as a dependency - // of `rustc-dev` to support the inherited `-lLLVM` when using the - // compiler libraries. - maybe_install_llvm_target(builder, target, tarball.image_dir()); - - Some(tarball.generate()) - } -} - -// Tarball intended for internal consumption to ease rustc/std development. -// -// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct RustDev { - pub target: TargetSelection, -} - -impl Step for RustDev { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("rust-dev") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustDev { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let target = self.target; - - /* run only if llvm-config isn't used */ - if let Some(config) = builder.config.target_config.get(&target) { - if let Some(ref _s) = config.llvm_config { - builder.info(&format!("Skipping RustDev ({target}): external LLVM")); - return None; - } - } - - let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); - tarball.set_overlay(OverlayKind::LLVM); - - builder.ensure(crate::llvm::Llvm { target }); - - // We want to package `lld` to use it with `download-ci-llvm`. - builder.ensure(crate::llvm::Lld { target }); - - let src_bindir = builder.llvm_out(target).join("bin"); - // If updating this list, you likely want to change - // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users - // will not pick up the extra file until LLVM gets bumped. - for bin in &[ - "llvm-config", - "llvm-ar", - "llvm-objdump", - "llvm-profdata", - "llvm-bcanalyzer", - "llvm-cov", - "llvm-dwp", - "llvm-nm", - "llvm-dwarfdump", - "llvm-dis", - "llvm-tblgen", - ] { - tarball.add_file(src_bindir.join(exe(bin, target)), "bin", 0o755); - } - - // We don't build LLD on some platforms, so only add it if it exists - let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target)); - if lld_path.exists() { - tarball.add_file(lld_path, "bin", 0o755); - } - - tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755); - - // Copy the include directory as well; needed mostly to build - // librustc_llvm properly (e.g., llvm-config.h is in here). But also - // just broadly useful to be able to link against the bundled LLVM. - tarball.add_dir(&builder.llvm_out(target).join("include"), "include"); - - // Copy libLLVM.so to the target lib dir as well, so the RPATH like - // `$ORIGIN/../lib` can find it. It may also be used as a dependency - // of `rustc-dev` to support the inherited `-lLLVM` when using the - // compiler libraries. - let dst_libdir = tarball.image_dir().join("lib"); - maybe_install_llvm(builder, target, &dst_libdir); - let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" }; - t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir); - - Some(tarball.generate()) - } -} - -// Tarball intended for internal consumption to ease rustc/std development. -// -// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct Bootstrap { - pub target: TargetSelection, -} - -impl Step for Bootstrap { - type Output = Option; - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("bootstrap") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Bootstrap { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let target = self.target; - - let tarball = Tarball::new(builder, "bootstrap", &target.triple); - - let bootstrap_outdir = &builder.bootstrap_out; - for file in &["bootstrap", "rustc", "rustdoc", "sccache-plus-cl"] { - tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755); - } - - Some(tarball.generate()) - } -} - -/// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the -/// release process to avoid cloning the monorepo and building stuff. -/// -/// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct BuildManifest { - pub target: TargetSelection, -} - -impl Step for BuildManifest { - type Output = GeneratedTarball; - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("build-manifest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(BuildManifest { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> GeneratedTarball { - let build_manifest = builder.tool_exe(Tool::BuildManifest); - - let tarball = Tarball::new(builder, "build-manifest", &self.target.triple); - tarball.add_file(&build_manifest, "bin", 0o755); - tarball.generate() - } -} - -/// Tarball containing artifacts necessary to reproduce the build of rustc. -/// -/// Currently this is the PGO profile data. -/// -/// Should not be considered stable by end users. -#[derive(Clone, Debug, Eq, Hash, PartialEq)] -pub struct ReproducibleArtifacts { - pub target: TargetSelection, -} - -impl Step for ReproducibleArtifacts { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("reproducible-artifacts") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ReproducibleArtifacts { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut added_anything = false; - let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple); - if let Some(path) = builder.config.rust_profile_use.as_ref() { - tarball.add_file(path, ".", 0o644); - added_anything = true; - } - if let Some(path) = builder.config.llvm_profile_use.as_ref() { - tarball.add_file(path, ".", 0o644); - added_anything = true; - } - for profile in &builder.config.reproducible_artifacts { - tarball.add_file(profile, ".", 0o644); - added_anything = true; - } - if added_anything { Some(tarball.generate()) } else { None } - } -} diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs deleted file mode 100644 index 505f06ed1..000000000 --- a/src/bootstrap/doc.rs +++ /dev/null @@ -1,1085 +0,0 @@ -//! Documentation generation for rustbuilder. -//! -//! This module implements generation for all bits and pieces of documentation -//! for the Rust project. This notably includes suites like the rust book, the -//! nomicon, rust by example, standalone documentation, etc. -//! -//! Everything here is basically just a shim around calling either `rustbook` or -//! `rustdoc`. - -use std::fs; -use std::path::{Path, PathBuf}; - -use crate::builder::crate_description; -use crate::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::{Interned, INTERNER}; -use crate::compile; -use crate::config::{Config, TargetSelection}; -use crate::tool::{self, prepare_tool_cargo, SourceType, Tool}; -use crate::util::{dir_is_empty, symlink_dir, t, up_to_date}; -use crate::Mode; - -macro_rules! submodule_helper { - ($path:expr, submodule) => { - $path - }; - ($path:expr, submodule = $submodule:literal) => { - $submodule - }; -} - -macro_rules! book { - ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => { - $( - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - pub struct $name { - target: TargetSelection, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path($path).default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) { - $( - let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? )); - builder.update_submodule(&path); - )? - builder.ensure(RustbookSrc { - target: self.target, - name: INTERNER.intern_str($book_name), - src: INTERNER.intern_path(builder.src.join($path)), - parent: Some(self), - }) - } - } - )+ - } -} - -// NOTE: When adding a book here, make sure to ALSO build the book by -// adding a build step in `src/bootstrap/builder.rs`! -// NOTE: Make sure to add the corresponding submodule when adding a new book. -// FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules -// and checking against it?). -book!( - CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo"; - ClippyBook, "src/tools/clippy/book", "clippy"; - EditionGuide, "src/doc/edition-guide", "edition-guide", submodule; - EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule; - Nomicon, "src/doc/nomicon", "nomicon", submodule; - Reference, "src/doc/reference", "reference", submodule; - RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule; - RustdocBook, "src/doc/rustdoc", "rustdoc"; - StyleGuide, "src/doc/style-guide", "style-guide"; -); - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct UnstableBook { - target: TargetSelection, -} - -impl Step for UnstableBook { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/unstable-book").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(UnstableBook { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(UnstableBookGen { target: self.target }); - builder.ensure(RustbookSrc { - target: self.target, - name: INTERNER.intern_str("unstable-book"), - src: INTERNER.intern_path(builder.md_doc_out(self.target).join("unstable-book")), - parent: Some(self), - }) - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -struct RustbookSrc { - target: TargetSelection, - name: Interned, - src: Interned, - parent: Option

, -} - -impl Step for RustbookSrc

{ - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path. - /// - /// This will not actually generate any documentation if the documentation has - /// already been generated. - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let name = self.name; - let src = self.src; - let out = builder.doc_out(target); - t!(fs::create_dir_all(&out)); - - let out = out.join(name); - let index = out.join("index.html"); - let rustbook = builder.tool_exe(Tool::Rustbook); - let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - - if !builder.config.dry_run() && !(up_to_date(&src, &index) || up_to_date(&rustbook, &index)) - { - builder.info(&format!("Rustbook ({target}) - {name}")); - let _ = fs::remove_dir_all(&out); - - builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(out)); - } - - if self.parent.is_some() { - builder.maybe_open_in_browser::

(index) - } - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct TheBook { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for TheBook { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/book").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(TheBook { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Builds the book and associated stuff. - /// - /// We need to build: - /// - /// * Book - /// * Older edition redirects - /// * Version info and CSS - /// * Index page - /// * Redirect pages - fn run(self, builder: &Builder<'_>) { - let relative_path = Path::new("src").join("doc").join("book"); - builder.update_submodule(&relative_path); - - let compiler = self.compiler; - let target = self.target; - - let absolute_path = builder.src.join(&relative_path); - let redirect_path = absolute_path.join("redirects"); - if !absolute_path.exists() - || !redirect_path.exists() - || dir_is_empty(&absolute_path) - || dir_is_empty(&redirect_path) - { - eprintln!("Please checkout submodule: {}", relative_path.display()); - crate::exit!(1); - } - // build book - builder.ensure(RustbookSrc { - target, - name: INTERNER.intern_str("book"), - src: INTERNER.intern_path(absolute_path.clone()), - parent: Some(self), - }); - - // building older edition redirects - for edition in &["first-edition", "second-edition", "2018-edition"] { - builder.ensure(RustbookSrc { - target, - name: INTERNER.intern_string(format!("book/{edition}")), - src: INTERNER.intern_path(absolute_path.join(edition)), - // There should only be one book that is marked as the parent for each target, so - // treat the other editions as not having a parent. - parent: Option::::None, - }); - } - - // build the version info page and CSS - let shared_assets = builder.ensure(SharedAssets { target }); - - // build the command first so we don't nest GHA groups - builder.rustdoc_cmd(compiler); - - // build the redirect pages - let _guard = builder.msg_doc(compiler, "book redirect pages", target); - for file in t!(fs::read_dir(redirect_path)) { - let file = t!(file); - let path = file.path(); - let path = path.to_str().unwrap(); - - invoke_rustdoc(builder, compiler, &shared_assets, target, path); - } - } -} - -fn invoke_rustdoc( - builder: &Builder<'_>, - compiler: Compiler, - shared_assets: &SharedAssetsPaths, - target: TargetSelection, - markdown: &str, -) { - let out = builder.doc_out(target); - - let path = builder.src.join("src/doc").join(markdown); - - let header = builder.src.join("src/doc/redirect.inc"); - let footer = builder.src.join("src/doc/footer.inc"); - - let mut cmd = builder.rustdoc_cmd(compiler); - - let out = out.join("book"); - - cmd.arg("--html-after-content") - .arg(&footer) - .arg("--html-before-content") - .arg(&shared_assets.version_info) - .arg("--html-in-header") - .arg(&header) - .arg("--markdown-no-toc") - .arg("--markdown-playground-url") - .arg("https://play.rust-lang.org/") - .arg("-o") - .arg(&out) - .arg(&path) - .arg("--markdown-css") - .arg("../rust.css"); - - if !builder.config.docs_minification { - cmd.arg("-Z").arg("unstable-options").arg("--disable-minification"); - } - - builder.run(&mut cmd); -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Standalone { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for Standalone { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc").alias("standalone").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Standalone { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Generates all standalone documentation as compiled by the rustdoc in `stage` - /// for the `target` into `out`. - /// - /// This will list all of `src/doc` looking for markdown files and appropriately - /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and - /// `STAMP` along with providing the various header/footer HTML we've customized. - /// - /// In the end, this is just a glorified wrapper around rustdoc! - fn run(self, builder: &Builder<'_>) { - let target = self.target; - let compiler = self.compiler; - let _guard = builder.msg_doc(compiler, "standalone", target); - let out = builder.doc_out(target); - t!(fs::create_dir_all(&out)); - - let version_info = builder.ensure(SharedAssets { target: self.target }).version_info; - - let favicon = builder.src.join("src/doc/favicon.inc"); - let footer = builder.src.join("src/doc/footer.inc"); - let full_toc = builder.src.join("src/doc/full-toc.inc"); - - for file in t!(fs::read_dir(builder.src.join("src/doc"))) { - let file = t!(file); - let path = file.path(); - let filename = path.file_name().unwrap().to_str().unwrap(); - if !filename.ends_with(".md") || filename == "README.md" { - continue; - } - - let html = out.join(filename).with_extension("html"); - let rustdoc = builder.rustdoc(compiler); - if up_to_date(&path, &html) - && up_to_date(&footer, &html) - && up_to_date(&favicon, &html) - && up_to_date(&full_toc, &html) - && (builder.config.dry_run() || up_to_date(&version_info, &html)) - && (builder.config.dry_run() || up_to_date(&rustdoc, &html)) - { - continue; - } - - let mut cmd = builder.rustdoc_cmd(compiler); - // Needed for --index-page flag - cmd.arg("-Z").arg("unstable-options"); - - cmd.arg("--html-after-content") - .arg(&footer) - .arg("--html-before-content") - .arg(&version_info) - .arg("--html-in-header") - .arg(&favicon) - .arg("--markdown-no-toc") - .arg("--index-page") - .arg(&builder.src.join("src/doc/index.md")) - .arg("--markdown-playground-url") - .arg("https://play.rust-lang.org/") - .arg("-o") - .arg(&out) - .arg(&path); - - if !builder.config.docs_minification { - cmd.arg("--disable-minification"); - } - - if filename == "not_found.md" { - cmd.arg("--markdown-css").arg("https://doc.rust-lang.org/rust.css"); - } else { - cmd.arg("--markdown-css").arg("rust.css"); - } - builder.run(&mut cmd); - } - - // We open doc/index.html as the default if invoked as `x.py doc --open` - // with no particular explicit doc requested (e.g. library/core). - if builder.paths.is_empty() || builder.was_invoked_explicitly::(Kind::Doc) { - let index = out.join("index.html"); - builder.open_in_browser(&index); - } - } -} - -#[derive(Debug, Clone)] -pub struct SharedAssetsPaths { - pub version_info: PathBuf, -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct SharedAssets { - target: TargetSelection, -} - -impl Step for SharedAssets { - type Output = SharedAssetsPaths; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - // Other tasks depend on this, no need to execute it on its own - run.never() - } - - // Generate shared resources used by other pieces of documentation. - fn run(self, builder: &Builder<'_>) -> Self::Output { - let out = builder.doc_out(self.target); - - let version_input = builder.src.join("src").join("doc").join("version_info.html.template"); - let version_info = out.join("version_info.html"); - if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) { - let info = t!(fs::read_to_string(&version_input)) - .replace("VERSION", &builder.rust_release()) - .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or("")) - .replace("STAMP", builder.rust_info().sha().unwrap_or("")); - t!(fs::write(&version_info, &info)); - } - - builder.copy(&builder.src.join("src").join("doc").join("rust.css"), &out.join("rust.css")); - - SharedAssetsPaths { version_info } - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub struct Std { - pub stage: u32, - pub target: TargetSelection, - pub format: DocumentationFormat, - crates: Interned>, -} - -impl Std { - pub(crate) fn new( - stage: u32, - target: TargetSelection, - builder: &Builder<'_>, - format: DocumentationFormat, - ) -> Self { - let crates = builder - .in_tree_crates("sysroot", Some(target)) - .into_iter() - .map(|krate| krate.name.to_string()) - .collect(); - Std { stage, target, format, crates: INTERNER.intern_list(crates) } - } -} - -impl Step for Std { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.crate_or_deps("sysroot").path("library").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Std { - stage: run.builder.top_stage, - target: run.target, - format: if run.builder.config.cmd.json() { - DocumentationFormat::JSON - } else { - DocumentationFormat::HTML - }, - crates: run.make_run_crates(Alias::Library), - }); - } - - /// Compile all standard library documentation. - /// - /// This will generate all documentation for the standard library and its - /// dependencies. This is largely just a wrapper around `cargo doc`. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let target = self.target; - let out = match self.format { - DocumentationFormat::HTML => builder.doc_out(target), - DocumentationFormat::JSON => builder.json_doc_out(target), - }; - - t!(fs::create_dir_all(&out)); - - if self.format == DocumentationFormat::HTML { - builder.ensure(SharedAssets { target: self.target }); - } - - let index_page = builder - .src - .join("src/doc/index.md") - .into_os_string() - .into_string() - .expect("non-utf8 paths are unsupported"); - let mut extra_args = match self.format { - DocumentationFormat::HTML => { - vec!["--markdown-css", "rust.css", "--markdown-no-toc", "--index-page", &index_page] - } - DocumentationFormat::JSON => vec!["--output-format", "json"], - }; - - if !builder.config.docs_minification { - extra_args.push("--disable-minification"); - } - - doc_std(builder, self.format, stage, target, &out, &extra_args, &self.crates); - - // Don't open if the format is json - if let DocumentationFormat::JSON = self.format { - return; - } - - if builder.paths.iter().any(|path| path.ends_with("library")) { - // For `x.py doc library --open`, open `std` by default. - let index = out.join("std").join("index.html"); - builder.open_in_browser(index); - } else { - for requested_crate in &*self.crates { - if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) { - let index = out.join(requested_crate).join("index.html"); - builder.open_in_browser(index); - break; - } - } - } - } -} - -/// Name of the crates that are visible to consumers of the standard library. -/// Documentation for internal crates is handled by the rustc step, so internal crates will show -/// up there. -/// -/// Order here is important! -/// Crates need to be processed starting from the leaves, otherwise rustdoc will not -/// create correct links between crates because rustdoc depends on the -/// existence of the output directories to know if it should be a local -/// or remote link. -const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"]; - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] -pub enum DocumentationFormat { - HTML, - JSON, -} - -impl DocumentationFormat { - fn as_str(&self) -> &str { - match self { - DocumentationFormat::HTML => "HTML", - DocumentationFormat::JSON => "JSON", - } - } -} - -/// Build the documentation for public standard library crates. -fn doc_std( - builder: &Builder<'_>, - format: DocumentationFormat, - stage: u32, - target: TargetSelection, - out: &Path, - extra_args: &[&str], - requested_crates: &[String], -) { - if builder.no_std(target) == Some(true) { - panic!( - "building std documentation for no_std target {target} is not supported\n\ - Set `docs = false` in the config to disable documentation, or pass `--skip library`." - ); - } - - let compiler = builder.compiler(stage, builder.config.build); - - let target_doc_dir_name = if format == DocumentationFormat::JSON { "json-doc" } else { "doc" }; - let target_dir = - builder.stage_out(compiler, Mode::Std).join(target.triple).join(target_doc_dir_name); - - // This is directory where the compiler will place the output of the command. - // We will then copy the files from this directory into the final `out` directory, the specified - // as a function parameter. - let out_dir = target_dir.join(target.triple).join("doc"); - - let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "doc"); - compile::std_cargo(builder, target, compiler.stage, &mut cargo); - cargo - .arg("--no-deps") - .arg("--target-dir") - .arg(&*target_dir.to_string_lossy()) - .arg("-Zskip-rustdoc-fingerprint") - .rustdocflag("-Z") - .rustdocflag("unstable-options") - .rustdocflag("--resource-suffix") - .rustdocflag(&builder.version); - for arg in extra_args { - cargo.rustdocflag(arg); - } - - if builder.config.library_docs_private_items { - cargo.rustdocflag("--document-private-items").rustdocflag("--document-hidden-items"); - } - - for krate in requested_crates { - if krate == "sysroot" { - // The sysroot crate is an implementation detail, don't include it in public docs. - continue; - } - cargo.arg("-p").arg(krate); - } - - let description = - format!("library{} in {} format", crate_description(&requested_crates), format.as_str()); - let _guard = builder.msg_doc(compiler, &description, target); - - builder.run(&mut cargo.into()); - builder.cp_r(&out_dir, &out); -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Rustc { - pub stage: u32, - pub target: TargetSelection, - crates: Interned>, -} - -impl Rustc { - pub(crate) fn new(stage: u32, target: TargetSelection, builder: &Builder<'_>) -> Self { - let crates = builder - .in_tree_crates("rustc-main", Some(target)) - .into_iter() - .map(|krate| krate.name.to_string()) - .collect(); - Self { stage, target, crates: INTERNER.intern_list(crates) } - } -} - -impl Step for Rustc { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.crate_or_deps("rustc-main") - .path("compiler") - .default_condition(builder.config.compiler_docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustc { - stage: run.builder.top_stage, - target: run.target, - crates: run.make_run_crates(Alias::Compiler), - }); - } - - /// Generates compiler documentation. - /// - /// This will generate all documentation for compiler and dependencies. - /// Compiler documentation is distributed separately, so we make sure - /// we do not merge it with the other documentation from std, test and - /// proc_macros. This is largely just a wrapper around `cargo doc`. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let target = self.target; - - // This is the intended out directory for compiler documentation. - let out = builder.compiler_doc_out(target); - t!(fs::create_dir_all(&out)); - - // Build the standard library, so that proc-macros can use it. - // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.) - let compiler = builder.compiler(stage, builder.config.build); - builder.ensure(compile::Std::new(compiler, builder.config.build)); - - let _guard = builder.msg_sysroot_tool( - Kind::Doc, - stage, - &format!("compiler{}", crate_description(&self.crates)), - compiler.host, - target, - ); - - // This uses a shared directory so that librustdoc documentation gets - // correctly built and merged with the rustc documentation. This is - // needed because rustdoc is built in a different directory from - // rustc. rustdoc needs to be able to see everything, for example when - // merging the search index, or generating local (relative) links. - let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); - t!(fs::create_dir_all(out_dir.parent().unwrap())); - symlink_dir_force(&builder.config, &out, &out_dir); - // Cargo puts proc macros in `target/doc` even if you pass `--target` - // explicitly (https://github.com/rust-lang/cargo/issues/7677). - let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); - symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); - - // Build cargo command. - let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); - cargo.rustdocflag("--document-private-items"); - // Since we always pass --document-private-items, there's no need to warn about linking to private items. - cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); - cargo.rustdocflag("--enable-index-page"); - cargo.rustdocflag("-Zunstable-options"); - cargo.rustdocflag("-Znormalize-docs"); - cargo.rustdocflag("--show-type-layout"); - cargo.rustdocflag("--generate-link-to-definition"); - compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); - cargo.arg("-Zunstable-options"); - cargo.arg("-Zskip-rustdoc-fingerprint"); - - // Only include compiler crates, no dependencies of those, such as `libc`. - // Do link to dependencies on `docs.rs` however using `rustdoc-map`. - cargo.arg("--no-deps"); - cargo.arg("-Zrustdoc-map"); - - // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies, - // once this is no longer an issue the special case for `ena` can be removed. - cargo.rustdocflag("--extern-html-root-url"); - cargo.rustdocflag("ena=https://docs.rs/ena/latest/"); - - let mut to_open = None; - - for krate in &*self.crates { - // Create all crate output directories first to make sure rustdoc uses - // relative links. - // FIXME: Cargo should probably do this itself. - let dir_name = krate.replace("-", "_"); - t!(fs::create_dir_all(out_dir.join(&*dir_name))); - cargo.arg("-p").arg(krate); - if to_open.is_none() { - to_open = Some(dir_name); - } - } - - builder.run(&mut cargo.into()); - - if builder.paths.iter().any(|path| path.ends_with("compiler")) { - // For `x.py doc compiler --open`, open `rustc_middle` by default. - let index = out.join("rustc_middle").join("index.html"); - builder.open_in_browser(index); - } else if let Some(krate) = to_open { - // Let's open the first crate documentation page: - let index = out.join(krate).join("index.html"); - builder.open_in_browser(index); - } - } -} - -macro_rules! tool_doc { - ( - $tool: ident, - $should_run: literal, - $path: literal, - $(rustc_tool = $rustc_tool:literal, )? - $(in_tree = $in_tree:literal, )? - [$($extra_arg: literal),+ $(,)?] - $(,)? - ) => { - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - pub struct $tool { - target: TargetSelection, - } - - impl Step for $tool { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.crate_or_deps($should_run).default_condition(builder.config.compiler_docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($tool { target: run.target }); - } - - /// Generates compiler documentation. - /// - /// This will generate all documentation for compiler and dependencies. - /// Compiler documentation is distributed separately, so we make sure - /// we do not merge it with the other documentation from std, test and - /// proc_macros. This is largely just a wrapper around `cargo doc`. - fn run(self, builder: &Builder<'_>) { - let stage = builder.top_stage; - let target = self.target; - - // This is the intended out directory for compiler documentation. - let out = builder.compiler_doc_out(target); - t!(fs::create_dir_all(&out)); - - let compiler = builder.compiler(stage, builder.config.build); - builder.ensure(compile::Std::new(compiler, target)); - - if true $(&& $rustc_tool)? { - // Build rustc docs so that we generate relative links. - builder.ensure(Rustc::new(stage, target, builder)); - - // Rustdoc needs the rustc sysroot available to build. - // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed - // with strange errors, but only on a full bors test ... - builder.ensure(compile::Rustc::new(compiler, target)); - } - - let source_type = if true $(&& $in_tree)? { - SourceType::InTree - } else { - SourceType::Submodule - }; - - // Symlink compiler docs to the output directory of rustdoc documentation. - let out_dirs = [ - builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"), - // Cargo uses a different directory for proc macros. - builder.stage_out(compiler, Mode::ToolRustc).join("doc"), - ]; - for out_dir in out_dirs { - t!(fs::create_dir_all(&out_dir)); - symlink_dir_force(&builder.config, &out, &out_dir); - } - - // Build cargo command. - let mut cargo = prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - "doc", - $path, - source_type, - &[], - ); - - cargo.arg("-Zskip-rustdoc-fingerprint"); - // Only include compiler crates, no dependencies of those, such as `libc`. - cargo.arg("--no-deps"); - - $( - cargo.arg($extra_arg); - )+ - - cargo.rustdocflag("--document-private-items"); - // Since we always pass --document-private-items, there's no need to warn about linking to private items. - cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); - cargo.rustdocflag("--enable-index-page"); - cargo.rustdocflag("--show-type-layout"); - cargo.rustdocflag("--generate-link-to-definition"); - cargo.rustdocflag("-Zunstable-options"); - - let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); - builder.run(&mut cargo.into()); - } - } - } -} - -tool_doc!( - Rustdoc, - "rustdoc-tool", - "src/tools/rustdoc", - ["-p", "rustdoc", "-p", "rustdoc-json-types"] -); -tool_doc!( - Rustfmt, - "rustfmt-nightly", - "src/tools/rustfmt", - ["-p", "rustfmt-nightly", "-p", "rustfmt-config_proc_macro"], -); -tool_doc!(Clippy, "clippy", "src/tools/clippy", ["-p", "clippy_utils"]); -tool_doc!(Miri, "miri", "src/tools/miri", ["-p", "miri"]); -tool_doc!( - Cargo, - "cargo", - "src/tools/cargo", - rustc_tool = false, - in_tree = false, - [ - "-p", - "cargo", - "-p", - "cargo-platform", - "-p", - "cargo-util", - "-p", - "crates-io", - "-p", - "cargo-test-macro", - "-p", - "cargo-test-support", - "-p", - "cargo-credential", - "-p", - "mdman", - // FIXME: this trips a license check in tidy. - // "-p", - // "resolver-tests", - ] -); -tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, ["-p", "tidy"]); -tool_doc!( - Bootstrap, - "bootstrap", - "src/bootstrap", - rustc_tool = false, - ["--lib", "-p", "bootstrap"] -); - -#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct ErrorIndex { - pub target: TargetSelection, -} - -impl Step for ErrorIndex { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/error_index_generator").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - let target = run.target; - run.builder.ensure(ErrorIndex { target }); - } - - /// Generates the HTML rendered error-index by running the - /// `error_index_generator` tool. - fn run(self, builder: &Builder<'_>) { - builder.info(&format!("Documenting error index ({})", self.target)); - let out = builder.doc_out(self.target); - t!(fs::create_dir_all(&out)); - let mut index = tool::ErrorIndex::command(builder); - index.arg("html"); - index.arg(out); - index.arg(&builder.version); - - builder.run(&mut index); - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct UnstableBookGen { - target: TargetSelection, -} - -impl Step for UnstableBookGen { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/unstable-book-gen").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(UnstableBookGen { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let target = self.target; - - builder.info(&format!("Generating unstable book md files ({target})")); - let out = builder.md_doc_out(target).join("unstable-book"); - builder.create_dir(&out); - builder.remove_dir(&out); - let mut cmd = builder.tool_cmd(Tool::UnstableBookGen); - cmd.arg(builder.src.join("library")); - cmd.arg(builder.src.join("compiler")); - cmd.arg(builder.src.join("src")); - cmd.arg(out); - - builder.run(&mut cmd); - } -} - -fn symlink_dir_force(config: &Config, original: &Path, link: &Path) { - if config.dry_run() { - return; - } - if let Ok(m) = fs::symlink_metadata(link) { - if m.file_type().is_dir() { - t!(fs::remove_dir_all(link)); - } else { - // handle directory junctions on windows by falling back to - // `remove_dir`. - t!(fs::remove_file(link).or_else(|_| fs::remove_dir(link))); - } - } - - t!( - symlink_dir(config, original, link), - format!("failed to create link from {} -> {}", link.display(), original.display()) - ); -} - -#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustcBook { - pub compiler: Compiler, - pub target: TargetSelection, - pub validate: bool, -} - -impl Step for RustcBook { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/doc/rustc").default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcBook { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - validate: false, - }); - } - - /// Builds the rustc book. - /// - /// The lints are auto-generated by a tool, and then merged into the book - /// in the "md-doc" directory in the build output directory. Then - /// "rustbook" is used to convert it to HTML. - fn run(self, builder: &Builder<'_>) { - let out_base = builder.md_doc_out(self.target).join("rustc"); - t!(fs::create_dir_all(&out_base)); - let out_listing = out_base.join("src/lints"); - builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base); - builder.info(&format!("Generating lint docs ({})", self.target)); - - let rustc = builder.rustc(self.compiler); - // The tool runs `rustc` for extracting output examples, so it needs a - // functional sysroot. - builder.ensure(compile::Std::new(self.compiler, self.target)); - let mut cmd = builder.tool_cmd(Tool::LintDocs); - cmd.arg("--src"); - cmd.arg(builder.src.join("compiler")); - cmd.arg("--out"); - cmd.arg(&out_listing); - cmd.arg("--rustc"); - cmd.arg(&rustc); - cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg()); - if builder.is_verbose() { - cmd.arg("--verbose"); - } - if self.validate { - cmd.arg("--validate"); - } - // We need to validate nightly features, even on the stable channel. - // Set this unconditionally as the stage0 compiler may be being used to - // document. - cmd.env("RUSTC_BOOTSTRAP", "1"); - - // If the lib directories are in an unusual location (changed in - // config.toml), then this needs to explicitly update the dylib search - // path. - builder.add_rustc_lib_path(self.compiler, &mut cmd); - let doc_generator_guard = builder.msg( - Kind::Run, - self.compiler.stage, - "lint-docs", - self.compiler.host, - self.target, - ); - builder.run(&mut cmd); - drop(doc_generator_guard); - - // Run rustbook/mdbook to generate the HTML pages. - builder.ensure(RustbookSrc { - target: self.target, - name: INTERNER.intern_str("rustc"), - src: INTERNER.intern_path(out_base), - parent: Some(self), - }); - } -} diff --git a/src/bootstrap/download-ci-llvm-stamp b/src/bootstrap/download-ci-llvm-stamp index ffc380579..9998fe2f5 100644 --- a/src/bootstrap/download-ci-llvm-stamp +++ b/src/bootstrap/download-ci-llvm-stamp @@ -1,4 +1,4 @@ Change this file to make users of the `download-ci-llvm` configuration download a new version of LLVM from CI, even if the LLVM submodule hasn’t changed. -Last change is for: https://github.com/rust-lang/rust/pull/113996 +Last change is for: https://github.com/rust-lang/rust/pull/116881 diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs deleted file mode 100644 index 8e9614ec8..000000000 --- a/src/bootstrap/download.rs +++ /dev/null @@ -1,686 +0,0 @@ -use std::{ - env, - ffi::{OsStr, OsString}, - fs::{self, File}, - io::{BufRead, BufReader, BufWriter, ErrorKind, Write}, - path::{Path, PathBuf}, - process::{Command, Stdio}, -}; - -use build_helper::ci::CiEnv; -use once_cell::sync::OnceCell; -use xz2::bufread::XzDecoder; - -use crate::{ - config::RustfmtMetadata, - llvm::detect_llvm_sha, - t, - util::{check_run, exe, program_out_of_date}, - Config, -}; - -static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell = OnceCell::new(); - -/// `Config::try_run` wrapper for this module to avoid warnings on `try_run`, since we don't have access to a `builder` yet. -fn try_run(config: &Config, cmd: &mut Command) -> Result<(), ()> { - #[allow(deprecated)] - config.try_run(cmd) -} - -/// Generic helpers that are useful anywhere in bootstrap. -impl Config { - pub fn is_verbose(&self) -> bool { - self.verbose > 0 - } - - pub(crate) fn create(&self, path: &Path, s: &str) { - if self.dry_run() { - return; - } - t!(fs::write(path, s)); - } - - pub(crate) fn remove(&self, f: &Path) { - if self.dry_run() { - return; - } - fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f)); - } - - /// Create a temporary directory in `out` and return its path. - /// - /// NOTE: this temporary directory is shared between all steps; - /// if you need an empty directory, create a new subdirectory inside it. - pub(crate) fn tempdir(&self) -> PathBuf { - let tmp = self.out.join("tmp"); - t!(fs::create_dir_all(&tmp)); - tmp - } - - /// Runs a command, printing out nice contextual information if it fails. - /// Returns false if do not execute at all, otherwise returns its - /// `status.success()`. - pub(crate) fn check_run(&self, cmd: &mut Command) -> bool { - if self.dry_run() { - return true; - } - self.verbose(&format!("running: {cmd:?}")); - check_run(cmd, self.is_verbose()) - } - - /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true - /// on NixOS - fn should_fix_bins_and_dylibs(&self) -> bool { - let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| { - match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { - Err(_) => return false, - Ok(output) if !output.status.success() => return false, - Ok(output) => { - let mut os_name = output.stdout; - if os_name.last() == Some(&b'\n') { - os_name.pop(); - } - if os_name != b"Linux" { - return false; - } - } - } - - // If the user has asked binaries to be patched for Nix, then - // don't check for NixOS or `/lib`. - // NOTE: this intentionally comes after the Linux check: - // - patchelf only works with ELF files, so no need to run it on Mac or Windows - // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc. - if let Some(explicit_value) = self.patch_binaries_for_nix { - return explicit_value; - } - - // Use `/etc/os-release` instead of `/etc/NIXOS`. - // The latter one does not exist on NixOS when using tmpfs as root. - let is_nixos = match File::open("/etc/os-release") { - Err(e) if e.kind() == ErrorKind::NotFound => false, - Err(e) => panic!("failed to access /etc/os-release: {}", e), - Ok(os_release) => BufReader::new(os_release).lines().any(|l| { - let l = l.expect("reading /etc/os-release"); - matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"") - }), - }; - if !is_nixos { - let in_nix_shell = env::var("IN_NIX_SHELL"); - if let Ok(in_nix_shell) = in_nix_shell { - eprintln!( - "The IN_NIX_SHELL environment variable is `{in_nix_shell}`; \ - you may need to set `patch-binaries-for-nix=true` in config.toml" - ); - } - } - is_nixos - }); - if val { - eprintln!("info: You seem to be using Nix."); - } - val - } - - /// Modifies the interpreter section of 'fname' to fix the dynamic linker, - /// or the RPATH section, to fix the dynamic library search path - /// - /// This is only required on NixOS and uses the PatchELF utility to - /// change the interpreter/RPATH of ELF executables. - /// - /// Please see for more information - fn fix_bin_or_dylib(&self, fname: &Path) { - assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true)); - println!("attempting to patch {}", fname.display()); - - // Only build `.nix-deps` once. - static NIX_DEPS_DIR: OnceCell = OnceCell::new(); - let mut nix_build_succeeded = true; - let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { - // Run `nix-build` to "build" each dependency (which will likely reuse - // the existing `/nix/store` copy, or at most download a pre-built copy). - // - // Importantly, we create a gc-root called `.nix-deps` in the `build/` - // directory, but still reference the actual `/nix/store` path in the rpath - // as it makes it significantly more robust against changes to the location of - // the `.nix-deps` location. - // - // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). - // zlib: Needed as a system dependency of `libLLVM-*.so`. - // patchelf: Needed for patching ELF binaries (see doc comment above). - let nix_deps_dir = self.out.join(".nix-deps"); - const NIX_EXPR: &str = " - with (import {}); - symlinkJoin { - name = \"rust-stage0-dependencies\"; - paths = [ - zlib - patchelf - stdenv.cc.bintools - ]; - } - "; - nix_build_succeeded = try_run( - self, - Command::new("nix-build").args(&[ - Path::new("-E"), - Path::new(NIX_EXPR), - Path::new("-o"), - &nix_deps_dir, - ]), - ) - .is_ok(); - nix_deps_dir - }); - if !nix_build_succeeded { - return; - } - - let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); - let rpath_entries = { - // ORIGIN is a relative default, all binary and dynamic libraries we ship - // appear to have this (even when `../lib` is redundant). - // NOTE: there are only two paths here, delimited by a `:` - let mut entries = OsString::from("$ORIGIN/../lib:"); - entries.push(t!(fs::canonicalize(nix_deps_dir)).join("lib")); - entries - }; - patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); - if !fname.extension().map_or(false, |ext| ext == "so") { - // Finally, set the correct .interp for binaries - let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); - // FIXME: can we support utf8 here? `args` doesn't accept Vec, only OsString ... - let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); - patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); - } - - let _ = try_run(self, patchelf.arg(fname)); - } - - fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) { - self.verbose(&format!("download {url}")); - // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. - let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); - // While bootstrap itself only supports http and https downloads, downstream forks might - // need to download components from other protocols. The match allows them adding more - // protocols without worrying about merge conflicts if we change the HTTP implementation. - match url.split_once("://").map(|(proto, _)| proto) { - Some("http") | Some("https") => { - self.download_http_with_retries(&tempfile, url, help_on_error) - } - Some(other) => panic!("unsupported protocol {other} in {url}"), - None => panic!("no protocol in {url}"), - } - t!(std::fs::rename(&tempfile, dest_path)); - } - - fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { - println!("downloading {url}"); - // Try curl. If that fails and we are on windows, fallback to PowerShell. - let mut curl = Command::new("curl"); - curl.args(&[ - "-y", - "30", - "-Y", - "10", // timeout if speed is < 10 bytes/sec for > 30 seconds - "--connect-timeout", - "30", // timeout if cannot connect within 30 seconds - "-o", - tempfile.to_str().unwrap(), - "--retry", - "3", - "-SRf", - ]); - // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful. - if CiEnv::is_ci() { - curl.arg("-s"); - } else { - curl.arg("--progress-bar"); - } - curl.arg(url); - if !self.check_run(&mut curl) { - if self.build.contains("windows-msvc") { - eprintln!("Fallback to PowerShell"); - for _ in 0..3 { - if try_run(self, Command::new("PowerShell.exe").args(&[ - "/nologo", - "-Command", - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - &format!( - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), - ), - ])).is_err() { - return; - } - eprintln!("\nspurious failure, trying again"); - } - } - if !help_on_error.is_empty() { - eprintln!("{help_on_error}"); - } - crate::exit!(1); - } - } - - fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) { - eprintln!("extracting {} to {}", tarball.display(), dst.display()); - if !dst.exists() { - t!(fs::create_dir_all(dst)); - } - - // `tarball` ends with `.tar.xz`; strip that suffix - // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` - let uncompressed_filename = - Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); - let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); - - // decompress the file - let data = t!(File::open(tarball), format!("file {} not found", tarball.display())); - let decompressor = XzDecoder::new(BufReader::new(data)); - - let mut tar = tar::Archive::new(decompressor); - - // `compile::Sysroot` needs to know the contents of the `rustc-dev` tarball to avoid adding - // it to the sysroot unless it was explicitly requested. But parsing the 100 MB tarball is slow. - // Cache the entries when we extract it so we only have to read it once. - let mut recorded_entries = - if dst.ends_with("ci-rustc") { recorded_entries(dst, pattern) } else { None }; - - for member in t!(tar.entries()) { - let mut member = t!(member); - let original_path = t!(member.path()).into_owned(); - // skip the top-level directory - if original_path == directory_prefix { - continue; - } - let mut short_path = t!(original_path.strip_prefix(directory_prefix)); - if !short_path.starts_with(pattern) { - continue; - } - short_path = t!(short_path.strip_prefix(pattern)); - let dst_path = dst.join(short_path); - self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); - if !t!(member.unpack_in(dst)) { - panic!("path traversal attack ??"); - } - if let Some(record) = &mut recorded_entries { - t!(writeln!(record, "{}", short_path.to_str().unwrap())); - } - let src_path = dst.join(original_path); - if src_path.is_dir() && dst_path.exists() { - continue; - } - t!(fs::rename(src_path, dst_path)); - } - let dst_dir = dst.join(directory_prefix); - if dst_dir.exists() { - t!(fs::remove_dir_all(&dst_dir), format!("failed to remove {}", dst_dir.display())); - } - } - - /// Returns whether the SHA256 checksum of `path` matches `expected`. - fn verify(&self, path: &Path, expected: &str) -> bool { - use sha2::Digest; - - self.verbose(&format!("verifying {}", path.display())); - let mut hasher = sha2::Sha256::new(); - // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components. - // Consider using streaming IO instead? - let contents = if self.dry_run() { vec![] } else { t!(fs::read(path)) }; - hasher.update(&contents); - let found = hex::encode(hasher.finalize().as_slice()); - let verified = found == expected; - if !verified && !self.dry_run() { - println!( - "invalid checksum: \n\ - found: {found}\n\ - expected: {expected}", - ); - } - return verified; - } -} - -fn recorded_entries(dst: &Path, pattern: &str) -> Option> { - let name = if pattern == "rustc-dev" { - ".rustc-dev-contents" - } else if pattern.starts_with("rust-std") { - ".rust-std-contents" - } else { - return None; - }; - Some(BufWriter::new(t!(File::create(dst.join(name))))) -} - -enum DownloadSource { - CI, - Dist, -} - -/// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions. -impl Config { - pub(crate) fn maybe_download_rustfmt(&self) -> Option { - let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; - let channel = format!("{version}-{date}"); - - let host = self.build; - let bin_root = self.out.join(host.triple).join("rustfmt"); - let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); - let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); - if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { - return Some(rustfmt_path); - } - - self.download_component( - DownloadSource::Dist, - format!("rustfmt-{version}-{build}.tar.xz", build = host.triple), - "rustfmt-preview", - &date, - "rustfmt", - ); - self.download_component( - DownloadSource::Dist, - format!("rustc-{version}-{build}.tar.xz", build = host.triple), - "rustc", - &date, - "rustfmt", - ); - - if self.should_fix_bins_and_dylibs() { - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); - self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); - let lib_dir = bin_root.join("lib"); - for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { - let lib = t!(lib); - if lib.path().extension() == Some(OsStr::new("so")) { - self.fix_bin_or_dylib(&lib.path()); - } - } - } - - self.create(&rustfmt_stamp, &channel); - Some(rustfmt_path) - } - - pub(crate) fn ci_rust_std_contents(&self) -> Vec { - self.ci_component_contents(".rust-std-contents") - } - - pub(crate) fn ci_rustc_dev_contents(&self) -> Vec { - self.ci_component_contents(".rustc-dev-contents") - } - - fn ci_component_contents(&self, stamp_file: &str) -> Vec { - assert!(self.download_rustc()); - if self.dry_run() { - return vec![]; - } - - let ci_rustc_dir = self.ci_rustc_dir(); - let stamp_file = ci_rustc_dir.join(stamp_file); - let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string()); - t!(BufReader::new(contents_file).lines().collect()) - } - - pub(crate) fn download_ci_rustc(&self, commit: &str) { - self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); - - let version = self.artifact_version_part(commit); - // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the - // `rustc_private` crates for tools. - let extra_components = ["rustc-dev"]; - - self.download_toolchain( - &version, - "ci-rustc", - &format!("{commit}-{}", self.llvm_assertions), - &extra_components, - Self::download_ci_component, - ); - } - - pub(crate) fn download_beta_toolchain(&self) { - self.verbose("downloading stage0 beta artifacts"); - - let date = &self.stage0_metadata.compiler.date; - let version = &self.stage0_metadata.compiler.version; - let extra_components = ["cargo"]; - - let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { - config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") - }; - - self.download_toolchain( - version, - "stage0", - date, - &extra_components, - download_beta_component, - ); - } - - fn download_toolchain( - &self, - version: &str, - sysroot: &str, - stamp_key: &str, - extra_components: &[&str], - download_component: fn(&Config, String, &str, &str), - ) { - let host = self.build.triple; - let bin_root = self.out.join(host).join(sysroot); - let rustc_stamp = bin_root.join(".rustc-stamp"); - - if !bin_root.join("bin").join(exe("rustc", self.build)).exists() - || program_out_of_date(&rustc_stamp, stamp_key) - { - if bin_root.exists() { - t!(fs::remove_dir_all(&bin_root)); - } - let filename = format!("rust-std-{version}-{host}.tar.xz"); - let pattern = format!("rust-std-{host}"); - download_component(self, filename, &pattern, stamp_key); - let filename = format!("rustc-{version}-{host}.tar.xz"); - download_component(self, filename, "rustc", stamp_key); - - for component in extra_components { - let filename = format!("{component}-{version}-{host}.tar.xz"); - download_component(self, filename, component, stamp_key); - } - - if self.should_fix_bins_and_dylibs() { - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); - self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); - self.fix_bin_or_dylib( - &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"), - ); - let lib_dir = bin_root.join("lib"); - for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { - let lib = t!(lib); - if lib.path().extension() == Some(OsStr::new("so")) { - self.fix_bin_or_dylib(&lib.path()); - } - } - } - - t!(fs::write(rustc_stamp, stamp_key)); - } - } - - /// Download a single component of a CI-built toolchain (not necessarily a published nightly). - // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident - fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) { - Self::download_component( - self, - DownloadSource::CI, - filename, - prefix, - commit_with_assertions, - "ci-rustc", - ) - } - - fn download_component( - &self, - mode: DownloadSource, - filename: String, - prefix: &str, - key: &str, - destination: &str, - ) { - let cache_dst = self.out.join("cache"); - let cache_dir = cache_dst.join(key); - if !cache_dir.exists() { - t!(fs::create_dir_all(&cache_dir)); - } - - let bin_root = self.out.join(self.build.triple).join(destination); - let tarball = cache_dir.join(&filename); - let (base_url, url, should_verify) = match mode { - DownloadSource::CI => { - let dist_server = if self.llvm_assertions { - self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone() - } else { - self.stage0_metadata.config.artifacts_server.clone() - }; - let url = format!( - "{}/{filename}", - key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap() - ); - (dist_server, url, false) - } - DownloadSource::Dist => { - let dist_server = env::var("RUSTUP_DIST_SERVER") - .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); - // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json - (dist_server, format!("dist/{key}/{filename}"), true) - } - }; - - // For the beta compiler, put special effort into ensuring the checksums are valid. - // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update - // this on each and every nightly ... - let checksum = if should_verify { - let error = format!( - "src/stage0.json doesn't contain a checksum for {url}. \ - Pre-built artifacts might not be available for this \ - target at this time, see https://doc.rust-lang.org/nightly\ - /rustc/platform-support.html for more information." - ); - let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); - if tarball.exists() { - if self.verify(&tarball, sha256) { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - self.verbose(&format!( - "ignoring cached file {} due to failed verification", - tarball.display() - )); - self.remove(&tarball); - } - } - Some(sha256) - } else if tarball.exists() { - self.unpack(&tarball, &bin_root, prefix); - return; - } else { - None - }; - - let mut help_on_error = ""; - if destination == "ci-rustc" { - help_on_error = "error: failed to download pre-built rustc from CI - -note: old builds get deleted after a certain time -help: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: - -[rust] -download-rustc = false -"; - } - self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error); - if let Some(sha256) = checksum { - if !self.verify(&tarball, sha256) { - panic!("failed to verify {}", tarball.display()); - } - } - - self.unpack(&tarball, &bin_root, prefix); - } - - pub(crate) fn maybe_download_ci_llvm(&self) { - if !self.llvm_from_ci { - return; - } - let llvm_root = self.ci_llvm_root(); - let llvm_stamp = llvm_root.join(".llvm-stamp"); - let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository()); - let key = format!("{}{}", llvm_sha, self.llvm_assertions); - if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { - self.download_ci_llvm(&llvm_sha); - if self.should_fix_bins_and_dylibs() { - for entry in t!(fs::read_dir(llvm_root.join("bin"))) { - self.fix_bin_or_dylib(&t!(entry).path()); - } - } - - // Update the timestamp of llvm-config to force rustc_llvm to be - // rebuilt. This is a hacky workaround for a deficiency in Cargo where - // the rerun-if-changed directive doesn't handle changes very well. - // https://github.com/rust-lang/cargo/issues/10791 - // Cargo only compares the timestamp of the file relative to the last - // time `rustc_llvm` build script ran. However, the timestamps of the - // files in the tarball are in the past, so it doesn't trigger a - // rebuild. - let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); - let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); - t!(filetime::set_file_times(&llvm_config, now, now)); - - if self.should_fix_bins_and_dylibs() { - let llvm_lib = llvm_root.join("lib"); - for entry in t!(fs::read_dir(&llvm_lib)) { - let lib = t!(entry).path(); - if lib.extension().map_or(false, |ext| ext == "so") { - self.fix_bin_or_dylib(&lib); - } - } - } - - t!(fs::write(llvm_stamp, key)); - } - } - - fn download_ci_llvm(&self, llvm_sha: &str) { - let llvm_assertions = self.llvm_assertions; - - let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}"); - let cache_dst = self.out.join("cache"); - let rustc_cache = cache_dst.join(cache_prefix); - if !rustc_cache.exists() { - t!(fs::create_dir_all(&rustc_cache)); - } - let base = if llvm_assertions { - &self.stage0_metadata.config.artifacts_with_llvm_assertions_server - } else { - &self.stage0_metadata.config.artifacts_server - }; - let version = self.artifact_version_part(llvm_sha); - let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); - let tarball = rustc_cache.join(&filename); - if !tarball.exists() { - let help_on_error = "error: failed to download llvm from ci - - help: old builds get deleted after a certain time - help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: - - [llvm] - download-ci-llvm = false - "; - self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); - } - let llvm_root = self.ci_llvm_root(); - self.unpack(&tarball, &llvm_root, "rust-dev"); - } -} diff --git a/src/bootstrap/dylib_util.rs b/src/bootstrap/dylib_util.rs deleted file mode 100644 index b14c0bed6..000000000 --- a/src/bootstrap/dylib_util.rs +++ /dev/null @@ -1,30 +0,0 @@ -// Various utilities for working with dylib paths. -// -// This file is meant to be included directly to avoid a dependency on the bootstrap library from -// the rustc and rustdoc wrappers. This improves compilation time by reducing the linking time. - -/// Returns the environment variable which the dynamic library lookup path -/// resides in for this platform. -pub fn dylib_path_var() -> &'static str { - if cfg!(target_os = "windows") { - "PATH" - } else if cfg!(target_os = "macos") { - "DYLD_LIBRARY_PATH" - } else if cfg!(target_os = "haiku") { - "LIBRARY_PATH" - } else if cfg!(target_os = "aix") { - "LIBPATH" - } else { - "LD_LIBRARY_PATH" - } -} - -/// Parses the `dylib_path_var()` environment variable, returning a list of -/// paths that are members of this lookup path. -pub fn dylib_path() -> Vec { - let var = match env::var_os(dylib_path_var()) { - Some(v) => v, - None => return vec![], - }; - env::split_paths(&var).collect() -} diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs deleted file mode 100644 index e0291e407..000000000 --- a/src/bootstrap/flags.rs +++ /dev/null @@ -1,565 +0,0 @@ -//! Command-line interface of the rustbuild build system. -//! -//! This module implements the command-line parsing of the build system which -//! has various flags to configure how it's run. - -use std::path::{Path, PathBuf}; - -use clap::{CommandFactory, Parser, ValueEnum}; - -use crate::builder::{Builder, Kind}; -use crate::config::{target_selection_list, Config, TargetSelectionList}; -use crate::setup::Profile; -use crate::{Build, DocTests}; - -#[derive(Copy, Clone, Default, Debug, ValueEnum)] -pub enum Color { - Always, - Never, - #[default] - Auto, -} - -/// Whether to deny warnings, emit them as warnings, or use the default behavior -#[derive(Copy, Clone, Default, Debug, ValueEnum)] -pub enum Warnings { - Deny, - Warn, - #[default] - Default, -} - -/// Deserialized version of all flags for this compile. -#[derive(Debug, Parser)] -#[clap( - override_usage = "x.py [options] [...]", - disable_help_subcommand(true), - about = "", - next_line_help(false) -)] -pub struct Flags { - #[command(subcommand)] - pub cmd: Subcommand, - - #[arg(global(true), short, long, action = clap::ArgAction::Count)] - /// use verbose output (-vv for very verbose) - pub verbose: u8, // each extra -v after the first is passed to Cargo - #[arg(global(true), short, long)] - /// use incremental compilation - pub incremental: bool, - #[arg(global(true), long, value_hint = clap::ValueHint::FilePath, value_name = "FILE")] - /// TOML configuration file for build - pub config: Option, - #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] - /// Build directory, overrides `build.build-dir` in `config.toml` - pub build_dir: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "BUILD")] - /// build target of the stage0 compiler - pub build: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "HOST", value_parser = target_selection_list)] - /// host targets to build - pub host: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "TARGET", value_parser = target_selection_list)] - /// target targets to build - pub target: Option, - - #[arg(global(true), long, value_name = "PATH")] - /// build paths to exclude - pub exclude: Vec, // keeping for client backward compatibility - #[arg(global(true), long, value_name = "PATH")] - /// build paths to skip - pub skip: Vec, - #[arg(global(true), long)] - /// include default paths in addition to the provided ones - pub include_default_paths: bool, - - #[arg(global(true), value_hint = clap::ValueHint::Other, long)] - pub rustc_error_format: Option, - - #[arg(global(true), long, value_hint = clap::ValueHint::CommandString, value_name = "CMD")] - /// command to run on failure - pub on_fail: Option, - #[arg(global(true), long)] - /// dry run; don't build anything - pub dry_run: bool, - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the - /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) - pub stage: Option, - - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage(s) to keep without recompiling - /// (pass multiple times to keep e.g., both stages 0 and 1) - pub keep_stage: Vec, - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] - /// stage(s) of the standard library to keep without recompiling - /// (pass multiple times to keep e.g., both stages 0 and 1) - pub keep_stage_std: Vec, - #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] - /// path to the root of the rust checkout - pub src: Option, - - #[arg( - global(true), - short, - long, - value_hint = clap::ValueHint::Other, - default_value_t = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get), - value_name = "JOBS" - )] - /// number of jobs to run in parallel - pub jobs: usize, - // This overrides the deny-warnings configuration option, - // which passes -Dwarnings to the compiler invocations. - #[arg(global(true), long)] - #[clap(value_enum, default_value_t=Warnings::Default, value_name = "deny|warn")] - /// if value is deny, will deny warnings - /// if value is warn, will emit warnings - /// otherwise, use the default configured behaviour - pub warnings: Warnings, - - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "FORMAT")] - /// rustc error format - pub error_format: Option, - #[arg(global(true), long)] - /// use message-format=json - pub json_output: bool, - - #[arg(global(true), long, value_name = "STYLE")] - #[clap(value_enum, default_value_t = Color::Auto)] - /// whether to use color in cargo and rustc output - pub color: Color, - - /// whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml - #[arg(global(true), long, value_name = "VALUE")] - pub llvm_skip_rebuild: Option, - /// generate PGO profile with rustc build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub rust_profile_generate: Option, - /// use PGO profile for rustc build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub rust_profile_use: Option, - /// use PGO profile for LLVM build - #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] - pub llvm_profile_use: Option, - // LLVM doesn't support a custom location for generating profile - // information. - // - // llvm_out/build/profiles/ is the location this writes to. - /// generate PGO profile with llvm built for rustc - #[arg(global(true), long)] - pub llvm_profile_generate: bool, - /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. - #[arg(global(true), long)] - pub reproducible_artifact: Vec, - #[arg(global(true))] - /// paths for the subcommand - pub paths: Vec, - /// override options in config.toml - #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "section.option=value")] - pub set: Vec, - /// arguments passed to subcommands - #[arg(global(true), last(true), value_name = "ARGS")] - pub free_args: Vec, -} - -impl Flags { - pub fn parse(args: &[String]) -> Self { - let first = String::from("x.py"); - let it = std::iter::once(&first).chain(args.iter()); - // We need to check for ` -h -v`, in which case we list the paths - #[derive(Parser)] - #[clap(disable_help_flag(true))] - struct HelpVerboseOnly { - #[arg(short, long)] - help: bool, - #[arg(global(true), short, long, action = clap::ArgAction::Count)] - pub verbose: u8, - #[arg(value_enum)] - cmd: Kind, - } - if let Ok(HelpVerboseOnly { help: true, verbose: 1.., cmd: subcommand }) = - HelpVerboseOnly::try_parse_from(it.clone()) - { - println!("note: updating submodules before printing available paths"); - let config = Config::parse(&[String::from("build")]); - let build = Build::new(config); - let paths = Builder::get_help(&build, subcommand); - if let Some(s) = paths { - println!("{s}"); - } else { - panic!("No paths available for subcommand `{}`", subcommand.as_str()); - } - crate::exit!(0); - } - - Flags::parse_from(it) - } -} - -#[derive(Debug, Clone, Default, clap::Subcommand)] -pub enum Subcommand { - #[clap(aliases = ["b"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example, for a quick build of a usable - compiler: - ./x.py build --stage 1 library/std - This will build a compiler and standard library from the local source code. - Once this is done, build/$ARCH/stage1 contains a usable compiler. - If no arguments are passed then the default artifacts for that stage are - compiled. For example: - ./x.py build --stage 0 - ./x.py build ")] - /// Compile either the compiler or libraries - #[default] - Build, - #[clap(aliases = ["c"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to compile. For example: - ./x.py check library/std - If no arguments are passed then many artifacts are checked.")] - /// Compile either the compiler or libraries, using cargo check - Check { - #[arg(long)] - /// Check all targets - all_targets: bool, - }, - /// Run Clippy (uses rustup/cargo-installed clippy binary) - #[clap(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run clippy against. For example: - ./x.py clippy library/core - ./x.py clippy library/core library/proc_macro")] - Clippy { - #[arg(long)] - fix: bool, - /// clippy lints to allow - #[arg(global(true), short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] - allow: Vec, - /// clippy lints to deny - #[arg(global(true), short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] - deny: Vec, - /// clippy lints to warn on - #[arg(global(true), short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] - warn: Vec, - /// clippy lints to forbid - #[arg(global(true), short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] - forbid: Vec, - }, - /// Run cargo fix - #[clap(long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories to the crates - and/or artifacts to run `cargo fix` against. For example: - ./x.py fix library/core - ./x.py fix library/core library/proc_macro")] - Fix, - #[clap( - name = "fmt", - long_about = "\n - Arguments: - This subcommand optionally accepts a `--check` flag which succeeds if formatting is correct and - fails if it is not. For example: - ./x.py fmt - ./x.py fmt --check" - )] - /// Run rustfmt - Format { - /// check formatting instead of applying - #[arg(long)] - check: bool, - }, - #[clap(aliases = ["d"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to directories of documentation - to build. For example: - ./x.py doc src/doc/book - ./x.py doc src/doc/nomicon - ./x.py doc src/doc/book library/std - ./x.py doc library/std --json - ./x.py doc library/std --open - If no arguments are passed then everything is documented: - ./x.py doc - ./x.py doc --stage 1")] - /// Build documentation - Doc { - #[arg(long)] - /// open the docs in a browser - open: bool, - #[arg(long)] - /// render the documentation in JSON format in addition to the usual HTML format - json: bool, - }, - #[clap(aliases = ["t"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to test directories that - should be compiled and run. For example: - ./x.py test tests/ui - ./x.py test library/std --test-args hash_map - ./x.py test library/std --stage 0 --no-doc - ./x.py test tests/ui --bless - ./x.py test tests/ui --compare-mode next-solver - Note that `test tests/* --stage N` does NOT depend on `build compiler/rustc --stage N`; - just like `build library/std --stage N` it tests the compiler produced by the previous - stage. - Execute tool tests with a tool name argument: - ./x.py test tidy - If no arguments are passed then the complete artifacts for that stage are - compiled and tested. - ./x.py test - ./x.py test --stage 1")] - /// Build and run some test suites - Test { - #[arg(long)] - /// run all tests regardless of failure - no_fail_fast: bool, - #[arg(long, value_name = "SUBSTRING")] - /// skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times - skip: Vec, - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - /// extra arguments to be passed for the test tool being used - /// (e.g. libtest, compiletest or rustdoc) - test_args: Vec, - /// extra options to pass the compiler when running tests - #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] - rustc_args: Vec, - #[arg(long)] - /// do not run doc tests - no_doc: bool, - #[arg(long)] - /// only run doc tests - doc: bool, - #[arg(long)] - /// whether to automatically update stderr/stdout files - bless: bool, - #[arg(long)] - /// comma-separated list of other files types to check (accepts py, py:lint, - /// py:fmt, shell) - extra_checks: Option, - #[arg(long)] - /// rerun tests even if the inputs are unchanged - force_rerun: bool, - #[arg(long)] - /// only run tests that result has been changed - only_modified: bool, - #[arg(long, value_name = "COMPARE MODE")] - /// mode describing what file the actual ui output will be compared to - compare_mode: Option, - #[arg(long, value_name = "check | build | run")] - /// force {check,build,run}-pass tests to this mode. - pass: Option, - #[arg(long, value_name = "auto | always | never")] - /// whether to execute run-* tests - run: Option, - #[arg(long)] - /// enable this to generate a Rustfix coverage file, which is saved in - /// `//rustfix_missing_coverage.txt` - rustfix_coverage: bool, - }, - /// Build and run some benchmarks - Bench { - #[arg(long, allow_hyphen_values(true))] - test_args: Vec, - }, - /// Clean out build directories - Clean { - #[arg(long)] - /// Clean the entire build directory (not used by default) - all: bool, - #[arg(long, value_name = "N")] - /// Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used. - stage: Option, - }, - /// Build distribution artifacts - Dist, - /// Install distribution artifacts - Install, - #[clap(aliases = ["r"], long_about = "\n - Arguments: - This subcommand accepts a number of paths to tools to build and run. For - example: - ./x.py run src/tools/expand-yaml-anchors - At least a tool needs to be called.")] - /// Run tools contained in this repository - Run { - /// arguments for the tool - #[arg(long, allow_hyphen_values(true))] - args: Vec, - }, - /// Set up the environment for development - #[clap(long_about = format!( - "\n -x.py setup creates a `config.toml` which changes the defaults for x.py itself, -as well as setting up a git pre-push hook, VS Code config and toolchain link. -Arguments: - This subcommand accepts a 'profile' to use for builds. For example: - ./x.py setup library - The profile is optional and you will be prompted interactively if it is not given. - The following profiles are available: -{} - To only set up the git hook, VS Code config or toolchain link, you may use - ./x.py setup hook - ./x.py setup vscode - ./x.py setup link", Profile::all_for_help(" ").trim_end()))] - Setup { - /// Either the profile for `config.toml` or another setup action. - /// May be omitted to set up interactively - #[arg(value_name = "|hook|vscode|link")] - profile: Option, - }, - /// Suggest a subset of tests to run, based on modified files - #[clap(long_about = "\n")] - Suggest { - /// run suggested tests - #[arg(long)] - run: bool, - }, -} - -impl Subcommand { - pub fn kind(&self) -> Kind { - match self { - Subcommand::Bench { .. } => Kind::Bench, - Subcommand::Build { .. } => Kind::Build, - Subcommand::Check { .. } => Kind::Check, - Subcommand::Clippy { .. } => Kind::Clippy, - Subcommand::Doc { .. } => Kind::Doc, - Subcommand::Fix { .. } => Kind::Fix, - Subcommand::Format { .. } => Kind::Format, - Subcommand::Test { .. } => Kind::Test, - Subcommand::Clean { .. } => Kind::Clean, - Subcommand::Dist { .. } => Kind::Dist, - Subcommand::Install { .. } => Kind::Install, - Subcommand::Run { .. } => Kind::Run, - Subcommand::Setup { .. } => Kind::Setup, - Subcommand::Suggest { .. } => Kind::Suggest, - } - } - - pub fn rustc_args(&self) -> Vec<&str> { - match *self { - Subcommand::Test { ref rustc_args, .. } => { - rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() - } - _ => vec![], - } - } - - pub fn fail_fast(&self) -> bool { - match *self { - Subcommand::Test { no_fail_fast, .. } => !no_fail_fast, - _ => false, - } - } - - pub fn doc_tests(&self) -> DocTests { - match *self { - Subcommand::Test { doc, no_doc, .. } => { - if doc { - DocTests::Only - } else if no_doc { - DocTests::No - } else { - DocTests::Yes - } - } - _ => DocTests::Yes, - } - } - - pub fn bless(&self) -> bool { - match *self { - Subcommand::Test { bless, .. } => bless, - _ => false, - } - } - - pub fn extra_checks(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str), - _ => None, - } - } - - pub fn only_modified(&self) -> bool { - match *self { - Subcommand::Test { only_modified, .. } => only_modified, - _ => false, - } - } - - pub fn force_rerun(&self) -> bool { - match *self { - Subcommand::Test { force_rerun, .. } => force_rerun, - _ => false, - } - } - - pub fn rustfix_coverage(&self) -> bool { - match *self { - Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, - _ => false, - } - } - - pub fn compare_mode(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn pass(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn run(&self) -> Option<&str> { - match *self { - Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), - _ => None, - } - } - - pub fn open(&self) -> bool { - match *self { - Subcommand::Doc { open, .. } => open, - _ => false, - } - } - - pub fn json(&self) -> bool { - match *self { - Subcommand::Doc { json, .. } => json, - _ => false, - } - } -} - -/// Returns the shell completion for a given shell, if the result differs from the current -/// content of `path`. If `path` does not exist, always returns `Some`. -pub fn get_completion(shell: G, path: &Path) -> Option { - let mut cmd = Flags::command(); - let current = if !path.exists() { - String::new() - } else { - std::fs::read_to_string(path).unwrap_or_else(|_| { - eprintln!("couldn't read {}", path.display()); - crate::exit!(1) - }) - }; - let mut buf = Vec::new(); - clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); - if buf == current.as_bytes() { - return None; - } - Some(String::from_utf8(buf).expect("completion script should be UTF-8")) -} diff --git a/src/bootstrap/format.rs b/src/bootstrap/format.rs deleted file mode 100644 index 11f2762f7..000000000 --- a/src/bootstrap/format.rs +++ /dev/null @@ -1,322 +0,0 @@ -//! Runs rustfmt on the repository. - -use crate::builder::Builder; -use crate::util::{output, program_out_of_date, t}; -use build_helper::ci::CiEnv; -use build_helper::git::get_git_modified_files; -use ignore::WalkBuilder; -use std::collections::VecDeque; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::sync::mpsc::SyncSender; - -fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut(bool) -> bool { - let mut cmd = Command::new(&rustfmt); - // avoid the submodule config paths from coming into play, - // we only allow a single global config for the workspace for now - cmd.arg("--config-path").arg(&src.canonicalize().unwrap()); - cmd.arg("--edition").arg("2021"); - cmd.arg("--unstable-features"); - cmd.arg("--skip-children"); - if check { - cmd.arg("--check"); - } - cmd.args(paths); - let cmd_debug = format!("{cmd:?}"); - let mut cmd = cmd.spawn().expect("running rustfmt"); - // poor man's async: return a closure that'll wait for rustfmt's completion - move |block: bool| -> bool { - if !block { - match cmd.try_wait() { - Ok(Some(_)) => {} - _ => return false, - } - } - let status = cmd.wait().unwrap(); - if !status.success() { - eprintln!( - "Running `{}` failed.\nIf you're running `tidy`, \ - try again with `--bless`. Or, if you just want to format \ - code, run `./x.py fmt` instead.", - cmd_debug, - ); - crate::exit!(1); - } - true - } -} - -fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, PathBuf)> { - let stamp_file = build.out.join("rustfmt.stamp"); - - let mut cmd = Command::new(match build.initial_rustfmt() { - Some(p) => p, - None => return None, - }); - cmd.arg("--version"); - let output = match cmd.output() { - Ok(status) => status, - Err(_) => return None, - }; - if !output.status.success() { - return None; - } - Some((String::from_utf8(output.stdout).unwrap(), stamp_file)) -} - -/// Return whether the format cache can be reused. -fn verify_rustfmt_version(build: &Builder<'_>) -> bool { - let Some((version, stamp_file)) = get_rustfmt_version(build) else { - return false; - }; - !program_out_of_date(&stamp_file, &version) -} - -/// Updates the last rustfmt version used -fn update_rustfmt_version(build: &Builder<'_>) { - let Some((version, stamp_file)) = get_rustfmt_version(build) else { - return; - }; - t!(std::fs::write(stamp_file, version)) -} - -/// Returns the Rust files modified between the `merge-base` of HEAD and -/// rust-lang/master and what is now on the disk. -/// -/// Returns `None` if all files should be formatted. -fn get_modified_rs_files(build: &Builder<'_>) -> Result>, String> { - if !verify_rustfmt_version(build) { - return Ok(None); - } - - get_git_modified_files(Some(&build.config.src), &vec!["rs"]) -} - -#[derive(serde_derive::Deserialize)] -struct RustfmtConfig { - ignore: Vec, -} - -pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { - if build.config.dry_run() { - return; - } - let mut builder = ignore::types::TypesBuilder::new(); - builder.add_defaults(); - builder.select("rust"); - let matcher = builder.build().unwrap(); - let rustfmt_config = build.src.join("rustfmt.toml"); - if !rustfmt_config.exists() { - eprintln!("Not running formatting checks; rustfmt.toml does not exist."); - eprintln!("This may happen in distributed tarballs."); - return; - } - let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config)); - let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config)); - let mut fmt_override = ignore::overrides::OverrideBuilder::new(&build.src); - for ignore in rustfmt_config.ignore { - fmt_override.add(&format!("!{ignore}")).expect(&ignore); - } - let git_available = match Command::new("git") - .arg("--version") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - { - Ok(status) => status.success(), - Err(_) => false, - }; - - if git_available { - let in_working_tree = match build - .config - .git() - .arg("rev-parse") - .arg("--is-inside-work-tree") - .stdout(Stdio::null()) - .stderr(Stdio::null()) - .status() - { - Ok(status) => status.success(), - Err(_) => false, - }; - if in_working_tree { - let untracked_paths_output = output( - build.config.git().arg("status").arg("--porcelain").arg("--untracked-files=normal"), - ); - let untracked_paths = untracked_paths_output - .lines() - .filter(|entry| entry.starts_with("??")) - .map(|entry| { - entry.split(' ').nth(1).expect("every git status entry should list a path") - }); - let mut untracked_count = 0; - for untracked_path in untracked_paths { - println!("skip untracked path {untracked_path} during rustfmt invocations"); - // The leading `/` makes it an exact match against the - // repository root, rather than a glob. Without that, if you - // have `foo.rs` in the repository root it will also match - // against anything like `compiler/rustc_foo/src/foo.rs`, - // preventing the latter from being formatted. - untracked_count += 1; - fmt_override.add(&format!("!/{untracked_path}")).expect(&untracked_path); - } - // Only check modified files locally to speed up runtime. - // We still check all files in CI to avoid bugs in `get_modified_rs_files` letting regressions slip through; - // we also care about CI time less since this is still very fast compared to building the compiler. - if !CiEnv::is_ci() && paths.is_empty() { - match get_modified_rs_files(build) { - Ok(Some(files)) => { - if files.len() <= 10 { - for file in &files { - println!("formatting modified file {file}"); - } - } else { - let pluralized = |count| if count > 1 { "files" } else { "file" }; - let untracked_msg = if untracked_count == 0 { - "".to_string() - } else { - format!( - ", skipped {} untracked {}", - untracked_count, - pluralized(untracked_count), - ) - }; - println!( - "formatting {} modified {}{}", - files.len(), - pluralized(files.len()), - untracked_msg - ); - } - for file in files { - fmt_override.add(&format!("/{file}")).expect(&file); - } - } - Ok(None) => {} - Err(err) => { - println!( - "WARN: Something went wrong when running git commands:\n{err}\n\ - Falling back to formatting all files." - ); - } - } - } - } else { - println!("Not in git tree. Skipping git-aware format checks"); - } - } else { - println!("Could not find usable git. Skipping git-aware format checks"); - } - - let fmt_override = fmt_override.build().unwrap(); - - let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { - eprintln!("./x.py fmt is not supported on this channel"); - crate::exit!(1); - }); - assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); - let src = build.src.clone(); - let (tx, rx): (SyncSender, _) = std::sync::mpsc::sync_channel(128); - let walker = match paths.get(0) { - Some(first) => { - let find_shortcut_candidates = |p: &PathBuf| { - let mut candidates = Vec::new(); - for candidate in WalkBuilder::new(src.clone()).max_depth(Some(3)).build() { - if let Ok(entry) = candidate { - if let Some(dir_name) = p.file_name() { - if entry.path().is_dir() && entry.file_name() == dir_name { - candidates.push(entry.into_path()); - } - } - } - } - candidates - }; - - // Only try to look for shortcut candidates for single component paths like - // `std` and not for e.g. relative paths like `../library/std`. - let should_look_for_shortcut_dir = |p: &PathBuf| p.components().count() == 1; - - let mut walker = if should_look_for_shortcut_dir(first) { - if let [single_candidate] = &find_shortcut_candidates(first)[..] { - WalkBuilder::new(single_candidate) - } else { - WalkBuilder::new(first) - } - } else { - WalkBuilder::new(src.join(first)) - }; - - for path in &paths[1..] { - if should_look_for_shortcut_dir(path) { - if let [single_candidate] = &find_shortcut_candidates(path)[..] { - walker.add(single_candidate); - } else { - walker.add(path); - } - } else { - walker.add(src.join(path)); - } - } - - walker - } - None => WalkBuilder::new(src.clone()), - } - .types(matcher) - .overrides(fmt_override) - .build_parallel(); - - // there is a lot of blocking involved in spawning a child process and reading files to format. - // spawn more processes than available concurrency to keep the CPU busy - let max_processes = build.jobs() as usize * 2; - - // spawn child processes on a separate thread so we can batch entries we have received from ignore - let thread = std::thread::spawn(move || { - let mut children = VecDeque::new(); - while let Ok(path) = rx.recv() { - // try getting a few more paths from the channel to amortize the overhead of spawning processes - let paths: Vec<_> = rx.try_iter().take(7).chain(std::iter::once(path)).collect(); - - let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check); - children.push_back(child); - - // poll completion before waiting - for i in (0..children.len()).rev() { - if children[i](false) { - children.swap_remove_back(i); - break; - } - } - - if children.len() >= max_processes { - // await oldest child - children.pop_front().unwrap()(true); - } - } - - // await remaining children - for mut child in children { - child(true); - } - }); - - walker.run(|| { - let tx = tx.clone(); - Box::new(move |entry| { - let entry = t!(entry); - if entry.file_type().map_or(false, |t| t.is_file()) { - t!(tx.send(entry.into_path())); - } - ignore::WalkState::Continue - }) - }); - - drop(tx); - - thread.join().unwrap(); - if !check { - update_rustfmt_version(build); - } -} diff --git a/src/bootstrap/install.rs b/src/bootstrap/install.rs deleted file mode 100644 index 500b20b86..000000000 --- a/src/bootstrap/install.rs +++ /dev/null @@ -1,320 +0,0 @@ -//! Implementation of the install aspects of the compiler. -//! -//! This module is responsible for installing the standard library, -//! compiler, and documentation. - -use std::env; -use std::fs; -use std::path::{Component, Path, PathBuf}; -use std::process::Command; - -use crate::util::t; - -use crate::dist; -use crate::tarball::GeneratedTarball; -use crate::{Compiler, Kind}; - -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::config::{Config, TargetSelection}; - -#[cfg(target_os = "illumos")] -const SHELL: &str = "bash"; -#[cfg(not(target_os = "illumos"))] -const SHELL: &str = "sh"; - -// We have to run a few shell scripts, which choke quite a bit on both `\` -// characters and on `C:\` paths, so normalize both of them away. -fn sanitize_sh(path: &Path) -> String { - let path = path.to_str().unwrap().replace("\\", "/"); - return change_drive(unc_to_lfs(&path)).unwrap_or(path); - - fn unc_to_lfs(s: &str) -> &str { - s.strip_prefix("//?/").unwrap_or(s) - } - - fn change_drive(s: &str) -> Option { - let mut ch = s.chars(); - let drive = ch.next().unwrap_or('C'); - if ch.next() != Some(':') { - return None; - } - if ch.next() != Some('/') { - return None; - } - Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..])) - } -} - -fn is_dir_writable_for_user(dir: &PathBuf) -> bool { - let tmp = dir.join(".tmp"); - match fs::create_dir_all(&tmp) { - Ok(_) => { - fs::remove_dir_all(tmp).unwrap(); - true - } - Err(e) => { - if e.kind() == std::io::ErrorKind::PermissionDenied { - false - } else { - panic!("Failed the write access check for the current user. {}", e); - } - } - } -} - -fn install_sh( - builder: &Builder<'_>, - package: &str, - stage: u32, - host: Option, - tarball: &GeneratedTarball, -) { - let _guard = builder.msg(Kind::Install, stage, package, host, host); - - let prefix = default_path(&builder.config.prefix, "/usr/local"); - let sysconfdir = prefix.join(default_path(&builder.config.sysconfdir, "/etc")); - let destdir_env = env::var_os("DESTDIR").map(PathBuf::from); - - // Sanity checks on the write access of user. - // - // When the `DESTDIR` environment variable is present, there is no point to - // check write access for `prefix` and `sysconfdir` individually, as they - // are combined with the path from the `DESTDIR` environment variable. In - // this case, we only need to check the `DESTDIR` path, disregarding the - // `prefix` and `sysconfdir` paths. - if let Some(destdir) = &destdir_env { - assert!(is_dir_writable_for_user(destdir), "User doesn't have write access on DESTDIR."); - } else { - assert!( - is_dir_writable_for_user(&prefix), - "User doesn't have write access on `install.prefix` path in the `config.toml`.", - ); - assert!( - is_dir_writable_for_user(&sysconfdir), - "User doesn't have write access on `install.sysconfdir` path in `config.toml`." - ); - } - - let datadir = prefix.join(default_path(&builder.config.datadir, "share")); - let docdir = prefix.join(default_path(&builder.config.docdir, "share/doc/rust")); - let mandir = prefix.join(default_path(&builder.config.mandir, "share/man")); - let libdir = prefix.join(default_path(&builder.config.libdir, "lib")); - let bindir = prefix.join(&builder.config.bindir); // Default in config.rs - - let empty_dir = builder.out.join("tmp/empty_dir"); - t!(fs::create_dir_all(&empty_dir)); - - let mut cmd = Command::new(SHELL); - cmd.current_dir(&empty_dir) - .arg(sanitize_sh(&tarball.decompressed_output().join("install.sh"))) - .arg(format!("--prefix={}", prepare_dir(&destdir_env, prefix))) - .arg(format!("--sysconfdir={}", prepare_dir(&destdir_env, sysconfdir))) - .arg(format!("--datadir={}", prepare_dir(&destdir_env, datadir))) - .arg(format!("--docdir={}", prepare_dir(&destdir_env, docdir))) - .arg(format!("--bindir={}", prepare_dir(&destdir_env, bindir))) - .arg(format!("--libdir={}", prepare_dir(&destdir_env, libdir))) - .arg(format!("--mandir={}", prepare_dir(&destdir_env, mandir))) - .arg("--disable-ldconfig"); - builder.run(&mut cmd); - t!(fs::remove_dir_all(&empty_dir)); -} - -fn default_path(config: &Option, default: &str) -> PathBuf { - config.as_ref().cloned().unwrap_or_else(|| PathBuf::from(default)) -} - -fn prepare_dir(destdir_env: &Option, mut path: PathBuf) -> String { - // The DESTDIR environment variable is a standard way to install software in a subdirectory - // while keeping the original directory structure, even if the prefix or other directories - // contain absolute paths. - // - // More information on the environment variable is available here: - // https://www.gnu.org/prep/standards/html_node/DESTDIR.html - if let Some(destdir) = destdir_env { - let without_destdir = path.clone(); - path = destdir.clone(); - // Custom .join() which ignores disk roots. - for part in without_destdir.components() { - if let Component::Normal(s) = part { - path.push(s) - } - } - } - - // The installation command is not executed from the current directory, but from a temporary - // directory. To prevent relative paths from breaking this converts relative paths to absolute - // paths. std::fs::canonicalize is not used as that requires the path to actually be present. - if path.is_relative() { - path = std::env::current_dir().expect("failed to get the current directory").join(path); - assert!(path.is_absolute(), "could not make the path relative"); - } - - sanitize_sh(&path) -} - -macro_rules! install { - (($sel:ident, $builder:ident, $_config:ident), - $($name:ident, - $condition_name: ident = $path_or_alias: literal, - $default_cond:expr, - only_hosts: $only_hosts:expr, - $run_item:block $(, $c:ident)*;)+) => { - $( - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl $name { - #[allow(dead_code)] - fn should_build(config: &Config) -> bool { - config.extended && config.tools.as_ref() - .map_or(true, |t| t.contains($path_or_alias)) - } - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = $only_hosts; - $(const $c: bool = true;)* - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let $_config = &run.builder.config; - run.$condition_name($path_or_alias).default_condition($default_cond) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run($sel, $builder: &Builder<'_>) { - $run_item - } - })+ - } -} - -install!((self, builder, _config), - Docs, path = "src/doc", _config.docs, only_hosts: false, { - let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs"); - install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); - }; - Std, path = "library/std", true, only_hosts: false, { - for target in &builder.targets { - // `expect` should be safe, only None when host != build, but this - // only runs when host == build - let tarball = builder.ensure(dist::Std { - compiler: self.compiler, - target: *target - }).expect("missing std"); - install_sh(builder, "std", self.compiler.stage, Some(*target), &tarball); - } - }; - Cargo, alias = "cargo", Self::should_build(_config), only_hosts: true, { - let tarball = builder - .ensure(dist::Cargo { compiler: self.compiler, target: self.target }) - .expect("missing cargo"); - install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball); - }; - RustAnalyzer, alias = "rust-analyzer", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = - builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }) - { - install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install rust-analyzer stage{} ({})", self.compiler.stage, self.target), - ); - } - }; - Clippy, alias = "clippy", Self::should_build(_config), only_hosts: true, { - let tarball = builder - .ensure(dist::Clippy { compiler: self.compiler, target: self.target }) - .expect("missing clippy"); - install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball); - }; - Miri, alias = "miri", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::Miri { compiler: self.compiler, target: self.target }) { - install_sh(builder, "miri", self.compiler.stage, Some(self.target), &tarball); - } else { - // Miri is only available on nightly - builder.info( - &format!("skipping Install miri stage{} ({})", self.compiler.stage, self.target), - ); - } - }; - LlvmTools, alias = "llvm-tools", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::LlvmTools { target: self.target }) { - install_sh(builder, "llvm-tools", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping llvm-tools stage{} ({}): external LLVM", self.compiler.stage, self.target), - ); - } - }; - Rustfmt, alias = "rustfmt", Self::should_build(_config), only_hosts: true, { - if let Some(tarball) = builder.ensure(dist::Rustfmt { - compiler: self.compiler, - target: self.target - }) { - install_sh(builder, "rustfmt", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install Rustfmt stage{} ({})", self.compiler.stage, self.target), - ); - } - }; - RustDemangler, alias = "rust-demangler", Self::should_build(_config), only_hosts: true, { - // Note: Even though `should_build` may return true for `extended` default tools, - // dist::RustDemangler may still return None, unless the target-dependent `profiler` config - // is also true, or the `tools` array explicitly includes "rust-demangler". - if let Some(tarball) = builder.ensure(dist::RustDemangler { - compiler: self.compiler, - target: self.target - }) { - install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball); - } else { - builder.info( - &format!("skipping Install RustDemangler stage{} ({})", - self.compiler.stage, self.target), - ); - } - }; - Rustc, path = "compiler/rustc", true, only_hosts: true, { - let tarball = builder.ensure(dist::Rustc { - compiler: builder.compiler(builder.top_stage, self.target), - }); - install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball); - }; -); - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Src { - pub stage: u32, -} - -impl Step for Src { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let config = &run.builder.config; - let cond = config.extended && config.tools.as_ref().map_or(true, |t| t.contains("src")); - run.path("src").default_condition(cond) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Src { stage: run.builder.top_stage }); - } - - fn run(self, builder: &Builder<'_>) { - let tarball = builder.ensure(dist::Src); - install_sh(builder, "src", self.stage, None, &tarball); - } -} diff --git a/src/bootstrap/job.rs b/src/bootstrap/job.rs deleted file mode 100644 index b0a97b540..000000000 --- a/src/bootstrap/job.rs +++ /dev/null @@ -1,143 +0,0 @@ -//! Job management on Windows for bootstrapping -//! -//! Most of the time when you're running a build system (e.g., make) you expect -//! Ctrl-C or abnormal termination to actually terminate the entire tree of -//! process in play, not just the one at the top. This currently works "by -//! default" on Unix platforms because Ctrl-C actually sends a signal to the -//! *process group* rather than the parent process, so everything will get torn -//! down. On Windows, however, this does not happen and Ctrl-C just kills the -//! parent process. -//! -//! To achieve the same semantics on Windows we use Job Objects to ensure that -//! all processes die at the same time. Job objects have a mode of operation -//! where when all handles to the object are closed it causes all child -//! processes associated with the object to be terminated immediately. -//! Conveniently whenever a process in the job object spawns a new process the -//! child will be associated with the job object as well. This means if we add -//! ourselves to the job object we create then everything will get torn down! -//! -//! Unfortunately most of the time the build system is actually called from a -//! python wrapper (which manages things like building the build system) so this -//! all doesn't quite cut it so far. To go the last mile we duplicate the job -//! object handle into our parent process (a python process probably) and then -//! close our own handle. This means that the only handle to the job object -//! resides in the parent python process, so when python dies the whole build -//! system dies (as one would probably expect!). -//! -//! Note that this module has a #[cfg(windows)] above it as none of this logic -//! is required on Unix. - -use crate::Build; -use std::env; -use std::ffi::c_void; -use std::io; -use std::mem; - -use windows::{ - core::PCWSTR, - Win32::Foundation::{CloseHandle, DuplicateHandle, DUPLICATE_SAME_ACCESS, HANDLE}, - Win32::System::Diagnostics::Debug::{SetErrorMode, SEM_NOGPFAULTERRORBOX, THREAD_ERROR_MODE}, - Win32::System::JobObjects::{ - AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, - SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, - JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, - }, - Win32::System::Threading::{ - GetCurrentProcess, OpenProcess, BELOW_NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, - }, -}; - -pub unsafe fn setup(build: &mut Build) { - // Enable the Windows Error Reporting dialog which msys disables, - // so we can JIT debug rustc - let mode = SetErrorMode(THREAD_ERROR_MODE::default()); - let mode = THREAD_ERROR_MODE(mode); - SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); - - // Create a new job object for us to use - let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); - - // Indicate that when all handles to the job object are gone that all - // process in the object should be killed. Note that this includes our - // entire process tree by default because we've added ourselves and our - // children will reside in the job by default. - let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); - info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; - if build.config.low_priority { - info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; - info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; - } - let r = SetInformationJobObject( - job, - JobObjectExtendedLimitInformation, - &info as *const _ as *const c_void, - mem::size_of_val(&info) as u32, - ) - .ok(); - assert!(r.is_ok(), "{}", io::Error::last_os_error()); - - // Assign our process to this job object. Note that if this fails, one very - // likely reason is that we are ourselves already in a job object! This can - // happen on the build bots that we've got for Windows, or if just anyone - // else is instrumenting the build. In this case we just bail out - // immediately and assume that they take care of it. - // - // Also note that nested jobs (why this might fail) are supported in recent - // versions of Windows, but the version of Windows that our bots are running - // at least don't support nested job objects. - let r = AssignProcessToJobObject(job, GetCurrentProcess()).ok(); - if r.is_err() { - CloseHandle(job); - return; - } - - // If we've got a parent process (e.g., the python script that called us) - // then move ownership of this job object up to them. That way if the python - // script is killed (e.g., via ctrl-c) then we'll all be torn down. - // - // If we don't have a parent (e.g., this was run directly) then we - // intentionally leak the job object handle. When our process exits - // (normally or abnormally) it will close the handle implicitly, causing all - // processes in the job to be cleaned up. - let pid = match env::var("BOOTSTRAP_PARENT_ID") { - Ok(s) => s, - Err(..) => return, - }; - - let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { - Some(parent) => parent, - _ => { - // If we get a null parent pointer here, it is possible that either - // we have an invalid pid or the parent process has been closed. - // Since the first case rarely happens - // (only when wrongly setting the environmental variable), - // it might be better to improve the experience of the second case - // when users have interrupted the parent process and we haven't finish - // duplicating the handle yet. We just need close the job object if that occurs. - CloseHandle(job); - return; - } - }; - - let mut parent_handle = HANDLE::default(); - let r = DuplicateHandle( - GetCurrentProcess(), - job, - parent, - &mut parent_handle, - 0, - false, - DUPLICATE_SAME_ACCESS, - ) - .ok(); - - // If this failed, well at least we tried! An example of DuplicateHandle - // failing in the past has been when the wrong python2 package spawned this - // build system (e.g., the `python2` package in MSYS instead of - // `mingw-w64-x86_64-python2`). Not sure why it failed, but the "failure - // mode" here is that we only clean everything up when the build system - // dies, not when the python parent does, so not too bad. - if r.is_err() { - CloseHandle(job); - } -} diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs deleted file mode 100644 index 8b8d4b237..000000000 --- a/src/bootstrap/lib.rs +++ /dev/null @@ -1,1846 +0,0 @@ -//! Implementation of rustbuild, the Rust build system. -//! -//! This module, and its descendants, are the implementation of the Rust build -//! system. Most of this build system is backed by Cargo but the outer layer -//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo -//! builds, building artifacts like LLVM, etc. The goals of rustbuild are: -//! -//! * To be an easily understandable, easily extensible, and maintainable build -//! system. -//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka -//! crates.io and Cargo. -//! * A standard interface to build across all platforms, including MSVC -//! -//! ## Further information -//! -//! More documentation can be found in each respective module below, and you can -//! also check out the `src/bootstrap/README.md` file for more information. - -use std::cell::{Cell, RefCell}; -use std::collections::{HashMap, HashSet}; -use std::env; -use std::fmt::Display; -use std::fs::{self, File}; -use std::io; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::str; - -use build_helper::ci::{gha, CiEnv}; -use build_helper::exit; -use channel::GitInfo; -use config::{DryRun, Target}; -use filetime::FileTime; -use once_cell::sync::OnceCell; - -use crate::builder::Kind; -use crate::config::{LlvmLibunwind, TargetSelection}; -use crate::util::{ - dir_is_empty, exe, libdir, mtime, output, run, run_suppressed, symlink_dir, try_run_suppressed, -}; - -mod builder; -mod cache; -mod cc_detect; -mod channel; -mod check; -mod clean; -mod compile; -mod config; -mod dist; -mod doc; -mod download; -mod flags; -mod format; -mod install; -mod llvm; -mod metadata; -mod render_tests; -mod run; -mod sanity; -mod setup; -mod suggest; -mod synthetic_targets; -mod tarball; -mod test; -mod tool; -mod toolstate; -pub mod util; - -#[cfg(feature = "build-metrics")] -mod metrics; - -#[cfg(windows)] -mod job; - -#[cfg(all(unix, not(target_os = "haiku")))] -mod job { - pub unsafe fn setup(build: &mut crate::Build) { - if build.config.low_priority { - libc::setpriority(libc::PRIO_PGRP as _, 0, 10); - } - } -} - -#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] -mod job { - pub unsafe fn setup(_build: &mut crate::Build) {} -} - -pub use crate::builder::PathSet; -use crate::cache::{Interned, INTERNER}; -pub use crate::config::Config; -pub use crate::flags::Subcommand; -use termcolor::{ColorChoice, StandardStream, WriteColor}; - -const LLVM_TOOLS: &[&str] = &[ - "llvm-cov", // used to generate coverage report - "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility - "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume - "llvm-objdump", // used to disassemble programs - "llvm-profdata", // used to inspect and merge files generated by profiles - "llvm-readobj", // used to get information from ELFs/objects that the other tools don't provide - "llvm-size", // used to prints the size of the linker sections of a program - "llvm-strip", // used to discard symbols from binary files to reduce their size - "llvm-ar", // used for creating and modifying archive files - "llvm-as", // used to convert LLVM assembly to LLVM bitcode - "llvm-dis", // used to disassemble LLVM bitcode - "llc", // used to compile LLVM bytecode - "opt", // used to optimize LLVM bytecode -]; - -/// LLD file names for all flavors. -const LLD_FILE_NAMES: &[&str] = &["ld.lld", "ld64.lld", "lld-link", "wasm-ld"]; - -pub const VERSION: usize = 2; - -/// Extra --check-cfg to add when building -/// (Mode restriction, config name, config values (if any)) -const EXTRA_CHECK_CFGS: &[(Option, &str, Option<&[&'static str]>)] = &[ - (None, "bootstrap", None), - (Some(Mode::Rustc), "parallel_compiler", None), - (Some(Mode::ToolRustc), "parallel_compiler", None), - (Some(Mode::Codegen), "parallel_compiler", None), - (Some(Mode::Std), "stdarch_intel_sde", None), - (Some(Mode::Std), "no_fp_fmt_parse", None), - (Some(Mode::Std), "no_global_oom_handling", None), - (Some(Mode::Std), "no_rc", None), - (Some(Mode::Std), "no_sync", None), - (Some(Mode::Std), "freebsd12", None), - (Some(Mode::Std), "freebsd13", None), - (Some(Mode::Std), "backtrace_in_libstd", None), - /* Extra values not defined in the built-in targets yet, but used in std */ - // #[cfg(bootstrap)] - (Some(Mode::Std), "target_vendor", Some(&["unikraft"])), - (Some(Mode::Std), "target_env", Some(&["libnx"])), - // #[cfg(bootstrap)] hurd - (Some(Mode::Std), "target_os", Some(&["teeos", "hurd"])), - (Some(Mode::Rustc), "target_os", Some(&["hurd"])), - // #[cfg(bootstrap)] mips32r6, mips64r6 - ( - Some(Mode::Std), - "target_arch", - Some(&["asmjs", "spirv", "nvptx", "xtensa", "mips32r6", "mips64r6", "csky"]), - ), - /* Extra names used by dependencies */ - // FIXME: Used by serde_json, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "no_btreemap_remove_entry", None), - (Some(Mode::ToolRustc), "no_btreemap_remove_entry", None), - // FIXME: Used by crossbeam-utils, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "crossbeam_loom", None), - (Some(Mode::ToolRustc), "crossbeam_loom", None), - // FIXME: Used by proc-macro2, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "span_locations", None), - (Some(Mode::ToolRustc), "span_locations", None), - // FIXME: Used by rustix, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "rustix_use_libc", None), - (Some(Mode::ToolRustc), "rustix_use_libc", None), - // FIXME: Used by filetime, but we should not be triggering on external dependencies. - (Some(Mode::Rustc), "emulate_second_only_system", None), - (Some(Mode::ToolRustc), "emulate_second_only_system", None), - // Needed to avoid the need to copy windows.lib into the sysroot. - (Some(Mode::Rustc), "windows_raw_dylib", None), - (Some(Mode::ToolRustc), "windows_raw_dylib", None), -]; - -/// A structure representing a Rust compiler. -/// -/// Each compiler has a `stage` that it is associated with and a `host` that -/// corresponds to the platform the compiler runs on. This structure is used as -/// a parameter to many methods below. -#[derive(Eq, PartialOrd, Ord, PartialEq, Clone, Copy, Hash, Debug)] -pub struct Compiler { - stage: u32, - host: TargetSelection, -} - -#[derive(PartialEq, Eq, Copy, Clone, Debug)] -pub enum DocTests { - /// Run normal tests and doc tests (default). - Yes, - /// Do not run any doc tests. - No, - /// Only run doc tests. - Only, -} - -pub enum GitRepo { - Rustc, - Llvm, -} - -/// Global configuration for the build system. -/// -/// This structure transitively contains all configuration for the build system. -/// All filesystem-encoded configuration is in `config`, all flags are in -/// `flags`, and then parsed or probed information is listed in the keys below. -/// -/// This structure is a parameter of almost all methods in the build system, -/// although most functions are implemented as free functions rather than -/// methods specifically on this structure itself (to make it easier to -/// organize). -#[derive(Clone)] -pub struct Build { - /// User-specified configuration from `config.toml`. - config: Config, - - // Version information - version: String, - - // Properties derived from the above configuration - src: PathBuf, - out: PathBuf, - bootstrap_out: PathBuf, - cargo_info: channel::GitInfo, - rust_analyzer_info: channel::GitInfo, - clippy_info: channel::GitInfo, - miri_info: channel::GitInfo, - rustfmt_info: channel::GitInfo, - in_tree_llvm_info: channel::GitInfo, - local_rebuild: bool, - fail_fast: bool, - doc_tests: DocTests, - verbosity: usize, - - // Targets for which to build - build: TargetSelection, - hosts: Vec, - targets: Vec, - - initial_rustc: PathBuf, - initial_cargo: PathBuf, - initial_lld: PathBuf, - initial_libdir: PathBuf, - initial_sysroot: PathBuf, - - // Runtime state filled in later on - // C/C++ compilers and archiver for all targets - cc: RefCell>, - cxx: RefCell>, - ar: RefCell>, - ranlib: RefCell>, - // Miscellaneous - // allow bidirectional lookups: both name -> path and path -> name - crates: HashMap, Crate>, - crate_paths: HashMap>, - is_sudo: bool, - ci_env: CiEnv, - delayed_failures: RefCell>, - prerelease_version: Cell>, - - #[cfg(feature = "build-metrics")] - metrics: metrics::BuildMetrics, -} - -#[derive(Debug, Clone)] -struct Crate { - name: Interned, - deps: HashSet>, - path: PathBuf, - has_lib: bool, -} - -impl Crate { - fn local_path(&self, build: &Build) -> PathBuf { - self.path.strip_prefix(&build.config.src).unwrap().into() - } -} - -/// When building Rust various objects are handled differently. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum DependencyType { - /// Libraries originating from proc-macros. - Host, - /// Typical Rust libraries. - Target, - /// Non Rust libraries and objects shipped to ease usage of certain targets. - TargetSelfContained, -} - -/// The various "modes" of invoking Cargo. -/// -/// These entries currently correspond to the various output directories of the -/// build system, with each mod generating output in a different directory. -#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] -pub enum Mode { - /// Build the standard library, placing output in the "stageN-std" directory. - Std, - - /// Build librustc, and compiler libraries, placing output in the "stageN-rustc" directory. - Rustc, - - /// Build a codegen backend for rustc, placing the output in the "stageN-codegen" directory. - Codegen, - - /// Build a tool, placing output in the "stage0-bootstrap-tools" - /// directory. This is for miscellaneous sets of tools that are built - /// using the bootstrap stage0 compiler in its entirety (target libraries - /// and all). Typically these tools compile with stable Rust. - ToolBootstrap, - - /// Build a tool which uses the locally built std, placing output in the - /// "stageN-tools" directory. Its usage is quite rare, mainly used by - /// compiletest which needs libtest. - ToolStd, - - /// Build a tool which uses the locally built rustc and the target std, - /// placing the output in the "stageN-tools" directory. This is used for - /// anything that needs a fully functional rustc, such as rustdoc, clippy, - /// cargo, rls, rustfmt, miri, etc. - ToolRustc, -} - -impl Mode { - pub fn is_tool(&self) -> bool { - matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd) - } - - pub fn must_support_dlopen(&self) -> bool { - matches!(self, Mode::Std | Mode::Codegen) - } -} - -pub enum CLang { - C, - Cxx, -} - -macro_rules! forward { - ( $( $fn:ident( $($param:ident: $ty:ty),* ) $( -> $ret:ty)? ),+ $(,)? ) => { - impl Build { - $( fn $fn(&self, $($param: $ty),* ) $( -> $ret)? { - self.config.$fn( $($param),* ) - } )+ - } - } -} - -forward! { - verbose(msg: &str), - is_verbose() -> bool, - create(path: &Path, s: &str), - remove(f: &Path), - tempdir() -> PathBuf, - llvm_link_shared() -> bool, - download_rustc() -> bool, - initial_rustfmt() -> Option, -} - -impl Build { - /// Creates a new set of build configuration from the `flags` on the command - /// line and the filesystem `config`. - /// - /// By default all build output will be placed in the current directory. - pub fn new(mut config: Config) -> Build { - let src = config.src.clone(); - let out = config.out.clone(); - - #[cfg(unix)] - // keep this consistent with the equivalent check in x.py: - // https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/bootstrap.py#L796-L797 - let is_sudo = match env::var_os("SUDO_USER") { - Some(_sudo_user) => { - let uid = unsafe { libc::getuid() }; - uid == 0 - } - None => false, - }; - #[cfg(not(unix))] - let is_sudo = false; - - let omit_git_hash = config.omit_git_hash; - let rust_info = channel::GitInfo::new(omit_git_hash, &src); - let cargo_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); - let rust_analyzer_info = - channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); - let clippy_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); - let miri_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); - let rustfmt_info = channel::GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); - - // we always try to use git for LLVM builds - let in_tree_llvm_info = channel::GitInfo::new(false, &src.join("src/llvm-project")); - - let initial_target_libdir_str = if config.dry_run() { - "/dummy/lib/path/to/lib/".to_string() - } else { - output( - Command::new(&config.initial_rustc) - .arg("--target") - .arg(config.build.rustc_target_arg()) - .arg("--print") - .arg("target-libdir"), - ) - }; - let initial_target_dir = Path::new(&initial_target_libdir_str).parent().unwrap(); - let initial_lld = initial_target_dir.join("bin").join("rust-lld"); - - let initial_sysroot = if config.dry_run() { - "/dummy".to_string() - } else { - output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot")) - } - .trim() - .to_string(); - - let initial_libdir = initial_target_dir - .parent() - .unwrap() - .parent() - .unwrap() - .strip_prefix(&initial_sysroot) - .unwrap() - .to_path_buf(); - - let version = std::fs::read_to_string(src.join("src").join("version")) - .expect("failed to read src/version"); - let version = version.trim(); - - let bootstrap_out = std::env::current_exe() - .expect("could not determine path to running process") - .parent() - .unwrap() - .to_path_buf(); - if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { - // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented - panic!( - "`rustc` not found in {}, run `cargo build --bins` before `cargo run`", - bootstrap_out.display() - ) - } - - if rust_info.is_from_tarball() && config.description.is_none() { - config.description = Some("built from a source tarball".to_owned()); - } - - let mut build = Build { - initial_rustc: config.initial_rustc.clone(), - initial_cargo: config.initial_cargo.clone(), - initial_lld, - initial_libdir, - initial_sysroot: initial_sysroot.into(), - local_rebuild: config.local_rebuild, - fail_fast: config.cmd.fail_fast(), - doc_tests: config.cmd.doc_tests(), - verbosity: config.verbose, - - build: config.build, - hosts: config.hosts.clone(), - targets: config.targets.clone(), - - config, - version: version.to_string(), - src, - out, - bootstrap_out, - - cargo_info, - rust_analyzer_info, - clippy_info, - miri_info, - rustfmt_info, - in_tree_llvm_info, - cc: RefCell::new(HashMap::new()), - cxx: RefCell::new(HashMap::new()), - ar: RefCell::new(HashMap::new()), - ranlib: RefCell::new(HashMap::new()), - crates: HashMap::new(), - crate_paths: HashMap::new(), - is_sudo, - ci_env: CiEnv::current(), - delayed_failures: RefCell::new(Vec::new()), - prerelease_version: Cell::new(None), - - #[cfg(feature = "build-metrics")] - metrics: metrics::BuildMetrics::init(), - }; - - // If local-rust is the same major.minor as the current version, then force a - // local-rebuild - let local_version_verbose = - output(Command::new(&build.initial_rustc).arg("--version").arg("--verbose")); - let local_release = local_version_verbose - .lines() - .filter_map(|x| x.strip_prefix("release:")) - .next() - .unwrap() - .trim(); - if local_release.split('.').take(2).eq(version.split('.').take(2)) { - build.verbose(&format!("auto-detected local-rebuild {local_release}")); - build.local_rebuild = true; - } - - build.verbose("finding compilers"); - cc_detect::find(&build); - // When running `setup`, the profile is about to change, so any requirements we have now may - // be different on the next invocation. Don't check for them until the next time x.py is - // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. - // - // Similarly, for `setup` we don't actually need submodules or cargo metadata. - if !matches!(build.config.cmd, Subcommand::Setup { .. }) { - build.verbose("running sanity check"); - sanity::check(&mut build); - - // Make sure we update these before gathering metadata so we don't get an error about missing - // Cargo.toml files. - let rust_submodules = ["src/tools/cargo", "library/backtrace", "library/stdarch"]; - for s in rust_submodules { - build.update_submodule(Path::new(s)); - } - // Now, update all existing submodules. - build.update_existing_submodules(); - - build.verbose("learning about cargo"); - metadata::build(&mut build); - } - - // Make a symbolic link so we can use a consistent directory in the documentation. - let build_triple = build.out.join(&build.build.triple); - t!(fs::create_dir_all(&build_triple)); - let host = build.out.join("host"); - if host.is_symlink() { - // Left over from a previous build; overwrite it. - // This matters if `build.build` has changed between invocations. - #[cfg(windows)] - t!(fs::remove_dir(&host)); - #[cfg(not(windows))] - t!(fs::remove_file(&host)); - } - t!( - symlink_dir(&build.config, &build_triple, &host), - format!("symlink_dir({} => {}) failed", host.display(), build_triple.display()) - ); - - build - } - - // modified from `check_submodule` and `update_submodule` in bootstrap.py - /// Given a path to the directory of a submodule, update it. - /// - /// `relative_path` should be relative to the root of the git repository, not an absolute path. - pub(crate) fn update_submodule(&self, relative_path: &Path) { - if !self.config.submodules(&self.rust_info()) { - return; - } - - let absolute_path = self.config.src.join(relative_path); - - // NOTE: The check for the empty directory is here because when running x.py the first time, - // the submodule won't be checked out. Check it out now so we can build it. - if !channel::GitInfo::new(false, &absolute_path).is_managed_git_subrepository() - && !dir_is_empty(&absolute_path) - { - return; - } - - // check_submodule - let checked_out_hash = - output(Command::new("git").args(&["rev-parse", "HEAD"]).current_dir(&absolute_path)); - // update_submodules - let recorded = output( - Command::new("git") - .args(&["ls-tree", "HEAD"]) - .arg(relative_path) - .current_dir(&self.config.src), - ); - let actual_hash = recorded - .split_whitespace() - .nth(2) - .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); - - // update_submodule - if actual_hash == checked_out_hash.trim_end() { - // already checked out - return; - } - - println!("Updating submodule {}", relative_path.display()); - self.run( - Command::new("git") - .args(&["submodule", "-q", "sync"]) - .arg(relative_path) - .current_dir(&self.config.src), - ); - - // Try passing `--progress` to start, then run git again without if that fails. - let update = |progress: bool| { - // Git is buggy and will try to fetch submodules from the tracking branch for *this* repository, - // even though that has no relation to the upstream for the submodule. - let current_branch = { - let output = self - .config - .git() - .args(["symbolic-ref", "--short", "HEAD"]) - .stderr(Stdio::inherit()) - .output(); - let output = t!(output); - if output.status.success() { - Some(String::from_utf8(output.stdout).unwrap().trim().to_owned()) - } else { - None - } - }; - - let mut git = self.config.git(); - if let Some(branch) = current_branch { - // If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name. - // This syntax isn't accepted by `branch.{branch}`. Strip it. - let branch = branch.strip_prefix("heads/").unwrap_or(&branch); - git.arg("-c").arg(format!("branch.{branch}.remote=origin")); - } - git.args(&["submodule", "update", "--init", "--recursive", "--depth=1"]); - if progress { - git.arg("--progress"); - } - git.arg(relative_path); - git - }; - // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails. - if !update(true).status().map_or(false, |status| status.success()) { - self.run(&mut update(false)); - } - - // Save any local changes, but avoid running `git stash pop` if there are none (since it will exit with an error). - #[allow(deprecated)] // diff-index reports the modifications through the exit status - let has_local_modifications = self - .config - .try_run( - Command::new("git") - .args(&["diff-index", "--quiet", "HEAD"]) - .current_dir(&absolute_path), - ) - .is_err(); - if has_local_modifications { - self.run(Command::new("git").args(&["stash", "push"]).current_dir(&absolute_path)); - } - - self.run(Command::new("git").args(&["reset", "-q", "--hard"]).current_dir(&absolute_path)); - self.run(Command::new("git").args(&["clean", "-qdfx"]).current_dir(&absolute_path)); - - if has_local_modifications { - self.run(Command::new("git").args(&["stash", "pop"]).current_dir(absolute_path)); - } - } - - /// If any submodule has been initialized already, sync it unconditionally. - /// This avoids contributors checking in a submodule change by accident. - pub fn update_existing_submodules(&self) { - // Avoid running git when there isn't a git checkout. - if !self.config.submodules(&self.rust_info()) { - return; - } - let output = output( - self.config - .git() - .args(&["config", "--file"]) - .arg(&self.config.src.join(".gitmodules")) - .args(&["--get-regexp", "path"]), - ); - for line in output.lines() { - // Look for `submodule.$name.path = $path` - // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer` - let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap()); - // Don't update the submodule unless it's already been cloned. - if channel::GitInfo::new(false, submodule).is_managed_git_subrepository() { - self.update_submodule(submodule); - } - } - } - - /// Executes the entire build, as configured by the flags and configuration. - pub fn build(&mut self) { - unsafe { - job::setup(self); - } - - // Download rustfmt early so that it can be used in rust-analyzer configs. - let _ = &builder::Builder::new(&self).initial_rustfmt(); - - // hardcoded subcommands - match &self.config.cmd { - Subcommand::Format { check } => { - return format::format(&builder::Builder::new(&self), *check, &self.config.paths); - } - Subcommand::Suggest { run } => { - return suggest::suggest(&builder::Builder::new(&self), *run); - } - _ => (), - } - - { - let builder = builder::Builder::new(&self); - if let Some(path) = builder.paths.get(0) { - if path == Path::new("nonexistent/path/to/trigger/cargo/metadata") { - return; - } - } - } - - if !self.config.dry_run() { - { - self.config.dry_run = DryRun::SelfCheck; - let builder = builder::Builder::new(&self); - builder.execute_cli(); - } - self.config.dry_run = DryRun::Disabled; - let builder = builder::Builder::new(&self); - builder.execute_cli(); - } else { - let builder = builder::Builder::new(&self); - builder.execute_cli(); - } - - // Check for postponed failures from `test --no-fail-fast`. - let failures = self.delayed_failures.borrow(); - if failures.len() > 0 { - eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); - for failure in failures.iter() { - eprintln!(" - {failure}\n"); - } - exit!(1); - } - - #[cfg(feature = "build-metrics")] - self.metrics.persist(self); - } - - /// Clear out `dir` if `input` is newer. - /// - /// After this executes, it will also ensure that `dir` exists. - fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { - let stamp = dir.join(".stamp"); - let mut cleared = false; - if mtime(&stamp) < mtime(input) { - self.verbose(&format!("Dirty - {}", dir.display())); - let _ = fs::remove_dir_all(dir); - cleared = true; - } else if stamp.exists() { - return cleared; - } - t!(fs::create_dir_all(dir)); - t!(File::create(stamp)); - cleared - } - - fn rust_info(&self) -> &GitInfo { - &self.config.rust_info - } - - /// Gets the space-separated set of activated features for the standard - /// library. - fn std_features(&self, target: TargetSelection) -> String { - let mut features = " panic-unwind".to_string(); - - match self.config.llvm_libunwind(target) { - LlvmLibunwind::InTree => features.push_str(" llvm-libunwind"), - LlvmLibunwind::System => features.push_str(" system-llvm-libunwind"), - LlvmLibunwind::No => {} - } - if self.config.backtrace { - features.push_str(" backtrace"); - } - if self.config.profiler_enabled(target) { - features.push_str(" profiler"); - } - features - } - - /// Gets the space-separated set of activated features for the compiler. - fn rustc_features(&self, kind: Kind) -> String { - let mut features = vec![]; - if self.config.jemalloc { - features.push("jemalloc"); - } - if self.config.llvm_enabled() || kind == Kind::Check { - features.push("llvm"); - } - // keep in sync with `bootstrap/compile.rs:rustc_cargo_env` - if self.config.rustc_parallel { - features.push("rustc_use_parallel_compiler"); - } - - // If debug logging is on, then we want the default for tracing: - // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26 - // which is everything (including debug/trace/etc.) - // if its unset, if debug_assertions is on, then debug_logging will also be on - // as well as tracing *ignoring* this feature when debug_assertions is on - if !self.config.rust_debug_logging { - features.push("max_level_info"); - } - - features.join(" ") - } - - /// Component directory that Cargo will produce output into (e.g. - /// release/debug) - fn cargo_dir(&self) -> &'static str { - if self.config.rust_optimize.is_release() { "release" } else { "debug" } - } - - fn tools_dir(&self, compiler: Compiler) -> PathBuf { - let out = self - .out - .join(&*compiler.host.triple) - .join(format!("stage{}-tools-bin", compiler.stage)); - t!(fs::create_dir_all(&out)); - out - } - - /// Returns the root directory for all output generated in a particular - /// stage when running with a particular host compiler. - /// - /// The mode indicates what the root directory is for. - fn stage_out(&self, compiler: Compiler, mode: Mode) -> PathBuf { - let suffix = match mode { - Mode::Std => "-std", - Mode::Rustc => "-rustc", - Mode::Codegen => "-codegen", - Mode::ToolBootstrap => "-bootstrap-tools", - Mode::ToolStd | Mode::ToolRustc => "-tools", - }; - self.out.join(&*compiler.host.triple).join(format!("stage{}{}", compiler.stage, suffix)) - } - - /// Returns the root output directory for all Cargo output in a given stage, - /// running a particular compiler, whether or not we're building the - /// standard library, and targeting the specified architecture. - fn cargo_out(&self, compiler: Compiler, mode: Mode, target: TargetSelection) -> PathBuf { - self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir()) - } - - /// Root output directory for LLVM compiled for `target` - /// - /// Note that if LLVM is configured externally then the directory returned - /// will likely be empty. - fn llvm_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("llvm") - } - - fn lld_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("lld") - } - - /// Output directory for all documentation for a target - fn doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("doc") - } - - /// Output directory for all JSON-formatted documentation for a target - fn json_doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("json-doc") - } - - fn test_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("test") - } - - /// Output directory for all documentation for a target - fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("compiler-doc") - } - - /// Output directory for some generated md crate documentation for a target (temporary) - fn md_doc_out(&self, target: TargetSelection) -> Interned { - INTERNER.intern_path(self.out.join(&*target.triple).join("md-doc")) - } - - /// Returns `true` if no custom `llvm-config` is set for the specified target. - /// - /// If no custom `llvm-config` was specified then Rust's llvm will be used. - fn is_rust_llvm(&self, target: TargetSelection) -> bool { - match self.config.target_config.get(&target) { - Some(Target { llvm_has_rust_patches: Some(patched), .. }) => *patched, - Some(Target { llvm_config, .. }) => { - // If the user set llvm-config we assume Rust is not patched, - // but first check to see if it was configured by llvm-from-ci. - (self.config.llvm_from_ci && target == self.config.build) || llvm_config.is_none() - } - None => true, - } - } - - /// Returns the path to `FileCheck` binary for the specified target - fn llvm_filecheck(&self, target: TargetSelection) -> PathBuf { - let target_config = self.config.target_config.get(&target); - if let Some(s) = target_config.and_then(|c| c.llvm_filecheck.as_ref()) { - s.to_path_buf() - } else if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { - let llvm_bindir = output(Command::new(s).arg("--bindir")); - let filecheck = Path::new(llvm_bindir.trim()).join(exe("FileCheck", target)); - if filecheck.exists() { - filecheck - } else { - // On Fedora the system LLVM installs FileCheck in the - // llvm subdirectory of the libdir. - let llvm_libdir = output(Command::new(s).arg("--libdir")); - let lib_filecheck = - Path::new(llvm_libdir.trim()).join("llvm").join(exe("FileCheck", target)); - if lib_filecheck.exists() { - lib_filecheck - } else { - // Return the most normal file name, even though - // it doesn't exist, so that any error message - // refers to that. - filecheck - } - } - } else { - let base = self.llvm_out(target).join("build"); - let base = if !self.ninja() && target.contains("msvc") { - if self.config.llvm_optimize { - if self.config.llvm_release_debuginfo { - base.join("RelWithDebInfo") - } else { - base.join("Release") - } - } else { - base.join("Debug") - } - } else { - base - }; - base.join("bin").join(exe("FileCheck", target)) - } - } - - /// Directory for libraries built from C/C++ code and shared between stages. - fn native_dir(&self, target: TargetSelection) -> PathBuf { - self.out.join(&*target.triple).join("native") - } - - /// Root output directory for rust_test_helpers library compiled for - /// `target` - fn test_helpers_out(&self, target: TargetSelection) -> PathBuf { - self.native_dir(target).join("rust-test-helpers") - } - - /// Adds the `RUST_TEST_THREADS` env var if necessary - fn add_rust_test_threads(&self, cmd: &mut Command) { - if env::var_os("RUST_TEST_THREADS").is_none() { - cmd.env("RUST_TEST_THREADS", self.jobs().to_string()); - } - } - - /// Returns the libdir of the snapshot compiler. - fn rustc_snapshot_libdir(&self) -> PathBuf { - self.rustc_snapshot_sysroot().join(libdir(self.config.build)) - } - - /// Returns the sysroot of the snapshot compiler. - fn rustc_snapshot_sysroot(&self) -> &Path { - static SYSROOT_CACHE: OnceCell = once_cell::sync::OnceCell::new(); - SYSROOT_CACHE.get_or_init(|| { - let mut rustc = Command::new(&self.initial_rustc); - rustc.args(&["--print", "sysroot"]); - output(&mut rustc).trim().into() - }) - } - - /// Runs a command, printing out nice contextual information if it fails. - fn run(&self, cmd: &mut Command) { - if self.config.dry_run() { - return; - } - self.verbose(&format!("running: {cmd:?}")); - run(cmd, self.is_verbose()) - } - - /// Runs a command, printing out nice contextual information if it fails. - fn run_quiet(&self, cmd: &mut Command) { - if self.config.dry_run() { - return; - } - self.verbose(&format!("running: {cmd:?}")); - run_suppressed(cmd) - } - - /// Runs a command, printing out nice contextual information if it fails. - /// Exits if the command failed to execute at all, otherwise returns its - /// `status.success()`. - fn run_quiet_delaying_failure(&self, cmd: &mut Command) -> bool { - if self.config.dry_run() { - return true; - } - if !self.fail_fast { - self.verbose(&format!("running: {cmd:?}")); - if !try_run_suppressed(cmd) { - let mut failures = self.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); - return false; - } - } else { - self.run_quiet(cmd); - } - true - } - - /// Runs a command, printing out contextual info if it fails, and delaying errors until the build finishes. - pub(crate) fn run_delaying_failure(&self, cmd: &mut Command) -> bool { - if !self.fail_fast { - #[allow(deprecated)] // can't use Build::try_run, that's us - if self.config.try_run(cmd).is_err() { - let mut failures = self.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); - return false; - } - } else { - self.run(cmd); - } - true - } - - pub fn is_verbose_than(&self, level: usize) -> bool { - self.verbosity > level - } - - /// Prints a message if this build is configured in more verbose mode than `level`. - fn verbose_than(&self, level: usize, msg: &str) { - if self.is_verbose_than(level) { - println!("{msg}"); - } - } - - fn info(&self, msg: &str) { - match self.config.dry_run { - DryRun::SelfCheck => (), - DryRun::Disabled | DryRun::UserSelected => { - println!("{msg}"); - } - } - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_check( - &self, - what: impl Display, - target: impl Into>, - ) -> Option { - self.msg(Kind::Check, self.config.stage, what, self.config.build, target) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_doc( - &self, - compiler: Compiler, - what: impl Display, - target: impl Into> + Copy, - ) -> Option { - self.msg(Kind::Doc, compiler.stage, what, compiler.host, target.into()) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_build( - &self, - compiler: Compiler, - what: impl Display, - target: impl Into>, - ) -> Option { - self.msg(Kind::Build, compiler.stage, what, compiler.host, target) - } - - /// Return a `Group` guard for a [`Step`] that is built for each `--stage`. - /// - /// [`Step`]: crate::builder::Step - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg( - &self, - action: impl Into, - stage: u32, - what: impl Display, - host: impl Into>, - target: impl Into>, - ) -> Option { - let action = action.into().description(); - let msg = |fmt| format!("{action} stage{stage} {what}{fmt}"); - let msg = if let Some(target) = target.into() { - let host = host.into().unwrap(); - if host == target { - msg(format_args!(" ({target})")) - } else { - msg(format_args!(" ({host} -> {target})")) - } - } else { - msg(format_args!("")) - }; - self.group(&msg) - } - - /// Return a `Group` guard for a [`Step`] that is only built once and isn't affected by `--stage`. - /// - /// [`Step`]: crate::builder::Step - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_unstaged( - &self, - action: impl Into, - what: impl Display, - target: TargetSelection, - ) -> Option { - let action = action.into().description(); - let msg = format!("{action} {what} for {target}"); - self.group(&msg) - } - - #[must_use = "Groups should not be dropped until the Step finishes running"] - #[track_caller] - fn msg_sysroot_tool( - &self, - action: impl Into, - stage: u32, - what: impl Display, - host: TargetSelection, - target: TargetSelection, - ) -> Option { - let action = action.into().description(); - let msg = |fmt| format!("{action} {what} {fmt}"); - let msg = if host == target { - msg(format_args!("(stage{stage} -> stage{}, {target})", stage + 1)) - } else { - msg(format_args!("(stage{stage}:{host} -> stage{}:{target})", stage + 1)) - }; - self.group(&msg) - } - - #[track_caller] - fn group(&self, msg: &str) -> Option { - match self.config.dry_run { - DryRun::SelfCheck => None, - DryRun::Disabled | DryRun::UserSelected => Some(gha::group(&msg)), - } - } - - /// Returns the number of parallel jobs that have been configured for this - /// build. - fn jobs(&self) -> u32 { - self.config.jobs.unwrap_or_else(|| { - std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 - }) - } - - fn debuginfo_map_to(&self, which: GitRepo) -> Option { - if !self.config.rust_remap_debuginfo { - return None; - } - - match which { - GitRepo::Rustc => { - let sha = self.rust_sha().unwrap_or(&self.version); - Some(format!("/rustc/{sha}")) - } - GitRepo::Llvm => Some(String::from("/rustc/llvm")), - } - } - - /// Returns the path to the C compiler for the target specified. - fn cc(&self, target: TargetSelection) -> PathBuf { - if self.config.dry_run() { - return PathBuf::new(); - } - self.cc.borrow()[&target].path().into() - } - - /// Returns a list of flags to pass to the C compiler for the target - /// specified. - fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec { - if self.config.dry_run() { - return Vec::new(); - } - let base = match c { - CLang::C => self.cc.borrow()[&target].clone(), - CLang::Cxx => self.cxx.borrow()[&target].clone(), - }; - - // Filter out -O and /O (the optimization flags) that we picked up from - // cc-rs because the build scripts will determine that for themselves. - let mut base = base - .args() - .iter() - .map(|s| s.to_string_lossy().into_owned()) - .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) - .collect::>(); - - // If we're compiling on macOS then we add a few unconditional flags - // indicating that we want libc++ (more filled out than libstdc++) and - // we want to compile for 10.7. This way we can ensure that - // LLVM/etc are all properly compiled. - if target.contains("apple-darwin") { - base.push("-stdlib=libc++".into()); - } - - // Work around an apparently bad MinGW / GCC optimization, - // See: https://lists.llvm.org/pipermail/cfe-dev/2016-December/051980.html - // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78936 - if &*target.triple == "i686-pc-windows-gnu" { - base.push("-fno-omit-frame-pointer".into()); - } - - if let Some(map_to) = self.debuginfo_map_to(which) { - let map = format!("{}={}", self.src.display(), map_to); - let cc = self.cc(target); - if cc.ends_with("clang") || cc.ends_with("gcc") { - base.push(format!("-fdebug-prefix-map={map}")); - } else if cc.ends_with("clang-cl.exe") { - base.push("-Xclang".into()); - base.push(format!("-fdebug-prefix-map={map}")); - } - } - base - } - - /// Returns the path to the `ar` archive utility for the target specified. - fn ar(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { - return None; - } - self.ar.borrow().get(&target).cloned() - } - - /// Returns the path to the `ranlib` utility for the target specified. - fn ranlib(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { - return None; - } - self.ranlib.borrow().get(&target).cloned() - } - - /// Returns the path to the C++ compiler for the target specified. - fn cxx(&self, target: TargetSelection) -> Result { - if self.config.dry_run() { - return Ok(PathBuf::new()); - } - match self.cxx.borrow().get(&target) { - Some(p) => Ok(p.path().into()), - None => Err(format!("target `{target}` is not configured as a host, only as a target")), - } - } - - /// Returns the path to the linker for the given target if it needs to be overridden. - fn linker(&self, target: TargetSelection) -> Option { - if self.config.dry_run() { - return Some(PathBuf::new()); - } - if let Some(linker) = self.config.target_config.get(&target).and_then(|c| c.linker.clone()) - { - Some(linker) - } else if target.contains("vxworks") { - // need to use CXX compiler as linker to resolve the exception functions - // that are only existed in CXX libraries - Some(self.cxx.borrow()[&target].path().into()) - } else if target != self.config.build - && util::use_host_linker(target) - && !target.contains("msvc") - { - Some(self.cc(target)) - } else if self.config.use_lld && !self.is_fuse_ld_lld(target) && self.build == target { - Some(self.initial_lld.clone()) - } else { - None - } - } - - // LLD is used through `-fuse-ld=lld` rather than directly. - // Only MSVC targets use LLD directly at the moment. - fn is_fuse_ld_lld(&self, target: TargetSelection) -> bool { - self.config.use_lld && !target.contains("msvc") - } - - fn lld_flags(&self, target: TargetSelection) -> impl Iterator { - let mut options = [None, None]; - - if self.config.use_lld { - if self.is_fuse_ld_lld(target) { - options[0] = Some("-Clink-arg=-fuse-ld=lld".to_string()); - } - - let no_threads = util::lld_flag_no_threads(target.contains("windows")); - options[1] = Some(format!("-Clink-arg=-Wl,{no_threads}")); - } - - IntoIterator::into_iter(options).flatten() - } - - /// Returns if this target should statically link the C runtime, if specified - fn crt_static(&self, target: TargetSelection) -> Option { - if target.contains("pc-windows-msvc") { - Some(true) - } else { - self.config.target_config.get(&target).and_then(|t| t.crt_static) - } - } - - /// Returns the "musl root" for this `target`, if defined - fn musl_root(&self, target: TargetSelection) -> Option<&Path> { - self.config - .target_config - .get(&target) - .and_then(|t| t.musl_root.as_ref()) - .or_else(|| self.config.musl_root.as_ref()) - .map(|p| &**p) - } - - /// Returns the "musl libdir" for this `target`. - fn musl_libdir(&self, target: TargetSelection) -> Option { - let t = self.config.target_config.get(&target)?; - if let libdir @ Some(_) = &t.musl_libdir { - return libdir.clone(); - } - self.musl_root(target).map(|root| root.join("lib")) - } - - /// Returns the sysroot for the wasi target, if defined - fn wasi_root(&self, target: TargetSelection) -> Option<&Path> { - self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p) - } - - /// Returns `true` if this is a no-std `target`, if defined - fn no_std(&self, target: TargetSelection) -> Option { - self.config.target_config.get(&target).map(|t| t.no_std) - } - - /// Returns `true` if the target will be tested using the `remote-test-client` - /// and `remote-test-server` binaries. - fn remote_tested(&self, target: TargetSelection) -> bool { - self.qemu_rootfs(target).is_some() - || target.contains("android") - || env::var_os("TEST_DEVICE_ADDR").is_some() - } - - /// Returns the root of the "rootfs" image that this target will be using, - /// if one was configured. - /// - /// If `Some` is returned then that means that tests for this target are - /// emulated with QEMU and binaries will need to be shipped to the emulator. - fn qemu_rootfs(&self, target: TargetSelection) -> Option<&Path> { - self.config.target_config.get(&target).and_then(|t| t.qemu_rootfs.as_ref()).map(|p| &**p) - } - - /// Path to the python interpreter to use - fn python(&self) -> &Path { - if self.config.build.ends_with("apple-darwin") { - // Force /usr/bin/python3 on macOS for LLDB tests because we're loading the - // LLDB plugin's compiled module which only works with the system python - // (namely not Homebrew-installed python) - Path::new("/usr/bin/python3") - } else { - self.config - .python - .as_ref() - .expect("python is required for running LLDB or rustdoc tests") - } - } - - /// Temporary directory that extended error information is emitted to. - fn extended_error_dir(&self) -> PathBuf { - self.out.join("tmp/extended-error-metadata") - } - - /// Tests whether the `compiler` compiling for `target` should be forced to - /// use a stage1 compiler instead. - /// - /// Currently, by default, the build system does not perform a "full - /// bootstrap" by default where we compile the compiler three times. - /// Instead, we compile the compiler two times. The final stage (stage2) - /// just copies the libraries from the previous stage, which is what this - /// method detects. - /// - /// Here we return `true` if: - /// - /// * The build isn't performing a full bootstrap - /// * The `compiler` is in the final stage, 2 - /// * We're not cross-compiling, so the artifacts are already available in - /// stage1 - /// - /// When all of these conditions are met the build will lift artifacts from - /// the previous stage forward. - fn force_use_stage1(&self, stage: u32, target: TargetSelection) -> bool { - !self.config.full_bootstrap - && !self.config.download_rustc() - && stage >= 2 - && (self.hosts.iter().any(|h| *h == target) || target == self.build) - } - - /// Checks whether the `compiler` compiling for `target` should be forced to - /// use a stage2 compiler instead. - /// - /// When we download the pre-compiled version of rustc and compiler stage is >= 2, - /// it should be forced to use a stage2 compiler. - fn force_use_stage2(&self, stage: u32) -> bool { - self.config.download_rustc() && stage >= 2 - } - - /// Given `num` in the form "a.b.c" return a "release string" which - /// describes the release version number. - /// - /// For example on nightly this returns "a.b.c-nightly", on beta it returns - /// "a.b.c-beta.1" and on stable it just returns "a.b.c". - fn release(&self, num: &str) -> String { - match &self.config.channel[..] { - "stable" => num.to_string(), - "beta" => { - if !self.config.omit_git_hash { - format!("{}-beta.{}", num, self.beta_prerelease_version()) - } else { - format!("{num}-beta") - } - } - "nightly" => format!("{num}-nightly"), - _ => format!("{num}-dev"), - } - } - - fn beta_prerelease_version(&self) -> u32 { - fn extract_beta_rev_from_file>(version_file: P) -> Option { - let version = fs::read_to_string(version_file).ok()?; - - extract_beta_rev(&version) - } - - if let Some(s) = self.prerelease_version.get() { - return s; - } - - // First check if there is a version file available. - // If available, we read the beta revision from that file. - // This only happens when building from a source tarball when Git should not be used. - let count = extract_beta_rev_from_file(self.src.join("version")).unwrap_or_else(|| { - // Figure out how many merge commits happened since we branched off master. - // That's our beta number! - // (Note that we use a `..` range, not the `...` symmetric difference.) - output(self.config.git().arg("rev-list").arg("--count").arg("--merges").arg(format!( - "refs/remotes/origin/{}..HEAD", - self.config.stage0_metadata.config.nightly_branch - ))) - }); - let n = count.trim().parse().unwrap(); - self.prerelease_version.set(Some(n)); - n - } - - /// Returns the value of `release` above for Rust itself. - fn rust_release(&self) -> String { - self.release(&self.version) - } - - /// Returns the "package version" for a component given the `num` release - /// number. - /// - /// The package version is typically what shows up in the names of tarballs. - /// For channels like beta/nightly it's just the channel name, otherwise - /// it's the `num` provided. - fn package_vers(&self, num: &str) -> String { - match &self.config.channel[..] { - "stable" => num.to_string(), - "beta" => "beta".to_string(), - "nightly" => "nightly".to_string(), - _ => format!("{num}-dev"), - } - } - - /// Returns the value of `package_vers` above for Rust itself. - fn rust_package_vers(&self) -> String { - self.package_vers(&self.version) - } - - /// Returns the `version` string associated with this compiler for Rust - /// itself. - /// - /// Note that this is a descriptive string which includes the commit date, - /// sha, version, etc. - fn rust_version(&self) -> String { - let mut version = self.rust_info().version(self, &self.version); - if let Some(ref s) = self.config.description { - version.push_str(" ("); - version.push_str(s); - version.push(')'); - } - version - } - - /// Returns the full commit hash. - fn rust_sha(&self) -> Option<&str> { - self.rust_info().sha() - } - - /// Returns the `a.b.c` version that the given package is at. - fn release_num(&self, package: &str) -> String { - let toml_file_name = self.src.join(&format!("src/tools/{package}/Cargo.toml")); - let toml = t!(fs::read_to_string(&toml_file_name)); - for line in toml.lines() { - if let Some(stripped) = - line.strip_prefix("version = \"").and_then(|s| s.strip_suffix("\"")) - { - return stripped.to_owned(); - } - } - - panic!("failed to find version in {package}'s Cargo.toml") - } - - /// Returns `true` if unstable features should be enabled for the compiler - /// we're building. - fn unstable_features(&self) -> bool { - match &self.config.channel[..] { - "stable" | "beta" => false, - "nightly" | _ => true, - } - } - - /// Returns a Vec of all the dependencies of the given root crate, - /// including transitive dependencies and the root itself. Only includes - /// "local" crates (those in the local source tree, not from a registry). - fn in_tree_crates(&self, root: &str, target: Option) -> Vec<&Crate> { - let mut ret = Vec::new(); - let mut list = vec![INTERNER.intern_str(root)]; - let mut visited = HashSet::new(); - while let Some(krate) = list.pop() { - let krate = self - .crates - .get(&krate) - .unwrap_or_else(|| panic!("metadata missing for {krate}: {:?}", self.crates)); - ret.push(krate); - for dep in &krate.deps { - if !self.crates.contains_key(dep) { - // Ignore non-workspace members. - continue; - } - // Don't include optional deps if their features are not - // enabled. Ideally this would be computed from `cargo - // metadata --features …`, but that is somewhat slow. In - // the future, we may want to consider just filtering all - // build and dev dependencies in metadata::build. - if visited.insert(dep) - && (dep != "profiler_builtins" - || target - .map(|t| self.config.profiler_enabled(t)) - .unwrap_or_else(|| self.config.any_profiler_enabled())) - && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled()) - { - list.push(*dep); - } - } - } - ret.sort_unstable_by_key(|krate| krate.name); // reproducible order needed for tests - ret - } - - fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> { - if self.config.dry_run() { - return Vec::new(); - } - - if !stamp.exists() { - eprintln!( - "Error: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", - stamp.display() - ); - crate::exit!(1); - } - - let mut paths = Vec::new(); - let contents = t!(fs::read(stamp), &stamp); - // This is the method we use for extracting paths from the stamp file passed to us. See - // run_cargo for more information (in compile.rs). - for part in contents.split(|b| *b == 0) { - if part.is_empty() { - continue; - } - let dependency_type = match part[0] as char { - 'h' => DependencyType::Host, - 's' => DependencyType::TargetSelfContained, - 't' => DependencyType::Target, - _ => unreachable!(), - }; - let path = PathBuf::from(t!(str::from_utf8(&part[1..]))); - paths.push((path, dependency_type)); - } - paths - } - - /// Copies a file from `src` to `dst` - pub fn copy(&self, src: &Path, dst: &Path) { - self.copy_internal(src, dst, false); - } - - fn copy_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { - if self.config.dry_run() { - return; - } - self.verbose_than(1, &format!("Copy {src:?} to {dst:?}")); - if src == dst { - return; - } - let _ = fs::remove_file(&dst); - let metadata = t!(src.symlink_metadata()); - let mut src = src.to_path_buf(); - if metadata.file_type().is_symlink() { - if dereference_symlinks { - src = t!(fs::canonicalize(src)); - } else { - let link = t!(fs::read_link(src)); - t!(self.symlink_file(link, dst)); - return; - } - } - if let Ok(()) = fs::hard_link(&src, dst) { - // Attempt to "easy copy" by creating a hard link - // (symlinks don't work on windows), but if that fails - // just fall back to a slow `copy` operation. - } else { - if let Err(e) = fs::copy(&src, dst) { - panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e) - } - t!(fs::set_permissions(dst, metadata.permissions())); - let atime = FileTime::from_last_access_time(&metadata); - let mtime = FileTime::from_last_modification_time(&metadata); - t!(filetime::set_file_times(dst, atime, mtime)); - } - } - - /// Copies the `src` directory recursively to `dst`. Both are assumed to exist - /// when this function is called. - pub fn cp_r(&self, src: &Path, dst: &Path) { - if self.config.dry_run() { - return; - } - for f in self.read_dir(src) { - let path = f.path(); - let name = path.file_name().unwrap(); - let dst = dst.join(name); - if t!(f.file_type()).is_dir() { - t!(fs::create_dir_all(&dst)); - self.cp_r(&path, &dst); - } else { - let _ = fs::remove_file(&dst); - self.copy(&path, &dst); - } - } - } - - /// Copies the `src` directory recursively to `dst`. Both are assumed to exist - /// when this function is called. Unwanted files or directories can be skipped - /// by returning `false` from the filter function. - pub fn cp_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { - // Immediately recurse with an empty relative path - self.recurse_(src, dst, Path::new(""), filter) - } - - // Inner function does the actual work - fn recurse_(&self, src: &Path, dst: &Path, relative: &Path, filter: &dyn Fn(&Path) -> bool) { - for f in self.read_dir(src) { - let path = f.path(); - let name = path.file_name().unwrap(); - let dst = dst.join(name); - let relative = relative.join(name); - // Only copy file or directory if the filter function returns true - if filter(&relative) { - if t!(f.file_type()).is_dir() { - let _ = fs::remove_dir_all(&dst); - self.create_dir(&dst); - self.recurse_(&path, &dst, &relative, filter); - } else { - let _ = fs::remove_file(&dst); - self.copy(&path, &dst); - } - } - } - } - - fn copy_to_folder(&self, src: &Path, dest_folder: &Path) { - let file_name = src.file_name().unwrap(); - let dest = dest_folder.join(file_name); - self.copy(src, &dest); - } - - fn install(&self, src: &Path, dstdir: &Path, perms: u32) { - if self.config.dry_run() { - return; - } - let dst = dstdir.join(src.file_name().unwrap()); - self.verbose_than(1, &format!("Install {src:?} to {dst:?}")); - t!(fs::create_dir_all(dstdir)); - if !src.exists() { - panic!("Error: File \"{}\" not found!", src.display()); - } - self.copy_internal(src, &dst, true); - chmod(&dst, perms); - } - - fn read(&self, path: &Path) -> String { - if self.config.dry_run() { - return String::new(); - } - t!(fs::read_to_string(path)) - } - - fn create_dir(&self, dir: &Path) { - if self.config.dry_run() { - return; - } - t!(fs::create_dir_all(dir)) - } - - fn remove_dir(&self, dir: &Path) { - if self.config.dry_run() { - return; - } - t!(fs::remove_dir_all(dir)) - } - - fn read_dir(&self, dir: &Path) -> impl Iterator { - let iter = match fs::read_dir(dir) { - Ok(v) => v, - Err(_) if self.config.dry_run() => return vec![].into_iter(), - Err(err) => panic!("could not read dir {dir:?}: {err:?}"), - }; - iter.map(|e| t!(e)).collect::>().into_iter() - } - - fn symlink_file, Q: AsRef>(&self, src: P, link: Q) -> io::Result<()> { - #[cfg(unix)] - use std::os::unix::fs::symlink as symlink_file; - #[cfg(windows)] - use std::os::windows::fs::symlink_file; - if !self.config.dry_run() { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) } - } - - /// Returns if config.ninja is enabled, and checks for ninja existence, - /// exiting with a nicer error message if not. - fn ninja(&self) -> bool { - let mut cmd_finder = crate::sanity::Finder::new(); - - if self.config.ninja_in_file { - // Some Linux distros rename `ninja` to `ninja-build`. - // CMake can work with either binary name. - if cmd_finder.maybe_have("ninja-build").is_none() - && cmd_finder.maybe_have("ninja").is_none() - { - eprintln!( - " -Couldn't find required command: ninja (or ninja-build) - -You should install ninja as described at -, -or set `ninja = false` in the `[llvm]` section of `config.toml`. -Alternatively, set `download-ci-llvm = true` in that `[llvm]` section -to download LLVM rather than building it. -" - ); - exit!(1); - } - } - - // If ninja isn't enabled but we're building for MSVC then we try - // doubly hard to enable it. It was realized in #43767 that the msbuild - // CMake generator for MSVC doesn't respect configuration options like - // disabling LLVM assertions, which can often be quite important! - // - // In these cases we automatically enable Ninja if we find it in the - // environment. - if !self.config.ninja_in_file - && self.config.build.contains("msvc") - && cmd_finder.maybe_have("ninja").is_some() - { - return true; - } - - self.config.ninja_in_file - } - - pub fn colored_stdout R>(&self, f: F) -> R { - self.colored_stream_inner(StandardStream::stdout, self.config.stdout_is_tty, f) - } - - pub fn colored_stderr R>(&self, f: F) -> R { - self.colored_stream_inner(StandardStream::stderr, self.config.stderr_is_tty, f) - } - - fn colored_stream_inner(&self, constructor: C, is_tty: bool, f: F) -> R - where - C: Fn(ColorChoice) -> StandardStream, - F: FnOnce(&mut dyn WriteColor) -> R, - { - let choice = match self.config.color { - flags::Color::Always => ColorChoice::Always, - flags::Color::Never => ColorChoice::Never, - flags::Color::Auto if !is_tty => ColorChoice::Never, - flags::Color::Auto => ColorChoice::Auto, - }; - let mut stream = constructor(choice); - let result = f(&mut stream); - stream.reset().unwrap(); - result - } -} - -/// Extract the beta revision from the full version string. -/// -/// The full version string looks like "a.b.c-beta.y". And we need to extract -/// the "y" part from the string. -pub fn extract_beta_rev(version: &str) -> Option { - let parts = version.splitn(2, "-beta.").collect::>(); - let count = parts.get(1).and_then(|s| s.find(' ').map(|p| (&s[..p]).to_string())); - - count -} - -#[cfg(unix)] -fn chmod(path: &Path, perms: u32) { - use std::os::unix::fs::*; - t!(fs::set_permissions(path, fs::Permissions::from_mode(perms))); -} -#[cfg(windows)] -fn chmod(_path: &Path, _perms: u32) {} - -impl Compiler { - pub fn with_stage(mut self, stage: u32) -> Compiler { - self.stage = stage; - self - } - - /// Returns `true` if this is a snapshot compiler for `build`'s configuration - pub fn is_snapshot(&self, build: &Build) -> bool { - self.stage == 0 && self.host == build.build - } - - /// Returns if this compiler should be treated as a final stage one in the - /// current build session. - /// This takes into account whether we're performing a full bootstrap or - /// not; don't directly compare the stage with `2`! - pub fn is_final_stage(&self, build: &Build) -> bool { - let final_stage = if build.config.full_bootstrap { 2 } else { 1 }; - self.stage >= final_stage - } -} - -fn envify(s: &str) -> String { - s.chars() - .map(|c| match c { - '-' => '_', - c => c, - }) - .flat_map(|c| c.to_uppercase()) - .collect() -} diff --git a/src/bootstrap/llvm.rs b/src/bootstrap/llvm.rs deleted file mode 100644 index 455683158..000000000 --- a/src/bootstrap/llvm.rs +++ /dev/null @@ -1,1357 +0,0 @@ -//! Compilation of native dependencies like LLVM. -//! -//! Native projects like LLVM unfortunately aren't suited just yet for -//! compilation in build scripts that Cargo has. This is because the -//! compilation takes a *very* long time but also because we don't want to -//! compile LLVM 3 times as part of a normal bootstrap (we want it cached). -//! -//! LLVM and compiler-rt are essentially just wired up to everything else to -//! ensure that they're always in place if needed. - -use std::env; -use std::env::consts::EXE_EXTENSION; -use std::ffi::{OsStr, OsString}; -use std::fs::{self, File}; -use std::io; -use std::path::{Path, PathBuf}; -use std::process::Command; - -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::channel; -use crate::config::{Config, TargetSelection}; -use crate::util::get_clang_cl_resource_dir; -use crate::util::{self, exe, output, t, up_to_date}; -use crate::{CLang, GitRepo, Kind}; - -use build_helper::ci::CiEnv; -use build_helper::git::get_git_merge_base; - -#[derive(Clone)] -pub struct LlvmResult { - /// Path to llvm-config binary. - /// NB: This is always the host llvm-config! - pub llvm_config: PathBuf, - /// Path to LLVM cmake directory for the target. - pub llvm_cmake_dir: PathBuf, -} - -pub struct Meta { - stamp: HashStamp, - res: LlvmResult, - out_dir: PathBuf, - root: String, -} - -// Linker flags to pass to LLVM's CMake invocation. -#[derive(Debug, Clone, Default)] -struct LdFlags { - // CMAKE_EXE_LINKER_FLAGS - exe: OsString, - // CMAKE_SHARED_LINKER_FLAGS - shared: OsString, - // CMAKE_MODULE_LINKER_FLAGS - module: OsString, -} - -impl LdFlags { - fn push_all(&mut self, s: impl AsRef) { - let s = s.as_ref(); - self.exe.push(" "); - self.exe.push(s); - self.shared.push(" "); - self.shared.push(s); - self.module.push(" "); - self.module.push(s); - } -} - -/// This returns whether we've already previously built LLVM. -/// -/// It's used to avoid busting caches during x.py check -- if we've already built -/// LLVM, it's fine for us to not try to avoid doing so. -/// -/// This will return the llvm-config if it can get it (but it will not build it -/// if not). -pub fn prebuilt_llvm_config( - builder: &Builder<'_>, - target: TargetSelection, -) -> Result { - builder.config.maybe_download_ci_llvm(); - - // If we're using a custom LLVM bail out here, but we can only use a - // custom LLVM for the build triple. - if let Some(config) = builder.config.target_config.get(&target) { - if let Some(ref s) = config.llvm_config { - check_llvm_version(builder, s); - let llvm_config = s.to_path_buf(); - let mut llvm_cmake_dir = llvm_config.clone(); - llvm_cmake_dir.pop(); - llvm_cmake_dir.pop(); - llvm_cmake_dir.push("lib"); - llvm_cmake_dir.push("cmake"); - llvm_cmake_dir.push("llvm"); - return Ok(LlvmResult { llvm_config, llvm_cmake_dir }); - } - } - - let root = "src/llvm-project/llvm"; - let out_dir = builder.llvm_out(target); - - let mut llvm_config_ret_dir = builder.llvm_out(builder.config.build); - if !builder.config.build.contains("msvc") || builder.ninja() { - llvm_config_ret_dir.push("build"); - } - llvm_config_ret_dir.push("bin"); - let build_llvm_config = llvm_config_ret_dir.join(exe("llvm-config", builder.config.build)); - let llvm_cmake_dir = out_dir.join("lib/cmake/llvm"); - let res = LlvmResult { llvm_config: build_llvm_config, llvm_cmake_dir }; - - let stamp = out_dir.join("llvm-finished-building"); - let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); - - if stamp.is_done() { - if stamp.hash.is_none() { - builder.info( - "Could not determine the LLVM submodule commit hash. \ - Assuming that an LLVM rebuild is not necessary.", - ); - builder.info(&format!( - "To force LLVM to rebuild, remove the file `{}`", - stamp.path.display() - )); - } - return Ok(res); - } - - Err(Meta { stamp, res, out_dir, root: root.into() }) -} - -/// This retrieves the LLVM sha we *want* to use, according to git history. -pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { - let llvm_sha = if is_git { - // We proceed in 2 steps. First we get the closest commit that is actually upstream. Then we - // walk back further to the last bors merge commit that actually changed LLVM. The first - // step will fail on CI because only the `auto` branch exists; we just fall back to `HEAD` - // in that case. - let closest_upstream = - get_git_merge_base(Some(&config.src)).unwrap_or_else(|_| "HEAD".into()); - let mut rev_list = config.git(); - rev_list.args(&[ - PathBuf::from("rev-list"), - format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(), - "-n1".into(), - "--first-parent".into(), - closest_upstream.into(), - "--".into(), - config.src.join("src/llvm-project"), - config.src.join("src/bootstrap/download-ci-llvm-stamp"), - // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` - config.src.join("src/version"), - ]); - output(&mut rev_list).trim().to_owned() - } else if let Some(info) = channel::read_commit_info_file(&config.src) { - info.sha.trim().to_owned() - } else { - "".to_owned() - }; - - if llvm_sha.is_empty() { - eprintln!("error: could not find commit hash for downloading LLVM"); - eprintln!("help: maybe your repository history is too shallow?"); - eprintln!("help: consider disabling `download-ci-llvm`"); - eprintln!("help: or fetch enough history to include one upstream commit"); - panic!(); - } - - llvm_sha -} - -/// Returns whether the CI-found LLVM is currently usable. -/// -/// This checks both the build triple platform to confirm we're usable at all, -/// and then verifies if the current HEAD matches the detected LLVM SHA head, -/// in which case LLVM is indicated as not available. -pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { - // This is currently all tier 1 targets and tier 2 targets with host tools - // (since others may not have CI artifacts) - // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 - let supported_platforms = [ - // tier 1 - ("aarch64-unknown-linux-gnu", false), - ("i686-pc-windows-gnu", false), - ("i686-pc-windows-msvc", false), - ("i686-unknown-linux-gnu", false), - ("x86_64-unknown-linux-gnu", true), - ("x86_64-apple-darwin", true), - ("x86_64-pc-windows-gnu", true), - ("x86_64-pc-windows-msvc", true), - // tier 2 with host tools - ("aarch64-apple-darwin", false), - ("aarch64-pc-windows-msvc", false), - ("aarch64-unknown-linux-musl", false), - ("arm-unknown-linux-gnueabi", false), - ("arm-unknown-linux-gnueabihf", false), - ("armv7-unknown-linux-gnueabihf", false), - ("loongarch64-unknown-linux-gnu", false), - ("mips-unknown-linux-gnu", false), - ("mips64-unknown-linux-gnuabi64", false), - ("mips64el-unknown-linux-gnuabi64", false), - ("mipsel-unknown-linux-gnu", false), - ("powerpc-unknown-linux-gnu", false), - ("powerpc64-unknown-linux-gnu", false), - ("powerpc64le-unknown-linux-gnu", false), - ("riscv64gc-unknown-linux-gnu", false), - ("s390x-unknown-linux-gnu", false), - ("x86_64-unknown-freebsd", false), - ("x86_64-unknown-illumos", false), - ("x86_64-unknown-linux-musl", false), - ("x86_64-unknown-netbsd", false), - ]; - - if !supported_platforms.contains(&(&*config.build.triple, asserts)) - && (asserts || !supported_platforms.contains(&(&*config.build.triple, true))) - { - return false; - } - - if is_ci_llvm_modified(config) { - eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); - return false; - } - - true -} - -/// Returns true if we're running in CI with modified LLVM (and thus can't download it) -pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool { - CiEnv::is_ci() && config.rust_info.is_managed_git_subrepository() && { - // We assume we have access to git, so it's okay to unconditionally pass - // `true` here. - let llvm_sha = detect_llvm_sha(config, true); - let head_sha = output(config.git().arg("rev-parse").arg("HEAD")); - let head_sha = head_sha.trim(); - llvm_sha == head_sha - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Llvm { - pub target: TargetSelection, -} - -impl Step for Llvm { - type Output = LlvmResult; - - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project").path("src/llvm-project/llvm") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Llvm { target: run.target }); - } - - /// Compile LLVM for `target`. - fn run(self, builder: &Builder<'_>) -> LlvmResult { - let target = self.target; - let target_native = if self.target.starts_with("riscv") { - // RISC-V target triples in Rust is not named the same as C compiler target triples. - // This converts Rust RISC-V target triples to C compiler triples. - let idx = target.triple.find('-').unwrap(); - - format!("riscv{}{}", &target.triple[5..7], &target.triple[idx..]) - } else if self.target.starts_with("powerpc") && self.target.ends_with("freebsd") { - // FreeBSD 13 had incompatible ABI changes on all PowerPC platforms. - // Set the version suffix to 13.0 so the correct target details are used. - format!("{}{}", self.target, "13.0") - } else { - target.to_string() - }; - - let Meta { stamp, res, out_dir, root } = match prebuilt_llvm_config(builder, target) { - Ok(p) => return p, - Err(m) => m, - }; - - builder.update_submodule(&Path::new("src").join("llvm-project")); - if builder.llvm_link_shared() && target.contains("windows") { - panic!("shared linking to LLVM is not currently supported on {}", target.triple); - } - - let _guard = builder.msg_unstaged(Kind::Build, "LLVM", target); - t!(stamp.remove()); - let _time = util::timeit(&builder); - t!(fs::create_dir_all(&out_dir)); - - // https://llvm.org/docs/CMake.html - let mut cfg = cmake::Config::new(builder.src.join(root)); - let mut ldflags = LdFlags::default(); - - let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { - (false, _) => "Debug", - (true, false) => "Release", - (true, true) => "RelWithDebInfo", - }; - - // NOTE: remember to also update `config.example.toml` when changing the - // defaults! - let llvm_targets = match &builder.config.llvm_targets { - Some(s) => s, - None => { - "AArch64;ARM;BPF;Hexagon;LoongArch;MSP430;Mips;NVPTX;PowerPC;RISCV;\ - Sparc;SystemZ;WebAssembly;X86" - } - }; - - let llvm_exp_targets = match builder.config.llvm_experimental_targets { - Some(ref s) => s, - None => "AVR;M68k;CSKY", - }; - - let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" }; - let plugins = if builder.config.llvm_plugins { "ON" } else { "OFF" }; - let enable_tests = if builder.config.llvm_tests { "ON" } else { "OFF" }; - let enable_warnings = if builder.config.llvm_enable_warnings { "ON" } else { "OFF" }; - - cfg.out_dir(&out_dir) - .profile(profile) - .define("LLVM_ENABLE_ASSERTIONS", assertions) - .define("LLVM_UNREACHABLE_OPTIMIZE", "OFF") - .define("LLVM_ENABLE_PLUGINS", plugins) - .define("LLVM_TARGETS_TO_BUILD", llvm_targets) - .define("LLVM_EXPERIMENTAL_TARGETS_TO_BUILD", llvm_exp_targets) - .define("LLVM_INCLUDE_EXAMPLES", "OFF") - .define("LLVM_INCLUDE_DOCS", "OFF") - .define("LLVM_INCLUDE_BENCHMARKS", "OFF") - .define("LLVM_INCLUDE_TESTS", enable_tests) - .define("LLVM_ENABLE_TERMINFO", "OFF") - .define("LLVM_ENABLE_LIBEDIT", "OFF") - .define("LLVM_ENABLE_BINDINGS", "OFF") - .define("LLVM_ENABLE_Z3_SOLVER", "OFF") - .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string()) - .define("LLVM_TARGET_ARCH", target_native.split('-').next().unwrap()) - .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native) - .define("LLVM_ENABLE_WARNINGS", enable_warnings); - - // Parts of our test suite rely on the `FileCheck` tool, which is built by default in - // `build/$TARGET/llvm/build/bin` is but *not* then installed to `build/$TARGET/llvm/bin`. - // This flag makes sure `FileCheck` is copied in the final binaries directory. - cfg.define("LLVM_INSTALL_UTILS", "ON"); - - if builder.config.llvm_profile_generate { - cfg.define("LLVM_BUILD_INSTRUMENTED", "IR"); - if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") { - cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir); - } - cfg.define("LLVM_BUILD_RUNTIME", "No"); - } - if let Some(path) = builder.config.llvm_profile_use.as_ref() { - cfg.define("LLVM_PROFDATA_FILE", &path); - } - - // Disable zstd to avoid a dependency on libzstd.so. - cfg.define("LLVM_ENABLE_ZSTD", "OFF"); - - if !target.contains("windows") { - cfg.define("LLVM_ENABLE_ZLIB", "ON"); - } else { - cfg.define("LLVM_ENABLE_ZLIB", "OFF"); - } - - // Are we compiling for iOS/tvOS/watchOS? - if target.contains("apple-ios") - || target.contains("apple-tvos") - || target.contains("apple-watchos") - { - // These two defines prevent CMake from automatically trying to add a MacOSX sysroot, which leads to a compiler error. - cfg.define("CMAKE_OSX_SYSROOT", "/"); - cfg.define("CMAKE_OSX_DEPLOYMENT_TARGET", ""); - // Prevent cmake from adding -bundle to CFLAGS automatically, which leads to a compiler error because "-bitcode_bundle" also gets added. - cfg.define("LLVM_ENABLE_PLUGINS", "OFF"); - // Zlib fails to link properly, leading to a compiler error. - cfg.define("LLVM_ENABLE_ZLIB", "OFF"); - } - - // This setting makes the LLVM tools link to the dynamic LLVM library, - // which saves both memory during parallel links and overall disk space - // for the tools. We don't do this on every platform as it doesn't work - // equally well everywhere. - if builder.llvm_link_shared() { - cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); - } - - if (target.starts_with("riscv") || target.starts_with("csky")) - && !target.contains("freebsd") - && !target.contains("openbsd") - && !target.contains("netbsd") - { - // RISC-V and CSKY GCC erroneously requires linking against - // `libatomic` when using 1-byte and 2-byte C++ - // atomics but the LLVM build system check cannot - // detect this. Therefore it is set manually here. - // Some BSD uses Clang as its system compiler and - // provides no libatomic in its base system so does - // not want this. - ldflags.exe.push(" -latomic"); - ldflags.shared.push(" -latomic"); - } - - if target.contains("msvc") { - cfg.define("LLVM_USE_CRT_DEBUG", "MT"); - cfg.define("LLVM_USE_CRT_RELEASE", "MT"); - cfg.define("LLVM_USE_CRT_RELWITHDEBINFO", "MT"); - cfg.static_crt(true); - } - - if target.starts_with("i686") { - cfg.define("LLVM_BUILD_32_BITS", "ON"); - } - - let mut enabled_llvm_projects = Vec::new(); - - if util::forcing_clang_based_tests() { - enabled_llvm_projects.push("clang"); - enabled_llvm_projects.push("compiler-rt"); - } - - if builder.config.llvm_polly { - enabled_llvm_projects.push("polly"); - } - - if builder.config.llvm_clang { - enabled_llvm_projects.push("clang"); - } - - // We want libxml to be disabled. - // See https://github.com/rust-lang/rust/pull/50104 - cfg.define("LLVM_ENABLE_LIBXML2", "OFF"); - - if !enabled_llvm_projects.is_empty() { - enabled_llvm_projects.sort(); - enabled_llvm_projects.dedup(); - cfg.define("LLVM_ENABLE_PROJECTS", enabled_llvm_projects.join(";")); - } - - if let Some(num_linkers) = builder.config.llvm_link_jobs { - if num_linkers > 0 { - cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string()); - } - } - - // https://llvm.org/docs/HowToCrossCompileLLVM.html - if target != builder.config.build { - let LlvmResult { llvm_config, .. } = - builder.ensure(Llvm { target: builder.config.build }); - if !builder.config.dry_run() { - let llvm_bindir = output(Command::new(&llvm_config).arg("--bindir")); - let host_bin = Path::new(llvm_bindir.trim()); - cfg.define( - "LLVM_TABLEGEN", - host_bin.join("llvm-tblgen").with_extension(EXE_EXTENSION), - ); - // LLVM_NM is required for cross compiling using MSVC - cfg.define("LLVM_NM", host_bin.join("llvm-nm").with_extension(EXE_EXTENSION)); - } - cfg.define("LLVM_CONFIG_PATH", llvm_config); - if builder.config.llvm_clang { - let build_bin = builder.llvm_out(builder.config.build).join("build").join("bin"); - let clang_tblgen = build_bin.join("clang-tblgen").with_extension(EXE_EXTENSION); - if !builder.config.dry_run() && !clang_tblgen.exists() { - panic!("unable to find {}", clang_tblgen.display()); - } - cfg.define("CLANG_TABLEGEN", clang_tblgen); - } - } - - let llvm_version_suffix = if let Some(ref suffix) = builder.config.llvm_version_suffix { - // Allow version-suffix="" to not define a version suffix at all. - if !suffix.is_empty() { Some(suffix.to_string()) } else { None } - } else if builder.config.channel == "dev" { - // Changes to a version suffix require a complete rebuild of the LLVM. - // To avoid rebuilds during a time of version bump, don't include rustc - // release number on the dev channel. - Some("-rust-dev".to_string()) - } else { - Some(format!("-rust-{}-{}", builder.version, builder.config.channel)) - }; - if let Some(ref suffix) = llvm_version_suffix { - cfg.define("LLVM_VERSION_SUFFIX", suffix); - } - - configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); - configure_llvm(builder, target, &mut cfg); - - for (key, val) in &builder.config.llvm_build_config { - cfg.define(key, val); - } - - if builder.config.dry_run() { - return res; - } - - cfg.build(); - - // Helper to find the name of LLVM's shared library on darwin and linux. - let find_llvm_lib_name = |extension| { - let mut cmd = Command::new(&res.llvm_config); - let version = output(cmd.arg("--version")); - let major = version.split('.').next().unwrap(); - - match &llvm_version_suffix { - Some(version_suffix) => format!("libLLVM-{major}{version_suffix}.{extension}"), - None => format!("libLLVM-{major}.{extension}"), - } - }; - - // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned - // libLLVM.dylib will be built. However, llvm-config will still look - // for a versioned path like libLLVM-14.dylib. Manually create a symbolic - // link to make llvm-config happy. - if builder.llvm_link_shared() && target.contains("apple-darwin") { - let lib_name = find_llvm_lib_name("dylib"); - let lib_llvm = out_dir.join("build").join("lib").join(lib_name); - if !lib_llvm.exists() { - t!(builder.symlink_file("libLLVM.dylib", &lib_llvm)); - } - } - - // When building LLVM as a shared library on linux, it can contain unexpected debuginfo: - // some can come from the C++ standard library. Unless we're explicitly requesting LLVM to - // be built with debuginfo, strip it away after the fact, to make dist artifacts smaller. - if builder.llvm_link_shared() - && builder.config.llvm_optimize - && !builder.config.llvm_release_debuginfo - { - // Find the name of the LLVM shared library that we just built. - let lib_name = find_llvm_lib_name("so"); - - // If the shared library exists in LLVM's `/build/lib/` or `/lib/` folders, strip its - // debuginfo. - crate::compile::strip_debug(builder, target, &out_dir.join("lib").join(&lib_name)); - crate::compile::strip_debug( - builder, - target, - &out_dir.join("build").join("lib").join(&lib_name), - ); - } - - t!(stamp.write()); - - res - } -} - -fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) { - if builder.config.dry_run() { - return; - } - - let mut cmd = Command::new(llvm_config); - let version = output(cmd.arg("--version")); - let mut parts = version.split('.').take(2).filter_map(|s| s.parse::().ok()); - if let (Some(major), Some(_minor)) = (parts.next(), parts.next()) { - if major >= 15 { - return; - } - } - panic!("\n\nbad LLVM version: {version}, need >=15.0\n\n") -} - -fn configure_cmake( - builder: &Builder<'_>, - target: TargetSelection, - cfg: &mut cmake::Config, - use_compiler_launcher: bool, - mut ldflags: LdFlags, - extra_compiler_flags: &[&str], -) { - // Do not print installation messages for up-to-date files. - // LLVM and LLD builds can produce a lot of those and hit CI limits on log size. - cfg.define("CMAKE_INSTALL_MESSAGE", "LAZY"); - - // Do not allow the user's value of DESTDIR to influence where - // LLVM will install itself. LLVM must always be installed in our - // own build directories. - cfg.env("DESTDIR", ""); - - if builder.ninja() { - cfg.generator("Ninja"); - } - cfg.target(&target.triple).host(&builder.config.build.triple); - - if target != builder.config.build { - cfg.define("CMAKE_CROSSCOMPILING", "True"); - - if target.contains("netbsd") { - cfg.define("CMAKE_SYSTEM_NAME", "NetBSD"); - } else if target.contains("dragonfly") { - cfg.define("CMAKE_SYSTEM_NAME", "DragonFly"); - } else if target.contains("freebsd") { - cfg.define("CMAKE_SYSTEM_NAME", "FreeBSD"); - } else if target.contains("windows") { - cfg.define("CMAKE_SYSTEM_NAME", "Windows"); - } else if target.contains("haiku") { - cfg.define("CMAKE_SYSTEM_NAME", "Haiku"); - } else if target.contains("solaris") || target.contains("illumos") { - cfg.define("CMAKE_SYSTEM_NAME", "SunOS"); - } else if target.contains("linux") { - cfg.define("CMAKE_SYSTEM_NAME", "Linux"); - } else { - builder.info(&format!( - "could not determine CMAKE_SYSTEM_NAME from the target `{target}`, build may fail", - )); - } - - // When cross-compiling we should also set CMAKE_SYSTEM_VERSION, but in - // that case like CMake we cannot easily determine system version either. - // - // Since, the LLVM itself makes rather limited use of version checks in - // CMakeFiles (and then only in tests), and so far no issues have been - // reported, the system version is currently left unset. - - if target.contains("darwin") { - // Make sure that CMake does not build universal binaries on macOS. - // Explicitly specify the one single target architecture. - if target.starts_with("aarch64") { - // macOS uses a different name for building arm64 - cfg.define("CMAKE_OSX_ARCHITECTURES", "arm64"); - } else if target.starts_with("i686") { - // macOS uses a different name for building i386 - cfg.define("CMAKE_OSX_ARCHITECTURES", "i386"); - } else { - cfg.define("CMAKE_OSX_ARCHITECTURES", target.triple.split('-').next().unwrap()); - } - } - } - - let sanitize_cc = |cc: &Path| { - if target.contains("msvc") { - OsString::from(cc.to_str().unwrap().replace("\\", "/")) - } else { - cc.as_os_str().to_owned() - } - }; - - // MSVC with CMake uses msbuild by default which doesn't respect these - // vars that we'd otherwise configure. In that case we just skip this - // entirely. - if target.contains("msvc") && !builder.ninja() { - return; - } - - let (cc, cxx) = match builder.config.llvm_clang_cl { - Some(ref cl) => (cl.into(), cl.into()), - None => (builder.cc(target), builder.cxx(target).unwrap()), - }; - - // Handle msvc + ninja + ccache specially (this is what the bots use) - if target.contains("msvc") && builder.ninja() && builder.config.ccache.is_some() { - let mut wrap_cc = env::current_exe().expect("failed to get cwd"); - wrap_cc.set_file_name("sccache-plus-cl.exe"); - - cfg.define("CMAKE_C_COMPILER", sanitize_cc(&wrap_cc)) - .define("CMAKE_CXX_COMPILER", sanitize_cc(&wrap_cc)); - cfg.env("SCCACHE_PATH", builder.config.ccache.as_ref().unwrap()) - .env("SCCACHE_TARGET", target.triple) - .env("SCCACHE_CC", &cc) - .env("SCCACHE_CXX", &cxx); - - // Building LLVM on MSVC can be a little ludicrous at times. We're so far - // off the beaten path here that I'm not really sure this is even half - // supported any more. Here we're trying to: - // - // * Build LLVM on MSVC - // * Build LLVM with `clang-cl` instead of `cl.exe` - // * Build a project with `sccache` - // * Build for 32-bit as well - // * Build with Ninja - // - // For `cl.exe` there are different binaries to compile 32/64 bit which - // we use but for `clang-cl` there's only one which internally - // multiplexes via flags. As a result it appears that CMake's detection - // of a compiler's architecture and such on MSVC **doesn't** pass any - // custom flags we pass in CMAKE_CXX_FLAGS below. This means that if we - // use `clang-cl.exe` it's always diagnosed as a 64-bit compiler which - // definitely causes problems since all the env vars are pointing to - // 32-bit libraries. - // - // To hack around this... again... we pass an argument that's - // unconditionally passed in the sccache shim. This'll get CMake to - // correctly diagnose it's doing a 32-bit compilation and LLVM will - // internally configure itself appropriately. - if builder.config.llvm_clang_cl.is_some() && target.contains("i686") { - cfg.env("SCCACHE_EXTRA_ARGS", "-m32"); - } - } else { - // If ccache is configured we inform the build a little differently how - // to invoke ccache while also invoking our compilers. - if use_compiler_launcher { - if let Some(ref ccache) = builder.config.ccache { - cfg.define("CMAKE_C_COMPILER_LAUNCHER", ccache) - .define("CMAKE_CXX_COMPILER_LAUNCHER", ccache); - } - } - cfg.define("CMAKE_C_COMPILER", sanitize_cc(&cc)) - .define("CMAKE_CXX_COMPILER", sanitize_cc(&cxx)) - .define("CMAKE_ASM_COMPILER", sanitize_cc(&cc)); - } - - cfg.build_arg("-j").build_arg(builder.jobs().to_string()); - let mut cflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::C).join(" ").into(); - if let Some(ref s) = builder.config.llvm_cflags { - cflags.push(" "); - cflags.push(s); - } - - if builder.config.llvm_clang_cl.is_some() { - cflags.push(&format!(" --target={target}")); - } - for flag in extra_compiler_flags { - cflags.push(&format!(" {flag}")); - } - cfg.define("CMAKE_C_FLAGS", cflags); - let mut cxxflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::Cxx).join(" ").into(); - if let Some(ref s) = builder.config.llvm_cxxflags { - cxxflags.push(" "); - cxxflags.push(s); - } - if builder.config.llvm_clang_cl.is_some() { - cxxflags.push(&format!(" --target={target}")); - } - for flag in extra_compiler_flags { - cxxflags.push(&format!(" {flag}")); - } - cfg.define("CMAKE_CXX_FLAGS", cxxflags); - if let Some(ar) = builder.ar(target) { - if ar.is_absolute() { - // LLVM build breaks if `CMAKE_AR` is a relative path, for some reason it - // tries to resolve this path in the LLVM build directory. - cfg.define("CMAKE_AR", sanitize_cc(&ar)); - } - } - - if let Some(ranlib) = builder.ranlib(target) { - if ranlib.is_absolute() { - // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it - // tries to resolve this path in the LLVM build directory. - cfg.define("CMAKE_RANLIB", sanitize_cc(&ranlib)); - } - } - - if let Some(ref flags) = builder.config.llvm_ldflags { - ldflags.push_all(flags); - } - - if let Some(flags) = get_var("LDFLAGS", &builder.config.build.triple, &target.triple) { - ldflags.push_all(&flags); - } - - // For distribution we want the LLVM tools to be *statically* linked to libstdc++. - // We also do this if the user explicitly requested static libstdc++. - if builder.config.llvm_static_stdcpp - && !target.contains("msvc") - && !target.contains("netbsd") - && !target.contains("solaris") - { - if target.contains("apple") || target.contains("windows") { - ldflags.push_all("-static-libstdc++"); - } else { - ldflags.push_all("-Wl,-Bsymbolic -static-libstdc++"); - } - } - - cfg.define("CMAKE_SHARED_LINKER_FLAGS", &ldflags.shared); - cfg.define("CMAKE_MODULE_LINKER_FLAGS", &ldflags.module); - cfg.define("CMAKE_EXE_LINKER_FLAGS", &ldflags.exe); - - if env::var_os("SCCACHE_ERROR_LOG").is_some() { - cfg.env("RUSTC_LOG", "sccache=warn"); - } -} - -fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmake::Config) { - // ThinLTO is only available when building with LLVM, enabling LLD is required. - // Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin. - if builder.config.llvm_thin_lto { - cfg.define("LLVM_ENABLE_LTO", "Thin"); - if !target.contains("apple") { - cfg.define("LLVM_ENABLE_LLD", "ON"); - } - } - - if let Some(ref linker) = builder.config.llvm_use_linker { - cfg.define("LLVM_USE_LINKER", linker); - } - - if builder.config.llvm_allow_old_toolchain { - cfg.define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "YES"); - } -} - -// Adapted from https://github.com/alexcrichton/cc-rs/blob/fba7feded71ee4f63cfe885673ead6d7b4f2f454/src/lib.rs#L2347-L2365 -fn get_var(var_base: &str, host: &str, target: &str) -> Option { - let kind = if host == target { "HOST" } else { "TARGET" }; - let target_u = target.replace("-", "_"); - env::var_os(&format!("{var_base}_{target}")) - .or_else(|| env::var_os(&format!("{}_{}", var_base, target_u))) - .or_else(|| env::var_os(&format!("{}_{}", kind, var_base))) - .or_else(|| env::var_os(var_base)) -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Lld { - pub target: TargetSelection, -} - -impl Step for Lld { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/lld") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Lld { target: run.target }); - } - - /// Compile LLD for `target`. - fn run(self, builder: &Builder<'_>) -> PathBuf { - if builder.config.dry_run() { - return PathBuf::from("lld-out-dir-test-gen"); - } - let target = self.target; - - let LlvmResult { llvm_config, llvm_cmake_dir } = builder.ensure(Llvm { target }); - - // The `dist` step packages LLD next to LLVM's binaries for download-ci-llvm. The root path - // we usually expect here is `./build/$triple/ci-llvm/`, with the binaries in its `bin` - // subfolder. We check if that's the case, and if LLD's binary already exists there next to - // `llvm-config`: if so, we can use it instead of building LLVM/LLD from source. - let ci_llvm_bin = llvm_config.parent().unwrap(); - if ci_llvm_bin.is_dir() && ci_llvm_bin.file_name().unwrap() == "bin" { - let lld_path = ci_llvm_bin.join(exe("lld", target)); - if lld_path.exists() { - // The following steps copying `lld` as `rust-lld` to the sysroot, expect it in the - // `bin` subfolder of this step's out dir. - return ci_llvm_bin.parent().unwrap().to_path_buf(); - } - } - - let out_dir = builder.lld_out(target); - let done_stamp = out_dir.join("lld-finished-building"); - if done_stamp.exists() { - return out_dir; - } - - let _guard = builder.msg_unstaged(Kind::Build, "LLD", target); - let _time = util::timeit(&builder); - t!(fs::create_dir_all(&out_dir)); - - let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld")); - let mut ldflags = LdFlags::default(); - - // When building LLD as part of a build with instrumentation on windows, for example - // when doing PGO on CI, cmake or clang-cl don't automatically link clang's - // profiler runtime in. In that case, we need to manually ask cmake to do it, to avoid - // linking errors, much like LLVM's cmake setup does in that situation. - if builder.config.llvm_profile_generate && target.contains("msvc") { - if let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref() { - // Find clang's runtime library directory and push that as a search path to the - // cmake linker flags. - let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); - ldflags.push_all(&format!("/libpath:{}", clang_rt_dir.display())); - } - } - - // LLD is built as an LLVM tool, but is distributed outside of the `llvm-tools` component, - // which impacts where it expects to find LLVM's shared library. This causes #80703. - // - // LLD is distributed at "$root/lib/rustlib/$host/bin/rust-lld", but the `libLLVM-*.so` it - // needs is distributed at "$root/lib". The default rpath of "$ORIGIN/../lib" points at the - // lib path for LLVM tools, not the one for rust binaries. - // - // (The `llvm-tools` component copies the .so there for the other tools, and with that - // component installed, one can successfully invoke `rust-lld` directly without rustup's - // `LD_LIBRARY_PATH` overrides) - // - if builder.config.rpath_enabled(target) - && util::use_host_linker(target) - && builder.config.llvm_link_shared() - && target.contains("linux") - { - // So we inform LLD where it can find LLVM's libraries by adding an rpath entry to the - // expected parent `lib` directory. - // - // Be careful when changing this path, we need to ensure it's quoted or escaped: - // `$ORIGIN` would otherwise be expanded when the `LdFlags` are passed verbatim to - // cmake. - ldflags.push_all("-Wl,-rpath,'$ORIGIN/../../../'"); - } - - configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); - configure_llvm(builder, target, &mut cfg); - - // Re-use the same flags as llvm to control the level of debug information - // generated for lld. - let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { - (false, _) => "Debug", - (true, false) => "Release", - (true, true) => "RelWithDebInfo", - }; - - cfg.out_dir(&out_dir) - .profile(profile) - .define("LLVM_CMAKE_DIR", llvm_cmake_dir) - .define("LLVM_INCLUDE_TESTS", "OFF"); - - if target != builder.config.build { - // Use the host llvm-tblgen binary. - cfg.define( - "LLVM_TABLEGEN_EXE", - llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION), - ); - } - - cfg.build(); - - t!(File::create(&done_stamp)); - out_dir - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Sanitizers { - pub target: TargetSelection, -} - -impl Step for Sanitizers { - type Output = Vec; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("sanitizers") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Sanitizers { target: run.target }); - } - - /// Builds sanitizer runtime libraries. - fn run(self, builder: &Builder<'_>) -> Self::Output { - let compiler_rt_dir = builder.src.join("src/llvm-project/compiler-rt"); - if !compiler_rt_dir.exists() { - return Vec::new(); - } - - let out_dir = builder.native_dir(self.target).join("sanitizers"); - let runtimes = supported_sanitizers(&out_dir, self.target, &builder.config.channel); - if runtimes.is_empty() { - return runtimes; - } - - let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: builder.config.build }); - if builder.config.dry_run() { - return runtimes; - } - - let stamp = out_dir.join("sanitizers-finished-building"); - let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); - - if stamp.is_done() { - if stamp.hash.is_none() { - builder.info(&format!( - "Rebuild sanitizers by removing the file `{}`", - stamp.path.display() - )); - } - return runtimes; - } - - let _guard = builder.msg_unstaged(Kind::Build, "sanitizers", self.target); - t!(stamp.remove()); - let _time = util::timeit(&builder); - - let mut cfg = cmake::Config::new(&compiler_rt_dir); - cfg.profile("Release"); - cfg.define("CMAKE_C_COMPILER_TARGET", self.target.triple); - cfg.define("COMPILER_RT_BUILD_BUILTINS", "OFF"); - cfg.define("COMPILER_RT_BUILD_CRT", "OFF"); - cfg.define("COMPILER_RT_BUILD_LIBFUZZER", "OFF"); - cfg.define("COMPILER_RT_BUILD_PROFILE", "OFF"); - cfg.define("COMPILER_RT_BUILD_SANITIZERS", "ON"); - cfg.define("COMPILER_RT_BUILD_XRAY", "OFF"); - cfg.define("COMPILER_RT_DEFAULT_TARGET_ONLY", "ON"); - cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); - cfg.define("LLVM_CONFIG_PATH", &llvm_config); - - // On Darwin targets the sanitizer runtimes are build as universal binaries. - // Unfortunately sccache currently lacks support to build them successfully. - // Disable compiler launcher on Darwin targets to avoid potential issues. - let use_compiler_launcher = !self.target.contains("apple-darwin"); - let extra_compiler_flags: &[&str] = - if self.target.contains("apple") { &["-fembed-bitcode=off"] } else { &[] }; - configure_cmake( - builder, - self.target, - &mut cfg, - use_compiler_launcher, - LdFlags::default(), - extra_compiler_flags, - ); - - t!(fs::create_dir_all(&out_dir)); - cfg.out_dir(out_dir); - - for runtime in &runtimes { - cfg.build_target(&runtime.cmake_target); - cfg.build(); - } - t!(stamp.write()); - - runtimes - } -} - -#[derive(Clone, Debug)] -pub struct SanitizerRuntime { - /// CMake target used to build the runtime. - pub cmake_target: String, - /// Path to the built runtime library. - pub path: PathBuf, - /// Library filename that will be used rustc. - pub name: String, -} - -/// Returns sanitizers available on a given target. -fn supported_sanitizers( - out_dir: &Path, - target: TargetSelection, - channel: &str, -) -> Vec { - let darwin_libs = |os: &str, components: &[&str]| -> Vec { - components - .iter() - .map(move |c| SanitizerRuntime { - cmake_target: format!("clang_rt.{}_{}_dynamic", c, os), - path: out_dir - .join(&format!("build/lib/darwin/libclang_rt.{}_{}_dynamic.dylib", c, os)), - name: format!("librustc-{}_rt.{}.dylib", channel, c), - }) - .collect() - }; - - let common_libs = |os: &str, arch: &str, components: &[&str]| -> Vec { - components - .iter() - .map(move |c| SanitizerRuntime { - cmake_target: format!("clang_rt.{}-{}", c, arch), - path: out_dir.join(&format!("build/lib/{}/libclang_rt.{}-{}.a", os, c, arch)), - name: format!("librustc-{}_rt.{}.a", channel, c), - }) - .collect() - }; - - match &*target.triple { - "aarch64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "aarch64-apple-ios" => darwin_libs("ios", &["asan", "tsan"]), - "aarch64-apple-ios-sim" => darwin_libs("iossim", &["asan", "tsan"]), - "aarch64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "aarch64-unknown-fuchsia" => common_libs("fuchsia", "aarch64", &["asan"]), - "aarch64-unknown-linux-gnu" => { - common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) - } - "aarch64-unknown-linux-ohos" => { - common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) - } - "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "x86_64-unknown-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), - "x86_64-apple-ios" => darwin_libs("iossim", &["asan", "tsan"]), - "x86_64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), - "x86_64-unknown-freebsd" => common_libs("freebsd", "x86_64", &["asan", "msan", "tsan"]), - "x86_64-unknown-netbsd" => { - common_libs("netbsd", "x86_64", &["asan", "lsan", "msan", "tsan"]) - } - "x86_64-unknown-illumos" => common_libs("illumos", "x86_64", &["asan"]), - "x86_64-pc-solaris" => common_libs("solaris", "x86_64", &["asan"]), - "x86_64-unknown-linux-gnu" => { - common_libs("linux", "x86_64", &["asan", "lsan", "msan", "safestack", "tsan"]) - } - "x86_64-unknown-linux-musl" => { - common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) - } - "s390x-unknown-linux-gnu" => { - common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) - } - "s390x-unknown-linux-musl" => { - common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) - } - "x86_64-unknown-linux-ohos" => { - common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) - } - _ => Vec::new(), - } -} - -struct HashStamp { - path: PathBuf, - hash: Option>, -} - -impl HashStamp { - fn new(path: PathBuf, hash: Option<&str>) -> Self { - HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) } - } - - fn is_done(&self) -> bool { - match fs::read(&self.path) { - Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(), - Err(e) if e.kind() == io::ErrorKind::NotFound => false, - Err(e) => { - panic!("failed to read stamp file `{}`: {}", self.path.display(), e); - } - } - } - - fn remove(&self) -> io::Result<()> { - match fs::remove_file(&self.path) { - Ok(()) => Ok(()), - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - Ok(()) - } else { - Err(e) - } - } - } - } - - fn write(&self) -> io::Result<()> { - fs::write(&self.path, self.hash.as_deref().unwrap_or(b"")) - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrtBeginEnd { - pub target: TargetSelection, -} - -impl Step for CrtBeginEnd { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/compiler-rt/lib/crt") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CrtBeginEnd { target: run.target }); - } - - /// Build crtbegin.o/crtend.o for musl target. - fn run(self, builder: &Builder<'_>) -> Self::Output { - builder.update_submodule(&Path::new("src/llvm-project")); - - let out_dir = builder.native_dir(self.target).join("crt"); - - if builder.config.dry_run() { - return out_dir; - } - - let crtbegin_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtbegin.c"); - let crtend_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtend.c"); - if up_to_date(&crtbegin_src, &out_dir.join("crtbegin.o")) - && up_to_date(&crtend_src, &out_dir.join("crtendS.o")) - { - return out_dir; - } - - let _guard = builder.msg_unstaged(Kind::Build, "crtbegin.o and crtend.o", self.target); - t!(fs::create_dir_all(&out_dir)); - - let mut cfg = cc::Build::new(); - - if let Some(ar) = builder.ar(self.target) { - cfg.archiver(ar); - } - cfg.compiler(builder.cc(self.target)); - cfg.cargo_metadata(false) - .out_dir(&out_dir) - .target(&self.target.triple) - .host(&builder.config.build.triple) - .warnings(false) - .debug(false) - .opt_level(3) - .file(crtbegin_src) - .file(crtend_src); - - // Those flags are defined in src/llvm-project/compiler-rt/lib/crt/CMakeLists.txt - // Currently only consumer of those objects is musl, which use .init_array/.fini_array - // instead of .ctors/.dtors - cfg.flag("-std=c11") - .define("CRT_HAS_INITFINI_ARRAY", None) - .define("EH_USE_FRAME_REGISTRY", None); - - cfg.compile("crt"); - - t!(fs::copy(out_dir.join("crtbegin.o"), out_dir.join("crtbeginS.o"))); - t!(fs::copy(out_dir.join("crtend.o"), out_dir.join("crtendS.o"))); - out_dir - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Libunwind { - pub target: TargetSelection, -} - -impl Step for Libunwind { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/llvm-project/libunwind") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Libunwind { target: run.target }); - } - - /// Build libunwind.a - fn run(self, builder: &Builder<'_>) -> Self::Output { - builder.update_submodule(&Path::new("src/llvm-project")); - - if builder.config.dry_run() { - return PathBuf::new(); - } - - let out_dir = builder.native_dir(self.target).join("libunwind"); - let root = builder.src.join("src/llvm-project/libunwind"); - - if up_to_date(&root, &out_dir.join("libunwind.a")) { - return out_dir; - } - - let _guard = builder.msg_unstaged(Kind::Build, "libunwind.a", self.target); - t!(fs::create_dir_all(&out_dir)); - - let mut cc_cfg = cc::Build::new(); - let mut cpp_cfg = cc::Build::new(); - - cpp_cfg.cpp(true); - cpp_cfg.cpp_set_stdlib(None); - cpp_cfg.flag("-nostdinc++"); - cpp_cfg.flag("-fno-exceptions"); - cpp_cfg.flag("-fno-rtti"); - cpp_cfg.flag_if_supported("-fvisibility-global-new-delete-hidden"); - - for cfg in [&mut cc_cfg, &mut cpp_cfg].iter_mut() { - if let Some(ar) = builder.ar(self.target) { - cfg.archiver(ar); - } - cfg.target(&self.target.triple); - cfg.host(&builder.config.build.triple); - cfg.warnings(false); - cfg.debug(false); - // get_compiler() need set opt_level first. - cfg.opt_level(3); - cfg.flag("-fstrict-aliasing"); - cfg.flag("-funwind-tables"); - cfg.flag("-fvisibility=hidden"); - cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); - cfg.include(root.join("include")); - cfg.cargo_metadata(false); - cfg.out_dir(&out_dir); - - if self.target.contains("x86_64-fortanix-unknown-sgx") { - cfg.static_flag(true); - cfg.flag("-fno-stack-protector"); - cfg.flag("-ffreestanding"); - cfg.flag("-fexceptions"); - - // easiest way to undefine since no API available in cc::Build to undefine - cfg.flag("-U_FORTIFY_SOURCE"); - cfg.define("_FORTIFY_SOURCE", "0"); - cfg.define("RUST_SGX", "1"); - cfg.define("__NO_STRING_INLINES", None); - cfg.define("__NO_MATH_INLINES", None); - cfg.define("_LIBUNWIND_IS_BAREMETAL", None); - cfg.define("__LIBUNWIND_IS_NATIVE_ONLY", None); - cfg.define("NDEBUG", None); - } - if self.target.contains("windows") { - cfg.define("_LIBUNWIND_HIDE_SYMBOLS", "1"); - cfg.define("_LIBUNWIND_IS_NATIVE_ONLY", "1"); - } - } - - cc_cfg.compiler(builder.cc(self.target)); - if let Ok(cxx) = builder.cxx(self.target) { - cpp_cfg.compiler(cxx); - } else { - cc_cfg.compiler(builder.cc(self.target)); - } - - // Don't set this for clang - // By default, Clang builds C code in GNU C17 mode. - // By default, Clang builds C++ code according to the C++98 standard, - // with many C++11 features accepted as extensions. - if cc_cfg.get_compiler().is_like_gnu() { - cc_cfg.flag("-std=c99"); - } - if cpp_cfg.get_compiler().is_like_gnu() { - cpp_cfg.flag("-std=c++11"); - } - - if self.target.contains("x86_64-fortanix-unknown-sgx") || self.target.contains("musl") { - // use the same GCC C compiler command to compile C++ code so we do not need to setup the - // C++ compiler env variables on the builders. - // Don't set this for clang++, as clang++ is able to compile this without libc++. - if cpp_cfg.get_compiler().is_like_gnu() { - cpp_cfg.cpp(false); - cpp_cfg.compiler(builder.cc(self.target)); - } - } - - let mut c_sources = vec![ - "Unwind-sjlj.c", - "UnwindLevel1-gcc-ext.c", - "UnwindLevel1.c", - "UnwindRegistersRestore.S", - "UnwindRegistersSave.S", - ]; - - let cpp_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"]; - let cpp_len = cpp_sources.len(); - - if self.target.contains("x86_64-fortanix-unknown-sgx") { - c_sources.push("UnwindRustSgx.c"); - } - - for src in c_sources { - cc_cfg.file(root.join("src").join(src).canonicalize().unwrap()); - } - - for src in &cpp_sources { - cpp_cfg.file(root.join("src").join(src).canonicalize().unwrap()); - } - - cpp_cfg.compile("unwind-cpp"); - - // FIXME: https://github.com/alexcrichton/cc-rs/issues/545#issuecomment-679242845 - let mut count = 0; - for entry in fs::read_dir(&out_dir).unwrap() { - let file = entry.unwrap().path().canonicalize().unwrap(); - if file.is_file() && file.extension() == Some(OsStr::new("o")) { - // file name starts with "Unwind-EHABI", "Unwind-seh" or "libunwind" - let file_name = file.file_name().unwrap().to_str().expect("UTF-8 file name"); - if cpp_sources.iter().any(|f| file_name.starts_with(&f[..f.len() - 4])) { - cc_cfg.object(&file); - count += 1; - } - } - } - assert_eq!(cpp_len, count, "Can't get object files from {:?}", &out_dir); - - cc_cfg.compile("unwind"); - out_dir - } -} diff --git a/src/bootstrap/metadata.rs b/src/bootstrap/metadata.rs deleted file mode 100644 index 3b20ceac8..000000000 --- a/src/bootstrap/metadata.rs +++ /dev/null @@ -1,101 +0,0 @@ -use std::path::PathBuf; -use std::process::Command; - -use serde_derive::Deserialize; - -use crate::cache::INTERNER; -use crate::util::output; -use crate::{t, Build, Crate}; - -/// For more information, see the output of -/// -#[derive(Debug, Deserialize)] -struct Output { - packages: Vec, -} - -/// For more information, see the output of -/// -#[derive(Debug, Deserialize)] -struct Package { - name: String, - source: Option, - manifest_path: String, - dependencies: Vec, - targets: Vec, -} - -/// For more information, see the output of -/// -#[derive(Debug, Deserialize)] -struct Dependency { - name: String, - source: Option, -} - -#[derive(Debug, Deserialize)] -struct Target { - kind: Vec, -} - -/// Collects and stores package metadata of each workspace members into `build`, -/// by executing `cargo metadata` commands. -pub fn build(build: &mut Build) { - for package in workspace_members(build) { - if package.source.is_none() { - let name = INTERNER.intern_string(package.name); - let mut path = PathBuf::from(package.manifest_path); - path.pop(); - let deps = package - .dependencies - .into_iter() - .filter(|dep| dep.source.is_none()) - .map(|dep| INTERNER.intern_string(dep.name)) - .collect(); - let has_lib = package.targets.iter().any(|t| t.kind.iter().any(|k| k == "lib")); - let krate = Crate { name, deps, path, has_lib }; - let relative_path = krate.local_path(build); - build.crates.insert(name, krate); - let existing_path = build.crate_paths.insert(relative_path, name); - assert!( - existing_path.is_none(), - "multiple crates with the same path: {}", - existing_path.unwrap() - ); - } - } -} - -/// Invokes `cargo metadata` to get package metadata of each workspace member. -/// -/// Note that `src/tools/cargo` is no longer a workspace member but we still -/// treat it as one here, by invoking an additional `cargo metadata` command. -fn workspace_members(build: &Build) -> impl Iterator { - let collect_metadata = |manifest_path| { - let mut cargo = Command::new(&build.initial_cargo); - cargo - // Will read the libstd Cargo.toml - // which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .arg("metadata") - .arg("--format-version") - .arg("1") - .arg("--no-deps") - .arg("--manifest-path") - .arg(build.src.join(manifest_path)); - let metadata_output = output(&mut cargo); - let Output { packages, .. } = t!(serde_json::from_str(&metadata_output)); - packages - }; - - // Collects `metadata.packages` from all workspaces. - let packages = collect_metadata("Cargo.toml"); - let cargo_packages = collect_metadata("src/tools/cargo/Cargo.toml"); - let ra_packages = collect_metadata("src/tools/rust-analyzer/Cargo.toml"); - let bootstrap_packages = collect_metadata("src/bootstrap/Cargo.toml"); - - // We only care about the root package from `src/tool/cargo` workspace. - let cargo_package = cargo_packages.into_iter().find(|pkg| pkg.name == "cargo").into_iter(); - - packages.into_iter().chain(cargo_package).chain(ra_packages).chain(bootstrap_packages) -} diff --git a/src/bootstrap/metrics.rs b/src/bootstrap/metrics.rs deleted file mode 100644 index cf8d33dfc..000000000 --- a/src/bootstrap/metrics.rs +++ /dev/null @@ -1,258 +0,0 @@ -//! This module is responsible for collecting metrics profiling information for the current build -//! and dumping it to disk as JSON, to aid investigations on build and CI performance. -//! -//! As this module requires additional dependencies not present during local builds, it's cfg'd -//! away whenever the `build.metrics` config option is not set to `true`. - -use crate::builder::{Builder, Step}; -use crate::util::t; -use crate::Build; -use build_helper::metrics::{ - JsonInvocation, JsonInvocationSystemStats, JsonNode, JsonRoot, JsonStepSystemStats, Test, - TestOutcome, TestSuite, TestSuiteMetadata, -}; -use std::cell::RefCell; -use std::fs::File; -use std::io::BufWriter; -use std::time::{Duration, Instant, SystemTime}; -use sysinfo::{CpuExt, System, SystemExt}; - -// Update this number whenever a breaking change is made to the build metrics. -// -// The output format is versioned for two reasons: -// -// - The metadata is intended to be consumed by external tooling, and exposing a format version -// helps the tools determine whether they're compatible with a metrics file. -// -// - If a developer enables build metrics in their local checkout, making a breaking change to the -// metrics format would result in a hard-to-diagnose error message when an existing metrics file -// is not compatible with the new changes. With a format version number, bootstrap can discard -// incompatible metrics files instead of appending metrics to them. -// -// Version changelog: -// -// - v0: initial version -// - v1: replaced JsonNode::Test with JsonNode::TestSuite -// -const CURRENT_FORMAT_VERSION: usize = 1; - -pub(crate) struct BuildMetrics { - state: RefCell, -} - -/// NOTE: this isn't really cloning anything, but `x suggest` doesn't need metrics so this is probably ok. -impl Clone for BuildMetrics { - fn clone(&self) -> Self { - Self::init() - } -} - -impl BuildMetrics { - pub(crate) fn init() -> Self { - let state = RefCell::new(MetricsState { - finished_steps: Vec::new(), - running_steps: Vec::new(), - - system_info: System::new(), - timer_start: None, - invocation_timer_start: Instant::now(), - invocation_start: SystemTime::now(), - }); - - BuildMetrics { state } - } - - pub(crate) fn enter_step(&self, step: &S, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - - // Consider all the stats gathered so far as the parent's. - if !state.running_steps.is_empty() { - self.collect_stats(&mut *state); - } - - state.system_info.refresh_cpu(); - state.timer_start = Some(Instant::now()); - - state.running_steps.push(StepMetrics { - type_: std::any::type_name::().into(), - debug_repr: format!("{step:?}"), - - cpu_usage_time_sec: 0.0, - duration_excluding_children_sec: Duration::ZERO, - - children: Vec::new(), - test_suites: Vec::new(), - }); - } - - pub(crate) fn exit_step(&self, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - - self.collect_stats(&mut *state); - - let step = state.running_steps.pop().unwrap(); - if state.running_steps.is_empty() { - state.finished_steps.push(step); - state.timer_start = None; - } else { - state.running_steps.last_mut().unwrap().children.push(step); - - // Start collecting again for the parent step. - state.system_info.refresh_cpu(); - state.timer_start = Some(Instant::now()); - } - } - - pub(crate) fn begin_test_suite(&self, metadata: TestSuiteMetadata, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - let step = state.running_steps.last_mut().unwrap(); - step.test_suites.push(TestSuite { metadata, tests: Vec::new() }); - } - - pub(crate) fn record_test(&self, name: &str, outcome: TestOutcome, builder: &Builder<'_>) { - // Do not record dry runs, as they'd be duplicates of the actual steps. - if builder.config.dry_run() { - return; - } - - let mut state = self.state.borrow_mut(); - let step = state.running_steps.last_mut().unwrap(); - - if let Some(test_suite) = step.test_suites.last_mut() { - test_suite.tests.push(Test { name: name.to_string(), outcome }); - } else { - panic!("metrics.record_test() called without calling metrics.begin_test_suite() first"); - } - } - - fn collect_stats(&self, state: &mut MetricsState) { - let step = state.running_steps.last_mut().unwrap(); - - let elapsed = state.timer_start.unwrap().elapsed(); - step.duration_excluding_children_sec += elapsed; - - state.system_info.refresh_cpu(); - let cpu = state.system_info.cpus().iter().map(|p| p.cpu_usage()).sum::(); - step.cpu_usage_time_sec += cpu as f64 / 100.0 * elapsed.as_secs_f64(); - } - - pub(crate) fn persist(&self, build: &Build) { - let mut state = self.state.borrow_mut(); - assert!(state.running_steps.is_empty(), "steps are still executing"); - - let dest = build.out.join("metrics.json"); - - let mut system = System::new(); - system.refresh_cpu(); - system.refresh_memory(); - - let system_stats = JsonInvocationSystemStats { - cpu_threads_count: system.cpus().len(), - cpu_model: system.cpus()[0].brand().into(), - - memory_total_bytes: system.total_memory(), - }; - let steps = std::mem::take(&mut state.finished_steps); - - // Some of our CI builds consist of multiple independent CI invocations. Ensure all the - // previous invocations are still present in the resulting file. - let mut invocations = match std::fs::read(&dest) { - Ok(contents) => { - // We first parse just the format_version field to have the check succeed even if - // the rest of the contents are not valid anymore. - let version: OnlyFormatVersion = t!(serde_json::from_slice(&contents)); - if version.format_version == CURRENT_FORMAT_VERSION { - t!(serde_json::from_slice::(&contents)).invocations - } else { - println!( - "warning: overriding existing build/metrics.json, as it's not \ - compatible with build metrics format version {CURRENT_FORMAT_VERSION}." - ); - Vec::new() - } - } - Err(err) => { - if err.kind() != std::io::ErrorKind::NotFound { - panic!("failed to open existing metrics file at {}: {err}", dest.display()); - } - Vec::new() - } - }; - invocations.push(JsonInvocation { - start_time: state - .invocation_start - .duration_since(SystemTime::UNIX_EPOCH) - .unwrap() - .as_secs(), - duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(), - children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(), - }); - - let json = JsonRoot { format_version: CURRENT_FORMAT_VERSION, system_stats, invocations }; - - t!(std::fs::create_dir_all(dest.parent().unwrap())); - let mut file = BufWriter::new(t!(File::create(&dest))); - t!(serde_json::to_writer(&mut file, &json)); - } - - fn prepare_json_step(&self, step: StepMetrics) -> JsonNode { - let mut children = Vec::new(); - children.extend(step.children.into_iter().map(|child| self.prepare_json_step(child))); - children.extend(step.test_suites.into_iter().map(JsonNode::TestSuite)); - - JsonNode::RustbuildStep { - type_: step.type_, - debug_repr: step.debug_repr, - - duration_excluding_children_sec: step.duration_excluding_children_sec.as_secs_f64(), - system_stats: JsonStepSystemStats { - cpu_utilization_percent: step.cpu_usage_time_sec * 100.0 - / step.duration_excluding_children_sec.as_secs_f64(), - }, - - children, - } - } -} - -struct MetricsState { - finished_steps: Vec, - running_steps: Vec, - - system_info: System, - timer_start: Option, - invocation_timer_start: Instant, - invocation_start: SystemTime, -} - -struct StepMetrics { - type_: String, - debug_repr: String, - - cpu_usage_time_sec: f64, - duration_excluding_children_sec: Duration, - - children: Vec, - test_suites: Vec, -} - -#[derive(serde_derive::Deserialize)] -struct OnlyFormatVersion { - #[serde(default)] // For version 0 the field was not present. - format_version: usize, -} diff --git a/src/bootstrap/render_tests.rs b/src/bootstrap/render_tests.rs deleted file mode 100644 index 6802bf451..000000000 --- a/src/bootstrap/render_tests.rs +++ /dev/null @@ -1,400 +0,0 @@ -//! This module renders the JSON output of libtest into a human-readable form, trying to be as -//! similar to libtest's native output as possible. -//! -//! This is needed because we need to use libtest in JSON mode to extract granular information -//! about the executed tests. Doing so suppresses the human-readable output, and (compared to Cargo -//! and rustc) libtest doesn't include the rendered human-readable output as a JSON field. We had -//! to reimplement all the rendering logic in this module because of that. - -use crate::builder::Builder; -use std::io::{BufRead, BufReader, Read, Write}; -use std::process::{ChildStdout, Command, Stdio}; -use std::time::Duration; -use termcolor::{Color, ColorSpec, WriteColor}; - -const TERSE_TESTS_PER_LINE: usize = 88; - -pub(crate) fn add_flags_and_try_run_tests(builder: &Builder<'_>, cmd: &mut Command) -> bool { - if cmd.get_args().position(|arg| arg == "--").is_none() { - cmd.arg("--"); - } - cmd.args(&["-Z", "unstable-options", "--format", "json"]); - - try_run_tests(builder, cmd, false) -} - -pub(crate) fn try_run_tests(builder: &Builder<'_>, cmd: &mut Command, stream: bool) -> bool { - if builder.config.dry_run() { - return true; - } - - if !run_tests(builder, cmd, stream) { - if builder.fail_fast { - crate::exit!(1); - } else { - let mut failures = builder.delayed_failures.borrow_mut(); - failures.push(format!("{cmd:?}")); - false - } - } else { - true - } -} - -fn run_tests(builder: &Builder<'_>, cmd: &mut Command, stream: bool) -> bool { - cmd.stdout(Stdio::piped()); - - builder.verbose(&format!("running: {cmd:?}")); - - let mut process = cmd.spawn().unwrap(); - - // This runs until the stdout of the child is closed, which means the child exited. We don't - // run this on another thread since the builder is not Sync. - let renderer = Renderer::new(process.stdout.take().unwrap(), builder); - if stream { - renderer.stream_all(); - } else { - renderer.render_all(); - } - - let result = process.wait_with_output().unwrap(); - if !result.status.success() && builder.is_verbose() { - println!( - "\n\ncommand did not execute successfully: {cmd:?}\n\ - expected success, got: {}", - result.status - ); - } - - result.status.success() -} - -struct Renderer<'a> { - stdout: BufReader, - failures: Vec, - benches: Vec, - builder: &'a Builder<'a>, - tests_count: Option, - executed_tests: usize, - terse_tests_in_line: usize, -} - -impl<'a> Renderer<'a> { - fn new(stdout: ChildStdout, builder: &'a Builder<'a>) -> Self { - Self { - stdout: BufReader::new(stdout), - benches: Vec::new(), - failures: Vec::new(), - builder, - tests_count: None, - executed_tests: 0, - terse_tests_in_line: 0, - } - } - - fn render_all(mut self) { - let mut line = Vec::new(); - loop { - line.clear(); - match self.stdout.read_until(b'\n', &mut line) { - Ok(_) => {} - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(err) => panic!("failed to read output of test runner: {err}"), - } - if line.is_empty() { - break; - } - - match serde_json::from_slice(&line) { - Ok(parsed) => self.render_message(parsed), - Err(_err) => { - // Handle non-JSON output, for example when --nocapture is passed. - let mut stdout = std::io::stdout(); - stdout.write_all(&line).unwrap(); - let _ = stdout.flush(); - } - } - } - } - - /// Renders the stdout characters one by one - fn stream_all(mut self) { - let mut buffer = [0; 1]; - loop { - match self.stdout.read(&mut buffer) { - Ok(0) => break, - Ok(_) => { - let mut stdout = std::io::stdout(); - stdout.write_all(&buffer).unwrap(); - let _ = stdout.flush(); - } - Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(err) => panic!("failed to read output of test runner: {err}"), - } - } - } - - fn render_test_outcome(&mut self, outcome: Outcome<'_>, test: &TestOutcome) { - self.executed_tests += 1; - - #[cfg(feature = "build-metrics")] - self.builder.metrics.record_test( - &test.name, - match outcome { - Outcome::Ok | Outcome::BenchOk => build_helper::metrics::TestOutcome::Passed, - Outcome::Failed => build_helper::metrics::TestOutcome::Failed, - Outcome::Ignored { reason } => build_helper::metrics::TestOutcome::Ignored { - ignore_reason: reason.map(|s| s.to_string()), - }, - }, - self.builder, - ); - - if self.builder.config.verbose_tests { - self.render_test_outcome_verbose(outcome, test); - } else { - self.render_test_outcome_terse(outcome, test); - } - } - - fn render_test_outcome_verbose(&self, outcome: Outcome<'_>, test: &TestOutcome) { - print!("test {} ... ", test.name); - self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); - if let Some(exec_time) = test.exec_time { - print!(" ({exec_time:.2?})"); - } - println!(); - } - - fn render_test_outcome_terse(&mut self, outcome: Outcome<'_>, _: &TestOutcome) { - if self.terse_tests_in_line != 0 && self.terse_tests_in_line % TERSE_TESTS_PER_LINE == 0 { - if let Some(total) = self.tests_count { - let total = total.to_string(); - let executed = format!("{:>width$}", self.executed_tests - 1, width = total.len()); - print!(" {executed}/{total}"); - } - println!(); - self.terse_tests_in_line = 0; - } - - self.terse_tests_in_line += 1; - self.builder.colored_stdout(|stdout| outcome.write_short(stdout)).unwrap(); - let _ = std::io::stdout().flush(); - } - - fn render_suite_outcome(&self, outcome: Outcome<'_>, suite: &SuiteOutcome) { - // The terse output doesn't end with a newline, so we need to add it ourselves. - if !self.builder.config.verbose_tests { - println!(); - } - - if !self.failures.is_empty() { - println!("\nfailures:\n"); - for failure in &self.failures { - if failure.stdout.is_some() || failure.message.is_some() { - println!("---- {} stdout ----", failure.name); - if let Some(stdout) = &failure.stdout { - println!("{stdout}"); - } - if let Some(message) = &failure.message { - println!("note: {message}"); - } - } - } - - println!("\nfailures:"); - for failure in &self.failures { - println!(" {}", failure.name); - } - } - - if !self.benches.is_empty() { - println!("\nbenchmarks:"); - - let mut rows = Vec::new(); - for bench in &self.benches { - rows.push(( - &bench.name, - format!("{:.2?}/iter", Duration::from_nanos(bench.median)), - format!("+/- {:.2?}", Duration::from_nanos(bench.deviation)), - )); - } - - let max_0 = rows.iter().map(|r| r.0.len()).max().unwrap_or(0); - let max_1 = rows.iter().map(|r| r.1.len()).max().unwrap_or(0); - let max_2 = rows.iter().map(|r| r.2.len()).max().unwrap_or(0); - for row in &rows { - println!(" {:max_1$} {:>max_2$}", row.0, row.1, row.2); - } - } - - print!("\ntest result: "); - self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); - println!( - ". {} passed; {} failed; {} ignored; {} measured; {} filtered out; \ - finished in {:.2?}\n", - suite.passed, - suite.failed, - suite.ignored, - suite.measured, - suite.filtered_out, - Duration::from_secs_f64(suite.exec_time) - ); - } - - fn render_message(&mut self, message: Message) { - match message { - Message::Suite(SuiteMessage::Started { test_count }) => { - println!("\nrunning {test_count} tests"); - self.executed_tests = 0; - self.terse_tests_in_line = 0; - self.tests_count = Some(test_count); - } - Message::Suite(SuiteMessage::Ok(outcome)) => { - self.render_suite_outcome(Outcome::Ok, &outcome); - } - Message::Suite(SuiteMessage::Failed(outcome)) => { - self.render_suite_outcome(Outcome::Failed, &outcome); - } - Message::Bench(outcome) => { - // The formatting for benchmarks doesn't replicate 1:1 the formatting libtest - // outputs, mostly because libtest's formatting is broken in terse mode, which is - // the default used by our monorepo. We use a different formatting instead: - // successful benchmarks are just showed as "benchmarked"/"b", and the details are - // outputted at the bottom like failures. - let fake_test_outcome = TestOutcome { - name: outcome.name.clone(), - exec_time: None, - stdout: None, - message: None, - }; - self.render_test_outcome(Outcome::BenchOk, &fake_test_outcome); - self.benches.push(outcome); - } - Message::Test(TestMessage::Ok(outcome)) => { - self.render_test_outcome(Outcome::Ok, &outcome); - } - Message::Test(TestMessage::Ignored(outcome)) => { - self.render_test_outcome( - Outcome::Ignored { reason: outcome.message.as_deref() }, - &outcome, - ); - } - Message::Test(TestMessage::Failed(outcome)) => { - self.render_test_outcome(Outcome::Failed, &outcome); - self.failures.push(outcome); - } - Message::Test(TestMessage::Timeout { name }) => { - println!("test {name} has been running for a long time"); - } - Message::Test(TestMessage::Started) => {} // Not useful - } - } -} - -enum Outcome<'a> { - Ok, - BenchOk, - Failed, - Ignored { reason: Option<&'a str> }, -} - -impl Outcome<'_> { - fn write_short(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { - match self { - Outcome::Ok => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; - write!(writer, ".")?; - } - Outcome::BenchOk => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; - write!(writer, "b")?; - } - Outcome::Failed => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; - write!(writer, "F")?; - } - Outcome::Ignored { .. } => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; - write!(writer, "i")?; - } - } - writer.reset() - } - - fn write_long(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { - match self { - Outcome::Ok => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; - write!(writer, "ok")?; - } - Outcome::BenchOk => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; - write!(writer, "benchmarked")?; - } - Outcome::Failed => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; - write!(writer, "FAILED")?; - } - Outcome::Ignored { reason } => { - writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; - write!(writer, "ignored")?; - if let Some(reason) = reason { - write!(writer, ", {reason}")?; - } - } - } - writer.reset() - } -} - -#[derive(serde_derive::Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -enum Message { - Suite(SuiteMessage), - Test(TestMessage), - Bench(BenchOutcome), -} - -#[derive(serde_derive::Deserialize)] -#[serde(tag = "event", rename_all = "snake_case")] -enum SuiteMessage { - Ok(SuiteOutcome), - Failed(SuiteOutcome), - Started { test_count: usize }, -} - -#[derive(serde_derive::Deserialize)] -struct SuiteOutcome { - passed: usize, - failed: usize, - ignored: usize, - measured: usize, - filtered_out: usize, - exec_time: f64, -} - -#[derive(serde_derive::Deserialize)] -#[serde(tag = "event", rename_all = "snake_case")] -enum TestMessage { - Ok(TestOutcome), - Failed(TestOutcome), - Ignored(TestOutcome), - Timeout { name: String }, - Started, -} - -#[derive(serde_derive::Deserialize)] -struct BenchOutcome { - name: String, - median: u64, - deviation: u64, -} - -#[derive(serde_derive::Deserialize)] -struct TestOutcome { - name: String, - exec_time: Option, - stdout: Option, - message: Option, -} diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs deleted file mode 100644 index 4082f5bb9..000000000 --- a/src/bootstrap/run.rs +++ /dev/null @@ -1,297 +0,0 @@ -use std::path::PathBuf; -use std::process::Command; - -use clap_complete::shells; - -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::config::TargetSelection; -use crate::dist::distdir; -use crate::flags::get_completion; -use crate::test; -use crate::tool::{self, SourceType, Tool}; -use crate::util::output; -use crate::Mode; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ExpandYamlAnchors; - -impl Step for ExpandYamlAnchors { - type Output = (); - - /// Runs the `expand-yaml_anchors` tool. - /// - /// This tool in `src/tools` reads the CI configuration files written in YAML and expands the - /// anchors in them, since GitHub Actions doesn't support them. - fn run(self, builder: &Builder<'_>) { - builder.info("Expanding YAML anchors in the GitHub Actions configuration"); - builder.run_delaying_failure( - &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src), - ); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/expand-yaml-anchors") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ExpandYamlAnchors); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct BuildManifest; - -impl Step for BuildManifest { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/build-manifest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(BuildManifest); - } - - fn run(self, builder: &Builder<'_>) { - // This gets called by `promote-release` - // (https://github.com/rust-lang/promote-release). - let mut cmd = builder.tool_cmd(Tool::BuildManifest); - let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| { - panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") - }); - let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| { - panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") - }); - - let today = output(Command::new("date").arg("+%Y-%m-%d")); - - cmd.arg(sign); - cmd.arg(distdir(builder)); - cmd.arg(today.trim()); - cmd.arg(addr); - cmd.arg(&builder.config.channel); - - builder.create_dir(&distdir(builder)); - builder.run(&mut cmd); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct BumpStage0; - -impl Step for BumpStage0 { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/bump-stage0") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(BumpStage0); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut cmd = builder.tool_cmd(Tool::BumpStage0); - cmd.args(builder.config.args()); - builder.run(&mut cmd); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct ReplaceVersionPlaceholder; - -impl Step for ReplaceVersionPlaceholder { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/replace-version-placeholder") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ReplaceVersionPlaceholder); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let mut cmd = builder.tool_cmd(Tool::ReplaceVersionPlaceholder); - cmd.arg(&builder.src); - builder.run(&mut cmd); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Miri { - stage: u32, - host: TargetSelection, - target: TargetSelection, -} - -impl Step for Miri { - type Output = (); - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/miri") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { - stage: run.builder.top_stage, - host: run.build_triple(), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let target = self.target; - let compiler = builder.compiler(stage, host); - - let miri = builder - .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - let miri_sysroot = test::Miri::build_miri_sysroot(builder, compiler, &miri, target); - - // # Run miri. - // Running it via `cargo run` as that figures out the right dylib path. - // add_rustc_lib_path does not add the path that contains librustc_driver-<...>.so. - let mut miri = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "run", - "src/tools/miri", - SourceType::InTree, - &[], - ); - miri.add_rustc_lib_path(builder, compiler); - // Forward arguments. - miri.arg("--").arg("--target").arg(target.rustc_target_arg()); - miri.args(builder.config.args()); - - // miri tests need to know about the stage sysroot - miri.env("MIRI_SYSROOT", &miri_sysroot); - - let mut miri = Command::from(miri); - builder.run(&mut miri); - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct CollectLicenseMetadata; - -impl Step for CollectLicenseMetadata { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/collect-license-metadata") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CollectLicenseMetadata); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let Some(reuse) = &builder.config.reuse else { - panic!("REUSE is required to collect the license metadata"); - }; - - // Temporary location, it will be moved to src/etc once it's accurate. - let dest = builder.out.join("license-metadata.json"); - - let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata); - cmd.env("REUSE_EXE", reuse); - cmd.env("DEST", &dest); - builder.run(&mut cmd); - - dest - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct GenerateCopyright; - -impl Step for GenerateCopyright { - type Output = PathBuf; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/generate-copyright") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(GenerateCopyright); - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - let license_metadata = builder.ensure(CollectLicenseMetadata); - - // Temporary location, it will be moved to the proper one once it's accurate. - let dest = builder.out.join("COPYRIGHT.md"); - - let mut cmd = builder.tool_cmd(Tool::GenerateCopyright); - cmd.env("LICENSE_METADATA", &license_metadata); - cmd.env("DEST", &dest); - builder.run(&mut cmd); - - dest - } -} - -#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] -pub struct GenerateWindowsSys; - -impl Step for GenerateWindowsSys { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/generate-windows-sys") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(GenerateWindowsSys); - } - - fn run(self, builder: &Builder<'_>) { - let mut cmd = builder.tool_cmd(Tool::GenerateWindowsSys); - cmd.arg(&builder.src); - builder.run(&mut cmd); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct GenerateCompletions; - -impl Step for GenerateCompletions { - type Output = (); - - /// Uses `clap_complete` to generate shell completions. - fn run(self, builder: &Builder<'_>) { - // FIXME(clubby789): enable zsh when clap#4898 is fixed - let [bash, fish, powershell] = ["x.py.sh", "x.py.fish", "x.py.ps1"] - .map(|filename| builder.src.join("src/etc/completions").join(filename)); - if let Some(comp) = get_completion(shells::Bash, &bash) { - std::fs::write(&bash, comp).expect("writing bash completion"); - } - if let Some(comp) = get_completion(shells::Fish, &fish) { - std::fs::write(&fish, comp).expect("writing fish completion"); - } - if let Some(comp) = get_completion(shells::PowerShell, &powershell) { - std::fs::write(&powershell, comp).expect("writing powershell completion"); - } - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("generate-completions") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(GenerateCompletions); - } -} diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs deleted file mode 100644 index 0febdf250..000000000 --- a/src/bootstrap/sanity.rs +++ /dev/null @@ -1,267 +0,0 @@ -//! Sanity checking performed by rustbuild before actually executing anything. -//! -//! This module contains the implementation of ensuring that the build -//! environment looks reasonable before progressing. This will verify that -//! various programs like git and python exist, along with ensuring that all C -//! compilers for cross-compiling are found. -//! -//! In theory if we get past this phase it's a bug if a build fails, but in -//! practice that's likely not true! - -use std::collections::HashMap; -use std::env; -use std::ffi::{OsStr, OsString}; -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -use crate::cache::INTERNER; -use crate::config::Target; -use crate::util::output; -use crate::Build; - -pub struct Finder { - cache: HashMap>, - path: OsString, -} - -impl Finder { - pub fn new() -> Self { - Self { cache: HashMap::new(), path: env::var_os("PATH").unwrap_or_default() } - } - - pub fn maybe_have>(&mut self, cmd: S) -> Option { - let cmd: OsString = cmd.into(); - let path = &self.path; - self.cache - .entry(cmd.clone()) - .or_insert_with(|| { - for path in env::split_paths(path) { - let target = path.join(&cmd); - let mut cmd_exe = cmd.clone(); - cmd_exe.push(".exe"); - - if target.is_file() // some/path/git - || path.join(&cmd_exe).exists() // some/path/git.exe - || target.join(&cmd_exe).exists() - // some/path/git/git.exe - { - return Some(target); - } - } - None - }) - .clone() - } - - pub fn must_have>(&mut self, cmd: S) -> PathBuf { - self.maybe_have(&cmd).unwrap_or_else(|| { - panic!("\n\ncouldn't find required command: {:?}\n\n", cmd.as_ref()); - }) - } -} - -pub fn check(build: &mut Build) { - let skip_target_sanity = - env::var_os("BOOTSTRAP_SKIP_TARGET_SANITY").is_some_and(|s| s == "1" || s == "true"); - - let path = env::var_os("PATH").unwrap_or_default(); - // On Windows, quotes are invalid characters for filename paths, and if - // one is present as part of the PATH then that can lead to the system - // being unable to identify the files properly. See - // https://github.com/rust-lang/rust/issues/34959 for more details. - if cfg!(windows) && path.to_string_lossy().contains('\"') { - panic!("PATH contains invalid character '\"'"); - } - - let mut cmd_finder = Finder::new(); - // If we've got a git directory we're gonna need git to update - // submodules and learn about various other aspects. - if build.rust_info().is_managed_git_subrepository() { - cmd_finder.must_have("git"); - } - - // We need cmake, but only if we're actually building LLVM or sanitizers. - let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) - && build - .hosts - .iter() - .map(|host| { - build - .config - .target_config - .get(host) - .map(|config| config.llvm_config.is_none()) - .unwrap_or(true) - }) - .any(|build_llvm_ourselves| build_llvm_ourselves); - - let need_cmake = building_llvm || build.config.any_sanitizers_enabled(); - if need_cmake && cmd_finder.maybe_have("cmake").is_none() { - eprintln!( - " -Couldn't find required command: cmake - -You should install cmake, or set `download-ci-llvm = true` in the -`[llvm]` section of `config.toml` to download LLVM rather -than building it. -" - ); - crate::exit!(1); - } - - build.config.python = build - .config - .python - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| env::var_os("BOOTSTRAP_PYTHON").map(PathBuf::from)) // set by bootstrap.py - .or_else(|| cmd_finder.maybe_have("python")) - .or_else(|| cmd_finder.maybe_have("python3")) - .or_else(|| cmd_finder.maybe_have("python2")); - - build.config.nodejs = build - .config - .nodejs - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("node")) - .or_else(|| cmd_finder.maybe_have("nodejs")); - - build.config.npm = build - .config - .npm - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("npm")); - - build.config.gdb = build - .config - .gdb - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("gdb")); - - build.config.reuse = build - .config - .reuse - .take() - .map(|p| cmd_finder.must_have(p)) - .or_else(|| cmd_finder.maybe_have("reuse")); - - // We're gonna build some custom C code here and there, host triples - // also build some C++ shims for LLVM so we need a C++ compiler. - for target in &build.targets { - // On emscripten we don't actually need the C compiler to just - // build the target artifacts, only for testing. For the sake - // of easier bot configuration, just skip detection. - if target.contains("emscripten") { - continue; - } - - // We don't use a C compiler on wasm32 - if target.contains("wasm32") { - continue; - } - - // Some environments don't want or need these tools, such as when testing Miri. - // FIXME: it would be better to refactor this code to split necessary setup from pure sanity - // checks, and have a regular flag for skipping the latter. Also see - // . - if skip_target_sanity { - continue; - } - - if !build.config.dry_run() { - cmd_finder.must_have(build.cc(*target)); - if let Some(ar) = build.ar(*target) { - cmd_finder.must_have(ar); - } - } - } - - for host in &build.hosts { - if !build.config.dry_run() { - cmd_finder.must_have(build.cxx(*host).unwrap()); - } - } - - if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { - // Externally configured LLVM requires FileCheck to exist - let filecheck = build.llvm_filecheck(build.build); - if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { - panic!("FileCheck executable {filecheck:?} does not exist"); - } - } - - for target in &build.targets { - build - .config - .target_config - .entry(*target) - .or_insert_with(|| Target::from_triple(&target.triple)); - - if (target.contains("-none-") || target.contains("nvptx")) - && build.no_std(*target) == Some(false) - { - panic!("All the *-none-* and nvptx* targets are no-std targets") - } - - // Some environments don't want or need these tools, such as when testing Miri. - // FIXME: it would be better to refactor this code to split necessary setup from pure sanity - // checks, and have a regular flag for skipping the latter. Also see - // . - if skip_target_sanity { - continue; - } - - // Make sure musl-root is valid. - if target.contains("musl") && !target.contains("unikraft") { - // If this is a native target (host is also musl) and no musl-root is given, - // fall back to the system toolchain in /usr before giving up - if build.musl_root(*target).is_none() && build.config.build == *target { - let target = build.config.target_config.entry(*target).or_default(); - target.musl_root = Some("/usr".into()); - } - match build.musl_libdir(*target) { - Some(libdir) => { - if fs::metadata(libdir.join("libc.a")).is_err() { - panic!("couldn't find libc.a in musl libdir: {}", libdir.display()); - } - } - None => panic!( - "when targeting MUSL either the rust.musl-root \ - option or the target.$TARGET.musl-root option must \ - be specified in config.toml" - ), - } - } - - if need_cmake && target.contains("msvc") { - // There are three builds of cmake on windows: MSVC, MinGW, and - // Cygwin. The Cygwin build does not have generators for Visual - // Studio, so detect that here and error. - let out = output(Command::new("cmake").arg("--help")); - if !out.contains("Visual Studio") { - panic!( - " -cmake does not support Visual Studio generators. - -This is likely due to it being an msys/cygwin build of cmake, -rather than the required windows version, built using MinGW -or Visual Studio. - -If you are building under msys2 try installing the mingw-w64-x86_64-cmake -package instead of cmake: - -$ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake -" - ); - } - } - } - - if let Some(ref s) = build.config.ccache { - cmd_finder.must_have(s); - } -} diff --git a/src/bootstrap/setup.rs b/src/bootstrap/setup.rs deleted file mode 100644 index ef0234957..000000000 --- a/src/bootstrap/setup.rs +++ /dev/null @@ -1,599 +0,0 @@ -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::Config; -use crate::{t, VERSION}; -use sha2::Digest; -use std::env::consts::EXE_SUFFIX; -use std::fmt::Write as _; -use std::fs::File; -use std::io::Write; -use std::path::{Path, PathBuf, MAIN_SEPARATOR}; -use std::process::Command; -use std::str::FromStr; -use std::{fmt, fs, io}; - -#[cfg(test)] -mod tests; - -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub enum Profile { - Compiler, - Codegen, - Library, - Tools, - Dist, - None, -} - -/// A list of historical hashes of `src/etc/rust_analyzer_settings.json`. -/// New entries should be appended whenever this is updated so we can detect -/// outdated vs. user-modified settings files. -static SETTINGS_HASHES: &[&str] = &[ - "ea67e259dedf60d4429b6c349a564ffcd1563cf41c920a856d1f5b16b4701ac8", - "56e7bf011c71c5d81e0bf42e84938111847a810eee69d906bba494ea90b51922", - "af1b5efe196aed007577899db9dae15d6dbc923d6fa42fa0934e68617ba9bbe0", - "3468fea433c25fff60be6b71e8a215a732a7b1268b6a83bf10d024344e140541", - "47d227f424bf889b0d899b9cc992d5695e1b78c406e183cd78eafefbe5488923", - "b526bd58d0262dd4dda2bff5bc5515b705fb668a46235ace3e057f807963a11a", -]; -static RUST_ANALYZER_SETTINGS: &str = include_str!("../etc/rust_analyzer_settings.json"); - -impl Profile { - fn include_path(&self, src_path: &Path) -> PathBuf { - PathBuf::from(format!("{}/src/bootstrap/defaults/config.{}.toml", src_path.display(), self)) - } - - pub fn all() -> impl Iterator { - use Profile::*; - // N.B. these are ordered by how they are displayed, not alphabetically - [Library, Compiler, Codegen, Tools, Dist, None].iter().copied() - } - - pub fn purpose(&self) -> String { - use Profile::*; - match self { - Library => "Contribute to the standard library", - Compiler => "Contribute to the compiler itself", - Codegen => "Contribute to the compiler, and also modify LLVM or codegen", - Tools => "Contribute to tools which depend on the compiler, but do not modify it directly (e.g. rustdoc, clippy, miri)", - Dist => "Install Rust from source", - None => "Do not modify `config.toml`" - } - .to_string() - } - - pub fn all_for_help(indent: &str) -> String { - let mut out = String::new(); - for choice in Profile::all() { - writeln!(&mut out, "{}{}: {}", indent, choice, choice.purpose()).unwrap(); - } - out - } - - pub fn as_str(&self) -> &'static str { - match self { - Profile::Compiler => "compiler", - Profile::Codegen => "codegen", - Profile::Library => "library", - Profile::Tools => "tools", - Profile::Dist => "dist", - Profile::None => "none", - } - } -} - -impl FromStr for Profile { - type Err = String; - - fn from_str(s: &str) -> Result { - match s { - "lib" | "library" => Ok(Profile::Library), - "compiler" => Ok(Profile::Compiler), - "llvm" | "codegen" => Ok(Profile::Codegen), - "maintainer" | "dist" | "user" => Ok(Profile::Dist), - "tools" | "tool" | "rustdoc" | "clippy" | "miri" | "rustfmt" | "rls" => { - Ok(Profile::Tools) - } - "none" => Ok(Profile::None), - _ => Err(format!("unknown profile: '{s}'")), - } - } -} - -impl fmt::Display for Profile { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(self.as_str()) - } -} - -impl Step for Profile { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(mut run: ShouldRun<'_>) -> ShouldRun<'_> { - for choice in Profile::all() { - run = run.alias(choice.as_str()); - } - run - } - - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - - // for Profile, `run.paths` will have 1 and only 1 element - // this is because we only accept at most 1 path from user input. - // If user calls `x.py setup` without arguments, the interactive TUI - // will guide user to provide one. - let profile = if run.paths.len() > 1 { - // HACK: `builder` runs this step with all paths if no path was passed. - t!(interactive_path()) - } else { - run.paths - .first() - .unwrap() - .assert_single_path() - .path - .as_path() - .as_os_str() - .to_str() - .unwrap() - .parse() - .unwrap() - }; - - run.builder.ensure(profile); - } - - fn run(self, builder: &Builder<'_>) { - setup(&builder.build.config, self) - } -} - -pub fn setup(config: &Config, profile: Profile) { - let suggestions: &[&str] = match profile { - Profile::Codegen | Profile::Compiler | Profile::None => &["check", "build", "test"], - Profile::Tools => &[ - "check", - "build", - "test tests/rustdoc*", - "test src/tools/clippy", - "test src/tools/miri", - "test src/tools/rustfmt", - ], - Profile::Library => &["check", "build", "test library/std", "doc"], - Profile::Dist => &["dist", "build"], - }; - - println!(); - - println!("To get started, try one of the following commands:"); - for cmd in suggestions { - println!("- `x.py {cmd}`"); - } - - if profile != Profile::Dist { - println!( - "For more suggestions, see https://rustc-dev-guide.rust-lang.org/building/suggested.html" - ); - } - - if profile == Profile::Tools { - eprintln!(); - eprintln!( - "note: the `tools` profile sets up the `stage2` toolchain (use \ - `rustup toolchain link 'name' host/build/stage2` to use rustc)" - ) - } - - let path = &config.config.clone().unwrap_or(PathBuf::from("config.toml")); - setup_config_toml(path, profile, config); -} - -fn setup_config_toml(path: &PathBuf, profile: Profile, config: &Config) { - if profile == Profile::None { - return; - } - if path.exists() { - eprintln!(); - eprintln!( - "error: you asked `x.py` to setup a new config file, but one already exists at `{}`", - path.display() - ); - eprintln!("help: try adding `profile = \"{}\"` at the top of {}", profile, path.display()); - eprintln!( - "note: this will use the configuration in {}", - profile.include_path(&config.src).display() - ); - crate::exit!(1); - } - - let settings = format!( - "# Includes one of the default files in src/bootstrap/defaults\n\ - profile = \"{profile}\"\n\ - changelog-seen = {VERSION}\n" - ); - - t!(fs::write(path, settings)); - - let include_path = profile.include_path(&config.src); - println!("`x.py` will now use the configuration at {}", include_path.display()); -} - -/// Creates a toolchain link for stage1 using `rustup` -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub struct Link; -impl Step for Link { - type Output = (); - const DEFAULT: bool = true; - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("link") - } - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - if let [cmd] = &run.paths[..] { - if cmd.assert_single_path().path.as_path().as_os_str() == "link" { - run.builder.ensure(Link); - } - } - } - fn run(self, builder: &Builder<'_>) -> Self::Output { - let config = &builder.config; - if config.dry_run() { - return; - } - let stage_path = - ["build", config.build.rustc_target_arg(), "stage1"].join(&MAIN_SEPARATOR.to_string()); - - if !rustup_installed() { - eprintln!("`rustup` is not installed; cannot link `stage1` toolchain"); - } else if stage_dir_exists(&stage_path[..]) && !config.dry_run() { - attempt_toolchain_link(&stage_path[..]); - } - } -} - -fn rustup_installed() -> bool { - Command::new("rustup") - .arg("--version") - .stdout(std::process::Stdio::null()) - .output() - .map_or(false, |output| output.status.success()) -} - -fn stage_dir_exists(stage_path: &str) -> bool { - match fs::create_dir(&stage_path) { - Ok(_) => true, - Err(_) => Path::new(&stage_path).exists(), - } -} - -fn attempt_toolchain_link(stage_path: &str) { - if toolchain_is_linked() { - return; - } - - if !ensure_stage1_toolchain_placeholder_exists(stage_path) { - eprintln!( - "Failed to create a template for stage 1 toolchain or confirm that it already exists" - ); - return; - } - - if try_link_toolchain(&stage_path) { - println!( - "Added `stage1` rustup toolchain; try `cargo +stage1 build` on a separate rust project to run a newly-built toolchain" - ); - } else { - eprintln!("`rustup` failed to link stage 1 build to `stage1` toolchain"); - eprintln!( - "To manually link stage 1 build to `stage1` toolchain, run:\n - `rustup toolchain link stage1 {}`", - &stage_path - ); - } -} - -fn toolchain_is_linked() -> bool { - match Command::new("rustup") - .args(&["toolchain", "list"]) - .stdout(std::process::Stdio::piped()) - .output() - { - Ok(toolchain_list) => { - if !String::from_utf8_lossy(&toolchain_list.stdout).contains("stage1") { - return false; - } - // The toolchain has already been linked. - println!( - "`stage1` toolchain already linked; not attempting to link `stage1` toolchain" - ); - } - Err(_) => { - // In this case, we don't know if the `stage1` toolchain has been linked; - // but `rustup` failed, so let's not go any further. - println!( - "`rustup` failed to list current toolchains; not attempting to link `stage1` toolchain" - ); - } - } - true -} - -fn try_link_toolchain(stage_path: &str) -> bool { - Command::new("rustup") - .stdout(std::process::Stdio::null()) - .args(&["toolchain", "link", "stage1", &stage_path]) - .output() - .map_or(false, |output| output.status.success()) -} - -fn ensure_stage1_toolchain_placeholder_exists(stage_path: &str) -> bool { - let pathbuf = PathBuf::from(stage_path); - - if fs::create_dir_all(pathbuf.join("lib")).is_err() { - return false; - }; - - let pathbuf = pathbuf.join("bin"); - if fs::create_dir_all(&pathbuf).is_err() { - return false; - }; - - let pathbuf = pathbuf.join(format!("rustc{EXE_SUFFIX}")); - - if pathbuf.exists() { - return true; - } - - // Take care not to overwrite the file - let result = File::options().append(true).create(true).open(&pathbuf); - if result.is_err() { - return false; - } - - return true; -} - -// Used to get the path for `Subcommand::Setup` -pub fn interactive_path() -> io::Result { - fn abbrev_all() -> impl Iterator { - ('a'..) - .zip(1..) - .map(|(letter, number)| (letter.to_string(), number.to_string())) - .zip(Profile::all()) - } - - fn parse_with_abbrev(input: &str) -> Result { - let input = input.trim().to_lowercase(); - for ((letter, number), profile) in abbrev_all() { - if input == letter || input == number { - return Ok(profile); - } - } - input.parse() - } - - println!("Welcome to the Rust project! What do you want to do with x.py?"); - for ((letter, _), profile) in abbrev_all() { - println!("{}) {}: {}", letter, profile, profile.purpose()); - } - let template = loop { - print!( - "Please choose one ({}): ", - abbrev_all().map(|((l, _), _)| l).collect::>().join("/") - ); - io::stdout().flush()?; - let mut input = String::new(); - io::stdin().read_line(&mut input)?; - if input.is_empty() { - eprintln!("EOF on stdin, when expecting answer to question. Giving up."); - crate::exit!(1); - } - break match parse_with_abbrev(&input) { - Ok(profile) => profile, - Err(err) => { - eprintln!("error: {err}"); - eprintln!("note: press Ctrl+C to exit"); - continue; - } - }; - }; - Ok(template) -} - -#[derive(PartialEq)] -enum PromptResult { - Yes, // y/Y/yes - No, // n/N/no - Print, // p/P/print -} - -/// Prompt a user for a answer, looping until they enter an accepted input or nothing -fn prompt_user(prompt: &str) -> io::Result> { - let mut input = String::new(); - loop { - print!("{prompt} "); - io::stdout().flush()?; - input.clear(); - io::stdin().read_line(&mut input)?; - match input.trim().to_lowercase().as_str() { - "y" | "yes" => return Ok(Some(PromptResult::Yes)), - "n" | "no" => return Ok(Some(PromptResult::No)), - "p" | "print" => return Ok(Some(PromptResult::Print)), - "" => return Ok(None), - _ => { - eprintln!("error: unrecognized option '{}'", input.trim()); - eprintln!("note: press Ctrl+C to exit"); - } - }; - } -} - -/// Installs `src/etc/pre-push.sh` as a Git hook -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub struct Hook; - -impl Step for Hook { - type Output = (); - const DEFAULT: bool = true; - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("hook") - } - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - if let [cmd] = &run.paths[..] { - if cmd.assert_single_path().path.as_path().as_os_str() == "hook" { - run.builder.ensure(Hook); - } - } - } - fn run(self, builder: &Builder<'_>) -> Self::Output { - let config = &builder.config; - if config.dry_run() { - return; - } - t!(install_git_hook_maybe(&config)); - } -} - -// install a git hook to automatically run tidy, if they want -fn install_git_hook_maybe(config: &Config) -> io::Result<()> { - let git = t!(config.git().args(&["rev-parse", "--git-common-dir"]).output().map(|output| { - assert!(output.status.success(), "failed to run `git`"); - PathBuf::from(t!(String::from_utf8(output.stdout)).trim()) - })); - let dst = git.join("hooks").join("pre-push"); - if dst.exists() { - // The git hook has already been set up, or the user already has a custom hook. - return Ok(()); - } - - println!( - "\nRust's CI will automatically fail if it doesn't pass `tidy`, the internal tool for ensuring code quality. -If you'd like, x.py can install a git hook for you that will automatically run `test tidy` before -pushing your code to ensure your code is up to par. If you decide later that this behavior is -undesirable, simply delete the `pre-push` file from .git/hooks." - ); - - if prompt_user("Would you like to install the git hook?: [y/N]")? != Some(PromptResult::Yes) { - println!("Ok, skipping installation!"); - return Ok(()); - } - let src = config.src.join("src").join("etc").join("pre-push.sh"); - match fs::hard_link(src, &dst) { - Err(e) => { - eprintln!( - "error: could not create hook {}: do you already have the git hook installed?\n{}", - dst.display(), - e - ); - return Err(e); - } - Ok(_) => println!("Linked `src/etc/pre-push.sh` to `.git/hooks/pre-push`"), - }; - Ok(()) -} - -/// Sets up or displays `src/etc/rust_analyzer_settings.json` -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -pub struct Vscode; - -impl Step for Vscode { - type Output = (); - const DEFAULT: bool = true; - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("vscode") - } - fn make_run(run: RunConfig<'_>) { - if run.builder.config.dry_run() { - return; - } - if let [cmd] = &run.paths[..] { - if cmd.assert_single_path().path.as_path().as_os_str() == "vscode" { - run.builder.ensure(Vscode); - } - } - } - fn run(self, builder: &Builder<'_>) -> Self::Output { - let config = &builder.config; - if config.dry_run() { - return; - } - t!(create_vscode_settings_maybe(&config)); - } -} - -/// Create a `.vscode/settings.json` file for rustc development, or just print it -fn create_vscode_settings_maybe(config: &Config) -> io::Result<()> { - let (current_hash, historical_hashes) = SETTINGS_HASHES.split_last().unwrap(); - let vscode_settings = config.src.join(".vscode").join("settings.json"); - // If None, no settings.json exists - // If Some(true), is a previous version of settings.json - // If Some(false), is not a previous version (i.e. user modified) - // If it's up to date we can just skip this - let mut mismatched_settings = None; - if let Ok(current) = fs::read_to_string(&vscode_settings) { - let mut hasher = sha2::Sha256::new(); - hasher.update(¤t); - let hash = hex::encode(hasher.finalize().as_slice()); - if hash == *current_hash { - return Ok(()); - } else if historical_hashes.contains(&hash.as_str()) { - mismatched_settings = Some(true); - } else { - mismatched_settings = Some(false); - } - } - println!( - "\nx.py can automatically install the recommended `.vscode/settings.json` file for rustc development" - ); - match mismatched_settings { - Some(true) => eprintln!( - "warning: existing `.vscode/settings.json` is out of date, x.py will update it" - ), - Some(false) => eprintln!( - "warning: existing `.vscode/settings.json` has been modified by user, x.py will back it up and replace it" - ), - _ => (), - } - let should_create = match prompt_user( - "Would you like to create/update `settings.json`, or only print suggested settings?: [y/p/N]", - )? { - Some(PromptResult::Yes) => true, - Some(PromptResult::Print) => false, - _ => { - println!("Ok, skipping settings!"); - return Ok(()); - } - }; - if should_create { - let path = config.src.join(".vscode"); - if !path.exists() { - fs::create_dir(&path)?; - } - let verb = match mismatched_settings { - // exists but outdated, we can replace this - Some(true) => "Updated", - // exists but user modified, back it up - Some(false) => { - // exists and is not current version or outdated, so back it up - let mut backup = vscode_settings.clone(); - backup.set_extension("json.bak"); - eprintln!("warning: copying `settings.json` to `settings.json.bak`"); - fs::copy(&vscode_settings, &backup)?; - "Updated" - } - _ => "Created", - }; - fs::write(&vscode_settings, &RUST_ANALYZER_SETTINGS)?; - println!("{verb} `.vscode/settings.json`"); - } else { - println!("\n{RUST_ANALYZER_SETTINGS}"); - } - Ok(()) -} diff --git a/src/bootstrap/setup/tests.rs b/src/bootstrap/setup/tests.rs deleted file mode 100644 index 0fe6e4a46..000000000 --- a/src/bootstrap/setup/tests.rs +++ /dev/null @@ -1,14 +0,0 @@ -use super::{RUST_ANALYZER_SETTINGS, SETTINGS_HASHES}; -use sha2::Digest; - -#[test] -fn check_matching_settings_hash() { - let mut hasher = sha2::Sha256::new(); - hasher.update(&RUST_ANALYZER_SETTINGS); - let hash = hex::encode(hasher.finalize().as_slice()); - assert_eq!( - &hash, - SETTINGS_HASHES.last().unwrap(), - "Update `SETTINGS_HASHES` with the new hash of `src/etc/rust_analyzer_settings.json`" - ); -} diff --git a/src/bootstrap/src/bin/main.rs b/src/bootstrap/src/bin/main.rs new file mode 100644 index 000000000..0a6072ae1 --- /dev/null +++ b/src/bootstrap/src/bin/main.rs @@ -0,0 +1,143 @@ +//! rustbuild, the Rust build system +//! +//! This is the entry point for the build system used to compile the `rustc` +//! compiler. Lots of documentation can be found in the `README.md` file in the +//! parent directory, and otherwise documentation can be found throughout the `build` +//! directory in each respective module. + +#[cfg(all(any(unix, windows), not(target_os = "solaris")))] +use std::io::Write; +#[cfg(all(any(unix, windows), not(target_os = "solaris")))] +use std::process; +use std::{env, fs}; + +#[cfg(all(any(unix, windows), not(target_os = "solaris")))] +use bootstrap::t; +use bootstrap::{find_recent_config_change_ids, Build, Config, Subcommand, CONFIG_CHANGE_HISTORY}; + +fn main() { + let args = env::args().skip(1).collect::>(); + let config = Config::parse(&args); + + #[cfg(all(any(unix, windows), not(target_os = "solaris")))] + let mut build_lock; + #[cfg(all(any(unix, windows), not(target_os = "solaris")))] + let _build_lock_guard; + #[cfg(all(any(unix, windows), not(target_os = "solaris")))] + // Display PID of process holding the lock + // PID will be stored in a lock file + { + let path = config.out.join("lock"); + let pid = match fs::read_to_string(&path) { + Ok(contents) => contents, + Err(_) => String::new(), + }; + + build_lock = + fd_lock::RwLock::new(t!(fs::OpenOptions::new().write(true).create(true).open(&path))); + _build_lock_guard = match build_lock.try_write() { + Ok(mut lock) => { + t!(lock.write(&process::id().to_string().as_ref())); + lock + } + err => { + drop(err); + println!("WARNING: build directory locked by process {pid}, waiting for lock"); + let mut lock = t!(build_lock.write()); + t!(lock.write(&process::id().to_string().as_ref())); + lock + } + }; + } + + #[cfg(any(not(any(unix, windows)), target_os = "solaris"))] + println!("WARNING: file locking not supported for target, not locking build directory"); + + // check_version warnings are not printed during setup + let changelog_suggestion = + if matches!(config.cmd, Subcommand::Setup { .. }) { None } else { check_version(&config) }; + + // NOTE: Since `./configure` generates a `config.toml`, distro maintainers will see the + // changelog warning, not the `x.py setup` message. + let suggest_setup = config.config.is_none() && !matches!(config.cmd, Subcommand::Setup { .. }); + if suggest_setup { + println!("WARNING: you have not made a `config.toml`"); + println!( + "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ + `cp config.example.toml config.toml`" + ); + } else if let Some(suggestion) = &changelog_suggestion { + println!("{suggestion}"); + } + + let pre_commit = config.src.join(".git").join("hooks").join("pre-commit"); + Build::new(config).build(); + + if suggest_setup { + println!("WARNING: you have not made a `config.toml`"); + println!( + "HELP: consider running `./x.py setup` or copying `config.example.toml` by running \ + `cp config.example.toml config.toml`" + ); + } else if let Some(suggestion) = &changelog_suggestion { + println!("{suggestion}"); + } + + // Give a warning if the pre-commit script is in pre-commit and not pre-push. + // HACK: Since the commit script uses hard links, we can't actually tell if it was installed by x.py setup or not. + // We could see if it's identical to src/etc/pre-push.sh, but pre-push may have been modified in the meantime. + // Instead, look for this comment, which is almost certainly not in any custom hook. + if fs::read_to_string(pre_commit).map_or(false, |contents| { + contents.contains("https://github.com/rust-lang/rust/issues/77620#issuecomment-705144570") + }) { + println!( + "WARNING: You have the pre-push script installed to .git/hooks/pre-commit. \ + Consider moving it to .git/hooks/pre-push instead, which runs less often." + ); + } + + if suggest_setup || changelog_suggestion.is_some() { + println!("NOTE: this message was printed twice to make it more likely to be seen"); + } +} + +fn check_version(config: &Config) -> Option { + let mut msg = String::new(); + + if config.changelog_seen.is_some() { + msg.push_str("WARNING: The use of `changelog-seen` is deprecated. Please refer to `change-id` option in `config.example.toml` instead.\n"); + } + + let latest_config_id = CONFIG_CHANGE_HISTORY.last().unwrap(); + if let Some(id) = config.change_id { + if &id == latest_config_id { + return None; + } + + let change_links: Vec = find_recent_config_change_ids(id) + .iter() + .map(|id| format!("https://github.com/rust-lang/rust/pull/{id}")) + .collect(); + if !change_links.is_empty() { + msg.push_str("WARNING: there have been changes to x.py since you last updated.\n"); + msg.push_str("To see more detail about these changes, visit the following PRs:\n"); + + for link in change_links { + msg.push_str(&format!(" - {link}\n")); + } + + msg.push_str("WARNING: there have been changes to x.py since you last updated.\n"); + + msg.push_str("NOTE: to silence this warning, "); + msg.push_str(&format!( + "update `config.toml` to use `change-id = {latest_config_id}` instead" + )); + } + } else { + msg.push_str("WARNING: The `change-id` is missing in the `config.toml`. This means that you will not be able to track the major changes made to the bootstrap configurations.\n"); + msg.push_str("NOTE: to silence this warning, "); + msg.push_str(&format!("add `change-id = {latest_config_id}` at the top of `config.toml`")); + }; + + Some(msg) +} diff --git a/src/bootstrap/src/bin/rustc.rs b/src/bootstrap/src/bin/rustc.rs new file mode 100644 index 000000000..070a2da6a --- /dev/null +++ b/src/bootstrap/src/bin/rustc.rs @@ -0,0 +1,406 @@ +//! Shim which is passed to Cargo as "rustc" when running the bootstrap. +//! +//! This shim will take care of some various tasks that our build process +//! requires that Cargo can't quite do through normal configuration: +//! +//! 1. When compiling build scripts and build dependencies, we need a guaranteed +//! full standard library available. The only compiler which actually has +//! this is the snapshot, so we detect this situation and always compile with +//! the snapshot compiler. +//! 2. We pass a bunch of `--cfg` and other flags based on what we're compiling +//! (and this slightly differs based on a whether we're using a snapshot or +//! not), so we do that all here. +//! +//! This may one day be replaced by RUSTFLAGS, but the dynamic nature of +//! switching compilers for the bootstrap and for build scripts will probably +//! never get replaced. + +use std::env; +use std::path::PathBuf; +use std::process::{Child, Command}; +use std::time::Instant; + +use dylib_util::{dylib_path, dylib_path_var}; + +#[path = "../utils/bin_helpers.rs"] +mod bin_helpers; + +#[path = "../utils/dylib.rs"] +mod dylib_util; + +fn main() { + let args = env::args_os().skip(1).collect::>(); + let arg = |name| args.windows(2).find(|args| args[0] == name).and_then(|args| args[1].to_str()); + + // We don't use the stage in this shim, but let's parse it to make sure that we're invoked + // by bootstrap, or that we provide a helpful error message if not. + bin_helpers::parse_rustc_stage(); + let verbose = bin_helpers::parse_rustc_verbose(); + + // Detect whether or not we're a build script depending on whether --target + // is passed (a bit janky...) + let target = arg("--target"); + let version = args.iter().find(|w| &**w == "-vV"); + + // Use a different compiler for build scripts, since there may not yet be a + // libstd for the real compiler to use. However, if Cargo is attempting to + // determine the version of the compiler, the real compiler needs to be + // used. Currently, these two states are differentiated based on whether + // --target and -vV is/isn't passed. + let (rustc, libdir) = if target.is_none() && version.is_none() { + ("RUSTC_SNAPSHOT", "RUSTC_SNAPSHOT_LIBDIR") + } else { + ("RUSTC_REAL", "RUSTC_LIBDIR") + }; + + let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); + let on_fail = env::var_os("RUSTC_ON_FAIL").map(Command::new); + + let rustc = env::var_os(rustc).unwrap_or_else(|| panic!("{:?} was not set", rustc)); + let libdir = env::var_os(libdir).unwrap_or_else(|| panic!("{:?} was not set", libdir)); + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&libdir)); + + let mut cmd = Command::new(rustc); + cmd.args(&args).env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + // Get the name of the crate we're compiling, if any. + let crate_name = arg("--crate-name"); + + if let Some(crate_name) = crate_name { + if let Some(target) = env::var_os("RUSTC_TIME") { + if target == "all" + || target.into_string().unwrap().split(',').any(|c| c.trim() == crate_name) + { + cmd.arg("-Ztime-passes"); + } + } + } + + // Print backtrace in case of ICE + if env::var("RUSTC_BACKTRACE_ON_ICE").is_ok() && env::var("RUST_BACKTRACE").is_err() { + cmd.env("RUST_BACKTRACE", "1"); + } + + if let Ok(lint_flags) = env::var("RUSTC_LINT_FLAGS") { + cmd.args(lint_flags.split_whitespace()); + } + + if target.is_some() { + // The stage0 compiler has a special sysroot distinct from what we + // actually downloaded, so we just always pass the `--sysroot` option, + // unless one is already set. + if !args.iter().any(|arg| arg == "--sysroot") { + cmd.arg("--sysroot").arg(&sysroot); + } + + // If we're compiling specifically the `panic_abort` crate then we pass + // the `-C panic=abort` option. Note that we do not do this for any + // other crate intentionally as this is the only crate for now that we + // ship with panic=abort. + // + // This... is a bit of a hack how we detect this. Ideally this + // information should be encoded in the crate I guess? Would likely + // require an RFC amendment to RFC 1513, however. + if crate_name == Some("panic_abort") { + cmd.arg("-C").arg("panic=abort"); + } + + // `-Ztls-model=initial-exec` must not be applied to proc-macros, see + // issue https://github.com/rust-lang/rust/issues/100530 + if env::var("RUSTC_TLS_MODEL_INITIAL_EXEC").is_ok() + && arg("--crate-type") != Some("proc-macro") + && !matches!(crate_name, Some("proc_macro2" | "quote" | "syn" | "synstructure")) + { + cmd.arg("-Ztls-model=initial-exec"); + } + } else { + // Find any host flags that were passed by bootstrap. + // The flags are stored in a RUSTC_HOST_FLAGS variable, separated by spaces. + if let Ok(flags) = std::env::var("RUSTC_HOST_FLAGS") { + for flag in flags.split(' ') { + cmd.arg(flag); + } + } + } + + if let Ok(map) = env::var("RUSTC_DEBUGINFO_MAP") { + cmd.arg("--remap-path-prefix").arg(&map); + } + // The remap flags for Cargo registry sources need to be passed after the remapping for the + // Rust source code directory, to handle cases when $CARGO_HOME is inside the source directory. + if let Ok(maps) = env::var("RUSTC_CARGO_REGISTRY_SRC_TO_REMAP") { + for map in maps.split('\t') { + cmd.arg("--remap-path-prefix").arg(map); + } + } + + // Force all crates compiled by this compiler to (a) be unstable and (b) + // allow the `rustc_private` feature to link to other unstable crates + // also in the sysroot. We also do this for host crates, since those + // may be proc macros, in which case we might ship them. + if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { + cmd.arg("-Z").arg("force-unstable-if-unmarked"); + } + + // allow-features is handled from within this rustc wrapper because of + // issues with build scripts. Some packages use build scripts to + // dynamically detect if certain nightly features are available. + // There are different ways this causes problems: + // + // * rustix runs `rustc` on a small test program to see if the feature is + // available (and sets a `cfg` if it is). It does not honor + // CARGO_ENCODED_RUSTFLAGS. + // * proc-macro2 detects if `rustc -vV` says "nighty" or "dev" and enables + // nightly features. It will scan CARGO_ENCODED_RUSTFLAGS for + // -Zallow-features. Unfortunately CARGO_ENCODED_RUSTFLAGS is not set + // for build-dependencies when --target is used. + // + // The issues above means we can't just use RUSTFLAGS, and we can't use + // `cargo -Zallow-features=…`. Passing it through here ensures that it + // always gets set. Unfortunately that also means we need to enable more + // features than we really want (like those for proc-macro2), but there + // isn't much of a way around it. + // + // I think it is unfortunate that build scripts are doing this at all, + // since changes to nightly features can cause crates to break even if the + // user didn't want or care about the use of the nightly features. I think + // nightly features should be opt-in only. Unfortunately the dynamic + // checks are now too wide spread that we just need to deal with it. + // + // If you want to try to remove this, I suggest working with the crate + // authors to remove the dynamic checking. Another option is to pursue + // https://github.com/rust-lang/cargo/issues/11244 and + // https://github.com/rust-lang/cargo/issues/4423, which will likely be + // very difficult, but could help expose -Zallow-features into build + // scripts so they could try to honor them. + if let Ok(allow_features) = env::var("RUSTC_ALLOW_FEATURES") { + cmd.arg(format!("-Zallow-features={allow_features}")); + } + + if let Ok(flags) = env::var("MAGIC_EXTRA_RUSTFLAGS") { + for flag in flags.split(' ') { + cmd.arg(flag); + } + } + + let is_test = args.iter().any(|a| a == "--test"); + if verbose > 2 { + let rust_env_vars = + env::vars().filter(|(k, _)| k.starts_with("RUST") || k.starts_with("CARGO")); + let prefix = if is_test { "[RUSTC-SHIM] rustc --test" } else { "[RUSTC-SHIM] rustc" }; + let prefix = match crate_name { + Some(crate_name) => format!("{prefix} {crate_name}"), + None => prefix.to_string(), + }; + for (i, (k, v)) in rust_env_vars.enumerate() { + eprintln!("{prefix} env[{i}]: {k:?}={v:?}"); + } + eprintln!("{} working directory: {}", prefix, env::current_dir().unwrap().display()); + eprintln!( + "{} command: {:?}={:?} {:?}", + prefix, + dylib_path_var(), + env::join_paths(&dylib_path).unwrap(), + cmd, + ); + eprintln!("{prefix} sysroot: {sysroot:?}"); + eprintln!("{prefix} libdir: {libdir:?}"); + } + + if env::var_os("RUSTC_BOLT_LINK_FLAGS").is_some() { + if let Some("rustc_driver") = crate_name { + cmd.arg("-Clink-args=-Wl,-q"); + } + } + + let start = Instant::now(); + let (child, status) = { + let errmsg = format!("\nFailed to run:\n{cmd:?}\n-------------"); + let mut child = cmd.spawn().expect(&errmsg); + let status = child.wait().expect(&errmsg); + (child, status) + }; + + if env::var_os("RUSTC_PRINT_STEP_TIMINGS").is_some() + || env::var_os("RUSTC_PRINT_STEP_RUSAGE").is_some() + { + if let Some(crate_name) = crate_name { + let dur = start.elapsed(); + // If the user requested resource usage data, then + // include that in addition to the timing output. + let rusage_data = + env::var_os("RUSTC_PRINT_STEP_RUSAGE").and_then(|_| format_rusage_data(child)); + eprintln!( + "[RUSTC-TIMING] {} test:{} {}.{:03}{}{}", + crate_name, + is_test, + dur.as_secs(), + dur.subsec_millis(), + if rusage_data.is_some() { " " } else { "" }, + rusage_data.unwrap_or(String::new()), + ); + } + } + + if status.success() { + std::process::exit(0); + // NOTE: everything below here is unreachable. do not put code that + // should run on success, after this block. + } + if verbose > 0 { + println!("\nDid not run successfully: {status}\n{cmd:?}\n-------------"); + } + + if let Some(mut on_fail) = on_fail { + on_fail.status().expect("Could not run the on_fail command"); + } + + // Preserve the exit code. In case of signal, exit with 0xfe since it's + // awkward to preserve this status in a cross-platform way. + match status.code() { + Some(i) => std::process::exit(i), + None => { + eprintln!("rustc exited with {status}"); + std::process::exit(0xfe); + } + } +} + +#[cfg(all(not(unix), not(windows)))] +// In the future we can add this for more platforms +fn format_rusage_data(_child: Child) -> Option { + None +} + +#[cfg(windows)] +fn format_rusage_data(child: Child) -> Option { + use std::os::windows::io::AsRawHandle; + + use windows::{ + Win32::Foundation::HANDLE, + Win32::System::ProcessStatus::{ + K32GetProcessMemoryInfo, PROCESS_MEMORY_COUNTERS, PROCESS_MEMORY_COUNTERS_EX, + }, + Win32::System::Threading::GetProcessTimes, + Win32::System::Time::FileTimeToSystemTime, + }; + + let handle = HANDLE(child.as_raw_handle() as isize); + + let mut user_filetime = Default::default(); + let mut user_time = Default::default(); + let mut kernel_filetime = Default::default(); + let mut kernel_time = Default::default(); + let mut memory_counters = PROCESS_MEMORY_COUNTERS::default(); + + unsafe { + GetProcessTimes( + handle, + &mut Default::default(), + &mut Default::default(), + &mut kernel_filetime, + &mut user_filetime, + ) + } + .ok()?; + unsafe { FileTimeToSystemTime(&user_filetime, &mut user_time) }.ok()?; + unsafe { FileTimeToSystemTime(&kernel_filetime, &mut kernel_time) }.ok()?; + + // Unlike on Linux with RUSAGE_CHILDREN, this will only return memory information for the process + // with the given handle and none of that process's children. + unsafe { + K32GetProcessMemoryInfo( + handle, + &mut memory_counters, + std::mem::size_of::() as u32, + ) + } + .ok() + .ok()?; + + // Guide on interpreting these numbers: + // https://docs.microsoft.com/en-us/windows/win32/psapi/process-memory-usage-information + let peak_working_set = memory_counters.PeakWorkingSetSize / 1024; + let peak_page_file = memory_counters.PeakPagefileUsage / 1024; + let peak_paged_pool = memory_counters.QuotaPeakPagedPoolUsage / 1024; + let peak_nonpaged_pool = memory_counters.QuotaPeakNonPagedPoolUsage / 1024; + Some(format!( + "user: {USER_SEC}.{USER_USEC:03} \ + sys: {SYS_SEC}.{SYS_USEC:03} \ + peak working set (kb): {PEAK_WORKING_SET} \ + peak page file usage (kb): {PEAK_PAGE_FILE} \ + peak paged pool usage (kb): {PEAK_PAGED_POOL} \ + peak non-paged pool usage (kb): {PEAK_NONPAGED_POOL} \ + page faults: {PAGE_FAULTS}", + USER_SEC = user_time.wSecond + (user_time.wMinute * 60), + USER_USEC = user_time.wMilliseconds, + SYS_SEC = kernel_time.wSecond + (kernel_time.wMinute * 60), + SYS_USEC = kernel_time.wMilliseconds, + PEAK_WORKING_SET = peak_working_set, + PEAK_PAGE_FILE = peak_page_file, + PEAK_PAGED_POOL = peak_paged_pool, + PEAK_NONPAGED_POOL = peak_nonpaged_pool, + PAGE_FAULTS = memory_counters.PageFaultCount, + )) +} + +#[cfg(unix)] +/// Tries to build a string with human readable data for several of the rusage +/// fields. Note that we are focusing mainly on data that we believe to be +/// supplied on Linux (the `rusage` struct has other fields in it but they are +/// currently unsupported by Linux). +fn format_rusage_data(_child: Child) -> Option { + let rusage: libc::rusage = unsafe { + let mut recv = std::mem::zeroed(); + // -1 is RUSAGE_CHILDREN, which means to get the rusage for all children + // (and grandchildren, etc) processes that have respectively terminated + // and been waited for. + let retval = libc::getrusage(-1, &mut recv); + if retval != 0 { + return None; + } + recv + }; + // Mac OS X reports the maxrss in bytes, not kb. + let divisor = if env::consts::OS == "macos" { 1024 } else { 1 }; + let maxrss = (rusage.ru_maxrss + (divisor - 1)) / divisor; + + let mut init_str = format!( + "user: {USER_SEC}.{USER_USEC:03} \ + sys: {SYS_SEC}.{SYS_USEC:03} \ + max rss (kb): {MAXRSS}", + USER_SEC = rusage.ru_utime.tv_sec, + USER_USEC = rusage.ru_utime.tv_usec, + SYS_SEC = rusage.ru_stime.tv_sec, + SYS_USEC = rusage.ru_stime.tv_usec, + MAXRSS = maxrss + ); + + // The remaining rusage stats vary in platform support. So we treat + // uniformly zero values in each category as "not worth printing", since it + // either means no events of that type occurred, or that the platform + // does not support it. + + let minflt = rusage.ru_minflt; + let majflt = rusage.ru_majflt; + if minflt != 0 || majflt != 0 { + init_str.push_str(&format!(" page reclaims: {minflt} page faults: {majflt}")); + } + + let inblock = rusage.ru_inblock; + let oublock = rusage.ru_oublock; + if inblock != 0 || oublock != 0 { + init_str.push_str(&format!(" fs block inputs: {inblock} fs block outputs: {oublock}")); + } + + let nvcsw = rusage.ru_nvcsw; + let nivcsw = rusage.ru_nivcsw; + if nvcsw != 0 || nivcsw != 0 { + init_str.push_str(&format!( + " voluntary ctxt switches: {nvcsw} involuntary ctxt switches: {nivcsw}" + )); + } + + return Some(init_str); +} diff --git a/src/bootstrap/src/bin/rustdoc.rs b/src/bootstrap/src/bin/rustdoc.rs new file mode 100644 index 000000000..dbbce6fe2 --- /dev/null +++ b/src/bootstrap/src/bin/rustdoc.rs @@ -0,0 +1,92 @@ +//! Shim which is passed to Cargo as "rustdoc" when running the bootstrap. +//! +//! See comments in `src/bootstrap/rustc.rs` for more information. + +use std::env; +use std::ffi::OsString; +use std::path::PathBuf; +use std::process::Command; + +use dylib_util::{dylib_path, dylib_path_var}; + +#[path = "../utils/bin_helpers.rs"] +mod bin_helpers; + +#[path = "../utils/dylib.rs"] +mod dylib_util; + +fn main() { + let args = env::args_os().skip(1).collect::>(); + + let stage = bin_helpers::parse_rustc_stage(); + let verbose = bin_helpers::parse_rustc_verbose(); + + let rustdoc = env::var_os("RUSTDOC_REAL").expect("RUSTDOC_REAL was not set"); + let libdir = env::var_os("RUSTDOC_LIBDIR").expect("RUSTDOC_LIBDIR was not set"); + let sysroot = env::var_os("RUSTC_SYSROOT").expect("RUSTC_SYSROOT was not set"); + + // Detect whether or not we're a build script depending on whether --target + // is passed (a bit janky...) + let target = args.windows(2).find(|w| &*w[0] == "--target").and_then(|w| w[1].to_str()); + + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(libdir.clone())); + + let mut cmd = Command::new(rustdoc); + + if target.is_some() { + // The stage0 compiler has a special sysroot distinct from what we + // actually downloaded, so we just always pass the `--sysroot` option, + // unless one is already set. + if !args.iter().any(|arg| arg == "--sysroot") { + cmd.arg("--sysroot").arg(&sysroot); + } + } + + cmd.args(&args); + cmd.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + // Force all crates compiled by this compiler to (a) be unstable and (b) + // allow the `rustc_private` feature to link to other unstable crates + // also in the sysroot. + if env::var_os("RUSTC_FORCE_UNSTABLE").is_some() { + cmd.arg("-Z").arg("force-unstable-if-unmarked"); + } + if let Some(linker) = env::var_os("RUSTDOC_LINKER") { + let mut arg = OsString::from("-Clinker="); + arg.push(&linker); + cmd.arg(arg); + } + if let Ok(no_threads) = env::var("RUSTDOC_LLD_NO_THREADS") { + cmd.arg("-Clink-arg=-fuse-ld=lld"); + cmd.arg(format!("-Clink-arg=-Wl,{no_threads}")); + } + // Cargo doesn't pass RUSTDOCFLAGS to proc_macros: + // https://github.com/rust-lang/cargo/issues/4423 + // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. + // We also declare that the flag is expected, which we need to do to not + // get warnings about it being unexpected. + if stage == "0" { + cmd.arg("--cfg=bootstrap"); + } + cmd.arg("-Zunstable-options"); + // #[cfg(bootstrap)] + cmd.arg("--check-cfg=values(bootstrap)"); + // cmd.arg("--check-cfg=cfg(bootstrap)"); + + if verbose > 1 { + eprintln!( + "rustdoc command: {:?}={:?} {:?}", + dylib_path_var(), + env::join_paths(&dylib_path).unwrap(), + cmd, + ); + eprintln!("sysroot: {sysroot:?}"); + eprintln!("libdir: {libdir:?}"); + } + + std::process::exit(match cmd.status() { + Ok(s) => s.code().unwrap_or(1), + Err(e) => panic!("\n\nfailed to run {cmd:?}: {e}\n\n"), + }) +} diff --git a/src/bootstrap/src/bin/sccache-plus-cl.rs b/src/bootstrap/src/bin/sccache-plus-cl.rs new file mode 100644 index 000000000..554c2dd4d --- /dev/null +++ b/src/bootstrap/src/bin/sccache-plus-cl.rs @@ -0,0 +1,38 @@ +use std::env; +use std::process::{self, Command}; + +fn main() { + let target = env::var("SCCACHE_TARGET").unwrap(); + // Locate the actual compiler that we're invoking + env::set_var("CC", env::var_os("SCCACHE_CC").unwrap()); + env::set_var("CXX", env::var_os("SCCACHE_CXX").unwrap()); + let mut cfg = cc::Build::new(); + cfg.cargo_metadata(false) + .out_dir("/") + .target(&target) + .host(&target) + .opt_level(0) + .warnings(false) + .debug(false); + let compiler = cfg.get_compiler(); + + // Invoke sccache with said compiler + let sccache_path = env::var_os("SCCACHE_PATH").unwrap(); + let mut cmd = Command::new(&sccache_path); + cmd.arg(compiler.path()); + for &(ref k, ref v) in compiler.env() { + cmd.env(k, v); + } + for arg in env::args().skip(1) { + cmd.arg(arg); + } + + if let Ok(s) = env::var("SCCACHE_EXTRA_ARGS") { + for s in s.split_whitespace() { + cmd.arg(s); + } + } + + let status = cmd.status().expect("failed to spawn"); + process::exit(status.code().unwrap_or(2)) +} diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs new file mode 100644 index 000000000..121925b56 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -0,0 +1,544 @@ +//! Implementation of compiling the compiler and standard library, in "check"-based modes. + +use crate::core::build_steps::compile::{ + add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, +}; +use crate::core::build_steps::tool::{prepare_tool_cargo, SourceType}; +use crate::core::builder::{crate_description, Alias, Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::cache::Interned; +use crate::INTERNER; +use crate::{Compiler, Mode, Subcommand}; +use std::path::{Path, PathBuf}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Std { + pub target: TargetSelection, + /// Whether to build only a subset of crates. + /// + /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. + /// + /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc + crates: Interned>, +} + +/// Returns args for the subcommand itself (not for cargo) +fn args(builder: &Builder<'_>) -> Vec { + fn strings<'a>(arr: &'a [&str]) -> impl Iterator + 'a { + arr.iter().copied().map(String::from) + } + + if let Subcommand::Clippy { fix, allow, deny, warn, forbid, .. } = &builder.config.cmd { + // disable the most spammy clippy lints + let ignored_lints = vec![ + "many_single_char_names", // there are a lot in stdarch + "collapsible_if", + "type_complexity", + "missing_safety_doc", // almost 3K warnings + "too_many_arguments", + "needless_lifetimes", // people want to keep the lifetimes + "wrong_self_convention", + ]; + let mut args = vec![]; + if *fix { + #[rustfmt::skip] + args.extend(strings(&[ + "--fix", "-Zunstable-options", + // FIXME: currently, `--fix` gives an error while checking tests for libtest, + // possibly because libtest is not yet built in the sysroot. + // As a workaround, avoid checking tests and benches when passed --fix. + "--lib", "--bins", "--examples", + ])); + } + args.extend(strings(&["--", "--cap-lints", "warn"])); + args.extend(ignored_lints.iter().map(|lint| format!("-Aclippy::{}", lint))); + let mut clippy_lint_levels: Vec = Vec::new(); + allow.iter().for_each(|v| clippy_lint_levels.push(format!("-A{}", v))); + deny.iter().for_each(|v| clippy_lint_levels.push(format!("-D{}", v))); + warn.iter().for_each(|v| clippy_lint_levels.push(format!("-W{}", v))); + forbid.iter().for_each(|v| clippy_lint_levels.push(format!("-F{}", v))); + args.extend(clippy_lint_levels); + args.extend(builder.config.free_args.clone()); + args + } else { + builder.config.free_args.clone() + } +} + +fn cargo_subcommand(kind: Kind) -> &'static str { + match kind { + Kind::Check => "check", + Kind::Clippy => "clippy", + Kind::Fix => "fix", + _ => unreachable!(), + } +} + +impl Std { + pub fn new(target: TargetSelection) -> Self { + Self { target, crates: INTERNER.intern_list(vec![]) } + } +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("sysroot").path("library") + } + + fn make_run(run: RunConfig<'_>) { + let crates = run.make_run_crates(Alias::Library); + run.builder.ensure(Std { target: run.target, crates }); + } + + fn run(self, builder: &Builder<'_>) { + builder.update_submodule(&Path::new("library").join("stdarch")); + + let target = self.target; + let compiler = builder.compiler(builder.top_stage, builder.config.build); + + let mut cargo = builder.cargo( + compiler, + Mode::Std, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + std_cargo(builder, target, compiler.stage, &mut cargo); + if matches!(builder.config.cmd, Subcommand::Fix { .. }) { + // By default, cargo tries to fix all targets. Tell it not to fix tests until we've added `test` to the sysroot. + cargo.arg("--lib"); + } + + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + let _guard = builder.msg_check( + format_args!("library artifacts{}", crate_description(&self.crates)), + target, + ); + run_cargo( + builder, + cargo, + args(builder), + &libstd_stamp(builder, compiler, target), + vec![], + true, + false, + ); + + // We skip populating the sysroot in non-zero stage because that'll lead + // to rlib/rmeta conflicts if std gets built during this session. + if compiler.stage == 0 { + let libdir = builder.sysroot_libdir(compiler, target); + let hostdir = builder.sysroot_libdir(compiler, compiler.host); + add_to_sysroot(&builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + } + drop(_guard); + + // don't run on std twice with x.py clippy + // don't check test dependencies if we haven't built libtest + if builder.kind == Kind::Clippy || !self.crates.iter().any(|krate| krate == "test") { + return; + } + + // Then run cargo again, once we've put the rmeta files for the library + // crates into the sysroot. This is needed because e.g., core's tests + // depend on `libtest` -- Cargo presumes it will exist, but it doesn't + // since we initialize with an empty sysroot. + // + // Currently only the "libtest" tree of crates does this. + let mut cargo = builder.cargo( + compiler, + Mode::Std, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + + // If we're not in stage 0, tests and examples will fail to compile + // from `core` definitions being loaded from two different `libcore` + // .rmeta and .rlib files. + if compiler.stage == 0 { + cargo.arg("--all-targets"); + } + + std_cargo(builder, target, compiler.stage, &mut cargo); + + // Explicitly pass -p for all dependencies krates -- this will force cargo + // to also check the tests/benches/examples for these crates, rather + // than just the leaf crate. + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + let _guard = builder.msg_check("library test/bench/example targets", target); + run_cargo( + builder, + cargo, + args(builder), + &libstd_test_stamp(builder, compiler, target), + vec![], + true, + false, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rustc { + pub target: TargetSelection, + /// Whether to build only a subset of crates. + /// + /// This shouldn't be used from other steps; see the comment on [`compile::Rustc`]. + /// + /// [`compile::Rustc`]: crate::core::build_steps::compile::Rustc + crates: Interned>, +} + +impl Rustc { + pub fn new(target: TargetSelection, builder: &Builder<'_>) -> Self { + let crates = builder + .in_tree_crates("rustc-main", Some(target)) + .into_iter() + .map(|krate| krate.name.to_string()) + .collect(); + Self { target, crates: INTERNER.intern_list(crates) } + } +} + +impl Step for Rustc { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("rustc-main").path("compiler") + } + + fn make_run(run: RunConfig<'_>) { + let crates = run.make_run_crates(Alias::Compiler); + run.builder.ensure(Rustc { target: run.target, crates }); + } + + /// Builds the compiler. + /// + /// This will build the compiler for a particular stage of the build using + /// the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + + if compiler.stage != 0 { + // If we're not in stage 0, then we won't have a std from the beta + // compiler around. That means we need to make sure there's one in + // the sysroot for the compiler to find. Otherwise, we're going to + // fail when building crates that need to generate code (e.g., build + // scripts and their dependencies). + builder.ensure(crate::core::build_steps::compile::Std::new(compiler, compiler.host)); + builder.ensure(crate::core::build_steps::compile::Std::new(compiler, target)); + } else { + builder.ensure(Std::new(target)); + } + + let mut cargo = builder.cargo( + compiler, + Mode::Rustc, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + rustc_cargo(builder, &mut cargo, target, compiler.stage); + + // For ./x.py clippy, don't run with --all-targets because + // linting tests and benchmarks can produce very noisy results + if builder.kind != Kind::Clippy { + cargo.arg("--all-targets"); + } + + // Explicitly pass -p for all compiler crates -- this will force cargo + // to also check the tests/benches/examples for these crates, rather + // than just the leaf crate. + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + let _guard = builder.msg_check( + format_args!("compiler artifacts{}", crate_description(&self.crates)), + target, + ); + run_cargo( + builder, + cargo, + args(builder), + &librustc_stamp(builder, compiler, target), + vec![], + true, + false, + ); + + let libdir = builder.sysroot_libdir(compiler, target); + let hostdir = builder.sysroot_libdir(compiler, compiler.host); + add_to_sysroot(&builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target)); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CodegenBackend { + pub target: TargetSelection, + pub backend: Interned, +} + +impl Step for CodegenBackend { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + for &backend in &[INTERNER.intern_str("cranelift"), INTERNER.intern_str("gcc")] { + run.builder.ensure(CodegenBackend { target: run.target, backend }); + } + } + + fn run(self, builder: &Builder<'_>) { + // FIXME: remove once https://github.com/rust-lang/rust/issues/112393 is resolved + if builder.build.config.vendor && &self.backend == "gcc" { + println!("Skipping checking of `rustc_codegen_gcc` with vendoring enabled."); + return; + } + + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + let backend = self.backend; + + builder.ensure(Rustc::new(target, builder)); + + let mut cargo = builder.cargo( + compiler, + Mode::Codegen, + SourceType::InTree, + target, + cargo_subcommand(builder.kind), + ); + cargo + .arg("--manifest-path") + .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); + rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + let _guard = builder.msg_check(&backend, target); + + run_cargo( + builder, + cargo, + args(builder), + &codegen_backend_stamp(builder, compiler, target, backend), + vec![], + true, + false, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustAnalyzer { + pub target: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/rust-analyzer").default_condition( + builder + .config + .tools + .as_ref() + .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + + builder.ensure(Std::new(target)); + + let mut cargo = prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + target, + cargo_subcommand(builder.kind), + "src/tools/rust-analyzer", + SourceType::InTree, + &["rust-analyzer/in-rust-tree".to_owned()], + ); + + cargo.allow_features(crate::core::build_steps::tool::RustAnalyzer::ALLOW_FEATURES); + + // For ./x.py clippy, don't check those targets because + // linting tests and benchmarks can produce very noisy results + if builder.kind != Kind::Clippy { + // can't use `--all-targets` because `--examples` doesn't work well + cargo.arg("--bins"); + cargo.arg("--tests"); + cargo.arg("--benches"); + } + + let _guard = builder.msg_check("rust-analyzer artifacts", target); + run_cargo( + builder, + cargo, + args(builder), + &stamp(builder, compiler, target), + vec![], + true, + false, + ); + + /// Cargo's output path in a given stage, compiled by a particular + /// compiler for the specified target. + fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::ToolStd, target).join(".rust-analyzer-check.stamp") + } + } +} + +macro_rules! tool_check_step { + ($name:ident, $path:literal, $($alias:literal, )* $source_type:path $(, $default:literal )?) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub target: TargetSelection, + } + + impl Step for $name { + type Output = (); + const ONLY_HOSTS: bool = true; + // don't ever check out-of-tree tools by default, they'll fail when toolstate is broken + const DEFAULT: bool = matches!($source_type, SourceType::InTree) $( && $default )?; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&[ $path, $($alias),* ]) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(builder.top_stage, builder.config.build); + let target = self.target; + + builder.ensure(Rustc::new(target, builder)); + + let mut cargo = prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + cargo_subcommand(builder.kind), + $path, + $source_type, + &[], + ); + + // For ./x.py clippy, don't run with --all-targets because + // linting tests and benchmarks can produce very noisy results + if builder.kind != Kind::Clippy { + cargo.arg("--all-targets"); + } + + // Enable internal lints for clippy and rustdoc + // NOTE: this doesn't enable lints for any other tools unless they explicitly add `#![warn(rustc::internal)]` + // See https://github.com/rust-lang/rust/pull/80573#issuecomment-754010776 + cargo.rustflag("-Zunstable-options"); + let _guard = builder.msg_check(&concat!(stringify!($name), " artifacts").to_lowercase(), target); + run_cargo( + builder, + cargo, + args(builder), + &stamp(builder, compiler, target), + vec![], + true, + false, + ); + + /// Cargo's output path in a given stage, compiled by a particular + /// compiler for the specified target. + fn stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + ) -> PathBuf { + builder + .cargo_out(compiler, Mode::ToolRustc, target) + .join(format!(".{}-check.stamp", stringify!($name).to_lowercase())) + } + } + } + }; +} + +tool_check_step!(Rustdoc, "src/tools/rustdoc", "src/librustdoc", SourceType::InTree); +// Clippy, miri and Rustfmt are hybrids. They are external tools, but use a git subtree instead +// of a submodule. Since the SourceType only drives the deny-warnings +// behavior, treat it as in-tree so that any new warnings in clippy will be +// rejected. +tool_check_step!(Clippy, "src/tools/clippy", SourceType::InTree); +tool_check_step!(Miri, "src/tools/miri", SourceType::InTree); +tool_check_step!(CargoMiri, "src/tools/miri/cargo-miri", SourceType::InTree); +tool_check_step!(Rls, "src/tools/rls", SourceType::InTree); +tool_check_step!(Rustfmt, "src/tools/rustfmt", SourceType::InTree); +tool_check_step!(MiroptTestTools, "src/tools/miropt-test-tools", SourceType::InTree); + +tool_check_step!(Bootstrap, "src/bootstrap", SourceType::InTree, false); + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp") +} + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +fn libstd_test_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp") +} + +/// Cargo's output path for librustc in a given stage, compiled by a particular +/// compiler for the specified target. +fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp") +} + +/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular +/// compiler for the specified target and backend. +fn codegen_backend_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + backend: Interned, +) -> PathBuf { + builder + .cargo_out(compiler, Mode::Codegen, target) + .join(format!(".librustc_codegen_{backend}-check.stamp")) +} diff --git a/src/bootstrap/src/core/build_steps/clean.rs b/src/bootstrap/src/core/build_steps/clean.rs new file mode 100644 index 000000000..cbb6b5f46 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/clean.rs @@ -0,0 +1,242 @@ +//! Implementation of `make clean` in rustbuild. +//! +//! Responsible for cleaning out a build directory of all old and stale +//! artifacts to prepare for a fresh build. Currently doesn't remove the +//! `build/cache` directory (download cache) or the `build/$target/llvm` +//! directory unless the `--all` flag is present. + +use std::fs; +use std::io::{self, ErrorKind}; +use std::path::Path; + +use crate::core::builder::{crate_description, Builder, RunConfig, ShouldRun, Step}; +use crate::utils::cache::Interned; +use crate::utils::helpers::t; +use crate::{Build, Compiler, Mode, Subcommand}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct CleanAll {} + +impl Step for CleanAll { + const DEFAULT: bool = true; + type Output = (); + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CleanAll {}) + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let Subcommand::Clean { all, stage } = builder.config.cmd else { + unreachable!("wrong subcommand?") + }; + + if all && stage.is_some() { + panic!("--all and --stage can't be used at the same time for `x clean`"); + } + + clean(builder.build, all, stage) + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() // handled by DEFAULT + } +} + +macro_rules! clean_crate_tree { + ( $( $name:ident, $mode:path, $root_crate:literal);+ $(;)? ) => { $( + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub struct $name { + compiler: Compiler, + crates: Interned>, + } + + impl Step for $name { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let crates = run.builder.in_tree_crates($root_crate, None); + run.crates(crates) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let compiler = builder.compiler(builder.top_stage, run.target); + builder.ensure(Self { crates: run.cargo_crates_in_set(), compiler }); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let compiler = self.compiler; + let target = compiler.host; + let mut cargo = builder.bare_cargo(compiler, $mode, target, "clean"); + + // Since https://github.com/rust-lang/rust/pull/111076 enables + // unstable cargo feature (`public-dependency`), we need to ensure + // that unstable features are enabled before reading libstd Cargo.toml. + cargo.env("RUSTC_BOOTSTRAP", "1"); + + for krate in &*self.crates { + cargo.arg("-p"); + cargo.arg(krate); + } + + builder.info(&format!( + "Cleaning{} stage{} {} artifacts ({} -> {})", + crate_description(&self.crates), compiler.stage, stringify!($name).to_lowercase(), &compiler.host, target, + )); + + // NOTE: doesn't use `run_cargo` because we don't want to save a stamp file, + // and doesn't use `stream_cargo` to avoid passing `--message-format` which `clean` doesn't accept. + builder.run(&mut cargo); + } + } + )+ } +} + +clean_crate_tree! { + Rustc, Mode::Rustc, "rustc-main"; + Std, Mode::Std, "sysroot"; +} + +fn clean(build: &Build, all: bool, stage: Option) { + if build.config.dry_run() { + return; + } + + rm_rf("tmp".as_ref()); + + // Clean the entire build directory + if all { + rm_rf(&build.out); + return; + } + + // Clean the target stage artifacts + if let Some(stage) = stage { + clean_specific_stage(build, stage); + return; + } + + // Follow the default behaviour + clean_default(build); +} + +fn clean_specific_stage(build: &Build, stage: u32) { + for host in &build.hosts { + let entries = match build.out.join(host.triple).read_dir() { + Ok(iter) => iter, + Err(_) => continue, + }; + + for entry in entries { + let entry = t!(entry); + let stage_prefix = format!("stage{}", stage); + + // if current entry is not related with the target stage, continue + if !entry.file_name().to_str().unwrap_or("").contains(&stage_prefix) { + continue; + } + + let path = t!(entry.path().canonicalize()); + rm_rf(&path); + } + } +} + +fn clean_default(build: &Build) { + rm_rf(&build.out.join("tmp")); + rm_rf(&build.out.join("dist")); + rm_rf(&build.out.join("rustfmt.stamp")); + + for host in &build.hosts { + let entries = match build.out.join(host.triple).read_dir() { + Ok(iter) => iter, + Err(_) => continue, + }; + + for entry in entries { + let entry = t!(entry); + if entry.file_name().to_str() == Some("llvm") { + continue; + } + let path = t!(entry.path().canonicalize()); + rm_rf(&path); + } + } +} + +fn rm_rf(path: &Path) { + match path.symlink_metadata() { + Err(e) => { + if e.kind() == ErrorKind::NotFound { + return; + } + panic!("failed to get metadata for file {}: {}", path.display(), e); + } + Ok(metadata) => { + if metadata.file_type().is_file() || metadata.file_type().is_symlink() { + do_op(path, "remove file", |p| { + fs::remove_file(p).or_else(|e| { + // Work around the fact that we cannot + // delete an executable while it runs on Windows. + #[cfg(windows)] + if e.kind() == std::io::ErrorKind::PermissionDenied + && p.file_name().and_then(std::ffi::OsStr::to_str) + == Some("bootstrap.exe") + { + eprintln!("WARNING: failed to delete '{}'.", p.display()); + return Ok(()); + } + Err(e) + }) + }); + return; + } + + for file in t!(fs::read_dir(path)) { + rm_rf(&t!(file).path()); + } + do_op(path, "remove dir", |p| { + fs::remove_dir(p).or_else(|e| { + // Check for dir not empty on Windows + // FIXME: Once `ErrorKind::DirectoryNotEmpty` is stabilized, + // match on `e.kind()` instead. + #[cfg(windows)] + if e.raw_os_error() == Some(145) { + return Ok(()); + } + + Err(e) + }) + }); + } + }; +} + +fn do_op(path: &Path, desc: &str, mut f: F) +where + F: FnMut(&Path) -> io::Result<()>, +{ + match f(path) { + Ok(()) => {} + // On windows we can't remove a readonly file, and git will often clone files as readonly. + // As a result, we have some special logic to remove readonly files on windows. + // This is also the reason that we can't use things like fs::remove_dir_all(). + Err(ref e) if cfg!(windows) && e.kind() == ErrorKind::PermissionDenied => { + let m = t!(path.symlink_metadata()); + let mut p = m.permissions(); + p.set_readonly(false); + t!(fs::set_permissions(path, p)); + f(path).unwrap_or_else(|e| { + // Delete symlinked directories on Windows + #[cfg(windows)] + if m.file_type().is_symlink() && path.is_dir() && fs::remove_dir(path).is_ok() { + return; + } + panic!("failed to {} {}: {}", desc, path.display(), e); + }); + } + Err(e) => { + panic!("failed to {} {}: {}", desc, path.display(), e); + } + } +} diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs new file mode 100644 index 000000000..7021a9543 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -0,0 +1,2069 @@ +//! Implementation of compiling various phases of the compiler and standard +//! library. +//! +//! This module contains some of the real meat in the rustbuild build system +//! which is where Cargo is used to compile the standard library, libtest, and +//! the compiler. This module is also responsible for assembling the sysroot as it +//! goes along from the output of the previous stage. + +use std::borrow::Cow; +use std::collections::HashSet; +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::io::prelude::*; +use std::io::BufReader; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::str; + +use serde_derive::Deserialize; + +use crate::core::build_steps::dist; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::SourceType; +use crate::core::builder::crate_description; +use crate::core::builder::Cargo; +use crate::core::builder::{Builder, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath}; +use crate::core::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{ + exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, output, symlink_dir, t, up_to_date, +}; +use crate::LLVM_TOOLS; +use crate::{CLang, Compiler, DependencyType, GitRepo, Mode}; +use filetime::FileTime; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Std { + pub target: TargetSelection, + pub compiler: Compiler, + /// Whether to build only a subset of crates in the standard library. + /// + /// This shouldn't be used from other steps; see the comment on [`Rustc`]. + crates: Interned>, + /// When using download-rustc, we need to use a new build of `std` for running unit tests of Std itself, + /// but we need to use the downloaded copy of std for linking to rustdoc. Allow this to be overriden by `builder.ensure` from other steps. + force_recompile: bool, + extra_rust_args: &'static [&'static str], +} + +impl Std { + pub fn new(compiler: Compiler, target: TargetSelection) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args: &[], + } + } + + pub fn force_recompile(compiler: Compiler, target: TargetSelection) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: true, + extra_rust_args: &[], + } + } + + pub fn new_with_extra_rust_args( + compiler: Compiler, + target: TargetSelection, + extra_rust_args: &'static [&'static str], + ) -> Self { + Self { + target, + compiler, + crates: Default::default(), + force_recompile: false, + extra_rust_args, + } + } +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // When downloading stage1, the standard library has already been copied to the sysroot, so + // there's no need to rebuild it. + let builder = run.builder; + run.crate_or_deps("sysroot") + .path("library") + .lazy_default_condition(Box::new(|| !builder.download_rustc())) + } + + fn make_run(run: RunConfig<'_>) { + // If the paths include "library", build the entire standard library. + let has_alias = + run.paths.iter().any(|set| set.assert_single_path().path.ends_with("library")); + let crates = if has_alias { Default::default() } else { run.cargo_crates_in_set() }; + + run.builder.ensure(Std { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + crates, + force_recompile: false, + extra_rust_args: &[], + }); + } + + /// Builds the standard library. + /// + /// This will build the standard library for a particular stage of the build + /// using the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let compiler = self.compiler; + + // When using `download-rustc`, we already have artifacts for the host available. Don't + // recompile them. + if builder.download_rustc() && target == builder.build.build + // NOTE: the beta compiler may generate different artifacts than the downloaded compiler, so + // its artifacts can't be reused. + && compiler.stage != 0 + // This check is specific to testing std itself; see `test::Std` for more details. + && !self.force_recompile + { + cp_rustc_component_to_ci_sysroot( + builder, + compiler, + builder.config.ci_rust_std_contents(), + ); + return; + } + + if builder.config.keep_stage.contains(&compiler.stage) + || builder.config.keep_stage_std.contains(&compiler.stage) + { + builder.info("WARNING: Using a potentially old libstd. This may not behave well."); + + copy_third_party_objects(builder, &compiler, target); + copy_self_contained_objects(builder, &compiler, target); + + builder.ensure(StdLink::from_std(self, compiler)); + return; + } + + builder.update_submodule(&Path::new("library").join("stdarch")); + + // Profiler information requires LLVM's compiler-rt + if builder.config.profiler { + builder.update_submodule(&Path::new("src/llvm-project")); + } + + let mut target_deps = builder.ensure(StartupObjects { compiler, target }); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(Std::new(compiler_to_use, target)); + let msg = if compiler_to_use.host == target { + format!( + "Uplifting library (stage{} -> stage{})", + compiler_to_use.stage, compiler.stage + ) + } else { + format!( + "Uplifting library (stage{}:{} -> stage{}:{})", + compiler_to_use.stage, compiler_to_use.host, compiler.stage, target + ) + }; + builder.info(&msg); + + // Even if we're not building std this stage, the new sysroot must + // still contain the third party objects needed by various targets. + copy_third_party_objects(builder, &compiler, target); + copy_self_contained_objects(builder, &compiler, target); + + builder.ensure(StdLink::from_std(self, compiler_to_use)); + return; + } + + target_deps.extend(copy_third_party_objects(builder, &compiler, target)); + target_deps.extend(copy_self_contained_objects(builder, &compiler, target)); + + // The LLD wrappers and `rust-lld` are self-contained linking components that can be + // necessary to link the stdlib on some targets. We'll also need to copy these binaries to + // the `stage0-sysroot` to ensure the linker is found when bootstrapping on such a target. + if compiler.stage == 0 && compiler.host == builder.config.build { + // We want to copy the host `bin` folder within the `rustlib` folder in the sysroot. + let src_sysroot_bin = builder + .rustc_snapshot_sysroot() + .join("lib") + .join("rustlib") + .join(compiler.host.triple) + .join("bin"); + if src_sysroot_bin.exists() { + let target_sysroot_bin = + builder.sysroot_libdir(compiler, target).parent().unwrap().join("bin"); + t!(fs::create_dir_all(&target_sysroot_bin)); + builder.cp_r(&src_sysroot_bin, &target_sysroot_bin); + } + } + + let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "build"); + std_cargo(builder, target, compiler.stage, &mut cargo); + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + // See src/bootstrap/synthetic_targets.rs + if target.is_synthetic() { + cargo.env("RUSTC_BOOTSTRAP_SYNTHETIC_TARGET", "1"); + } + for rustflag in self.extra_rust_args.into_iter() { + cargo.rustflag(rustflag); + } + + let _guard = builder.msg( + Kind::Build, + compiler.stage, + format_args!("library artifacts{}", crate_description(&self.crates)), + compiler.host, + target, + ); + run_cargo( + builder, + cargo, + vec![], + &libstd_stamp(builder, compiler, target), + target_deps, + false, + false, + ); + + builder.ensure(StdLink::from_std( + self, + builder.compiler(compiler.stage, builder.config.build), + )); + } +} + +fn copy_and_stamp( + builder: &Builder<'_>, + libdir: &Path, + sourcedir: &Path, + name: &str, + target_deps: &mut Vec<(PathBuf, DependencyType)>, + dependency_type: DependencyType, +) { + let target = libdir.join(name); + builder.copy(&sourcedir.join(name), &target); + + target_deps.push((target, dependency_type)); +} + +fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: &Path) -> PathBuf { + let libunwind_path = builder.ensure(llvm::Libunwind { target }); + let libunwind_source = libunwind_path.join("libunwind.a"); + let libunwind_target = libdir.join("libunwind.a"); + builder.copy(&libunwind_source, &libunwind_target); + libunwind_target +} + +/// Copies third party objects needed by various targets. +fn copy_third_party_objects( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec<(PathBuf, DependencyType)> { + let mut target_deps = vec![]; + + if builder.config.sanitizers_enabled(target) && compiler.stage != 0 { + // The sanitizers are only copied in stage1 or above, + // to avoid creating dependency on LLVM. + target_deps.extend( + copy_sanitizers(builder, &compiler, target) + .into_iter() + .map(|d| (d, DependencyType::Target)), + ); + } + + if target == "x86_64-fortanix-unknown-sgx" + || builder.config.llvm_libunwind(target) == LlvmLibunwind::InTree + && (target.contains("linux") || target.contains("fuchsia")) + { + let libunwind_path = + copy_llvm_libunwind(builder, target, &builder.sysroot_libdir(*compiler, target)); + target_deps.push((libunwind_path, DependencyType::Target)); + } + + target_deps +} + +/// Copies third party objects needed by various targets for self-contained linkage. +fn copy_self_contained_objects( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec<(PathBuf, DependencyType)> { + let libdir_self_contained = builder.sysroot_libdir(*compiler, target).join("self-contained"); + t!(fs::create_dir_all(&libdir_self_contained)); + let mut target_deps = vec![]; + + // Copies the libc and CRT objects. + // + // rustc historically provides a more self-contained installation for musl targets + // not requiring the presence of a native musl toolchain. For example, it can fall back + // to using gcc from a glibc-targeting toolchain for linking. + // To do that we have to distribute musl startup objects as a part of Rust toolchain + // and link with them manually in the self-contained mode. + if target.contains("musl") && !target.contains("unikraft") { + let srcdir = builder.musl_libdir(target).unwrap_or_else(|| { + panic!("Target {:?} does not have a \"musl-libdir\" key", target.triple) + }); + for &obj in &["libc.a", "crt1.o", "Scrt1.o", "rcrt1.o", "crti.o", "crtn.o"] { + copy_and_stamp( + builder, + &libdir_self_contained, + &srcdir, + obj, + &mut target_deps, + DependencyType::TargetSelfContained, + ); + } + let crt_path = builder.ensure(llvm::CrtBeginEnd { target }); + for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { + let src = crt_path.join(obj); + let target = libdir_self_contained.join(obj); + builder.copy(&src, &target); + target_deps.push((target, DependencyType::TargetSelfContained)); + } + + if !target.starts_with("s390x") { + let libunwind_path = copy_llvm_libunwind(builder, target, &libdir_self_contained); + target_deps.push((libunwind_path, DependencyType::TargetSelfContained)); + } + } else if target.contains("-wasi") { + let srcdir = builder + .wasi_root(target) + .unwrap_or_else(|| { + panic!("Target {:?} does not have a \"wasi-root\" key", target.triple) + }) + .join("lib") + .join(target.to_string().replace("-preview1", "")); + for &obj in &["libc.a", "crt1-command.o", "crt1-reactor.o"] { + copy_and_stamp( + builder, + &libdir_self_contained, + &srcdir, + obj, + &mut target_deps, + DependencyType::TargetSelfContained, + ); + } + } else if target.ends_with("windows-gnu") { + for obj in ["crt2.o", "dllcrt2.o"].iter() { + let src = compiler_file(builder, &builder.cc(target), target, CLang::C, obj); + let target = libdir_self_contained.join(obj); + builder.copy(&src, &target); + target_deps.push((target, DependencyType::TargetSelfContained)); + } + } + + target_deps +} + +/// Configure cargo to compile the standard library, adding appropriate env vars +/// and such. +pub fn std_cargo(builder: &Builder<'_>, target: TargetSelection, stage: u32, cargo: &mut Cargo) { + if let Some(target) = env::var_os("MACOSX_STD_DEPLOYMENT_TARGET") { + cargo.env("MACOSX_DEPLOYMENT_TARGET", target); + } + + if let Some(path) = builder.config.profiler_path(target) { + cargo.env("LLVM_PROFILER_RT_LIB", path); + } + + // Determine if we're going to compile in optimized C intrinsics to + // the `compiler-builtins` crate. These intrinsics live in LLVM's + // `compiler-rt` repository, but our `src/llvm-project` submodule isn't + // always checked out, so we need to conditionally look for this. (e.g. if + // an external LLVM is used we skip the LLVM submodule checkout). + // + // Note that this shouldn't affect the correctness of `compiler-builtins`, + // but only its speed. Some intrinsics in C haven't been translated to Rust + // yet but that's pretty rare. Other intrinsics have optimized + // implementations in C which have only had slower versions ported to Rust, + // so we favor the C version where we can, but it's not critical. + // + // If `compiler-rt` is available ensure that the `c` feature of the + // `compiler-builtins` crate is enabled and it's configured to learn where + // `compiler-rt` is located. + let compiler_builtins_root = builder.src.join("src/llvm-project/compiler-rt"); + let compiler_builtins_c_feature = if compiler_builtins_root.exists() { + // Note that `libprofiler_builtins/build.rs` also computes this so if + // you're changing something here please also change that. + cargo.env("RUST_COMPILER_RT_ROOT", &compiler_builtins_root); + " compiler-builtins-c" + } else { + "" + }; + + // `libtest` uses this to know whether or not to support + // `-Zunstable-options`. + if !builder.unstable_features() { + cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); + } + + let mut features = String::new(); + + if builder.no_std(target) == Some(true) { + features += " compiler-builtins-mem"; + if !target.starts_with("bpf") { + features.push_str(compiler_builtins_c_feature); + } + + // for no-std targets we only compile a few no_std crates + cargo + .args(&["-p", "alloc"]) + .arg("--manifest-path") + .arg(builder.src.join("library/alloc/Cargo.toml")) + .arg("--features") + .arg(features); + } else { + features += &builder.std_features(target); + features.push_str(compiler_builtins_c_feature); + + cargo + .arg("--features") + .arg(features) + .arg("--manifest-path") + .arg(builder.src.join("library/sysroot/Cargo.toml")); + + // Help the libc crate compile by assisting it in finding various + // sysroot native libraries. + if target.contains("musl") { + if let Some(p) = builder.musl_libdir(target) { + let root = format!("native={}", p.to_str().unwrap()); + cargo.rustflag("-L").rustflag(&root); + } + } + + if target.contains("-wasi") { + if let Some(p) = builder.wasi_root(target) { + let root = format!( + "native={}/lib/{}", + p.to_str().unwrap(), + target.to_string().replace("-preview1", "") + ); + cargo.rustflag("-L").rustflag(&root); + } + } + } + + // By default, rustc uses `-Cembed-bitcode=yes`, and Cargo overrides that + // with `-Cembed-bitcode=no` for non-LTO builds. However, libstd must be + // built with bitcode so that the produced rlibs can be used for both LTO + // builds (which use bitcode) and non-LTO builds (which use object code). + // So we override the override here! + // + // But we don't bother for the stage 0 compiler because it's never used + // with LTO. + if stage >= 1 { + cargo.rustflag("-Cembed-bitcode=yes"); + } + if builder.config.rust_lto == RustcLto::Off { + cargo.rustflag("-Clto=off"); + } + + // By default, rustc does not include unwind tables unless they are required + // for a particular target. They are not required by RISC-V targets, but + // compiling the standard library with them means that users can get + // backtraces without having to recompile the standard library themselves. + // + // This choice was discussed in https://github.com/rust-lang/rust/pull/69890 + if target.contains("riscv") { + cargo.rustflag("-Cforce-unwind-tables=yes"); + } + + let html_root = + format!("-Zcrate-attr=doc(html_root_url=\"{}/\")", builder.doc_rust_lang_org_channel(),); + cargo.rustflag(&html_root); + cargo.rustdocflag(&html_root); + + cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct StdLink { + pub compiler: Compiler, + pub target_compiler: Compiler, + pub target: TargetSelection, + /// Not actually used; only present to make sure the cache invalidation is correct. + crates: Interned>, + /// See [`Std::force_recompile`]. + force_recompile: bool, +} + +impl StdLink { + fn from_std(std: Std, host_compiler: Compiler) -> Self { + Self { + compiler: host_compiler, + target_compiler: std.compiler, + target: std.target, + crates: std.crates, + force_recompile: std.force_recompile, + } + } +} + +impl Step for StdLink { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Link all libstd rlibs/dylibs into the sysroot location. + /// + /// Links those artifacts generated by `compiler` to the `stage` compiler's + /// sysroot for the specified `host` and `target`. + /// + /// Note that this assumes that `compiler` has already generated the libstd + /// libraries for `target`, and this method will find them in the relevant + /// output directory. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target_compiler = self.target_compiler; + let target = self.target; + + // NOTE: intentionally does *not* check `target == builder.build` to avoid having to add the same check in `test::Crate`. + let (libdir, hostdir) = if self.force_recompile && builder.download_rustc() { + // NOTE: copies part of `sysroot_libdir` to avoid having to add a new `force_recompile` argument there too + let lib = builder.sysroot_libdir_relative(self.compiler); + let sysroot = builder.ensure(crate::core::build_steps::compile::Sysroot { + compiler: self.compiler, + force_recompile: self.force_recompile, + }); + let libdir = sysroot.join(lib).join("rustlib").join(target.triple).join("lib"); + let hostdir = sysroot.join(lib).join("rustlib").join(compiler.host.triple).join("lib"); + (INTERNER.intern_path(libdir), INTERNER.intern_path(hostdir)) + } else { + let libdir = builder.sysroot_libdir(target_compiler, target); + let hostdir = builder.sysroot_libdir(target_compiler, compiler.host); + (libdir, hostdir) + }; + + add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + + // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0` + // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta, + // and is not set to a custom path. + if compiler.stage == 0 + && builder + .build + .config + .initial_rustc + .starts_with(builder.out.join(&compiler.host.triple).join("stage0/bin")) + { + // Copy bin files from stage0/bin to stage0-sysroot/bin + let sysroot = builder.out.join(&compiler.host.triple).join("stage0-sysroot"); + + let host = compiler.host.triple; + let stage0_bin_dir = builder.out.join(&host).join("stage0/bin"); + let sysroot_bin_dir = sysroot.join("bin"); + t!(fs::create_dir_all(&sysroot_bin_dir)); + builder.cp_r(&stage0_bin_dir, &sysroot_bin_dir); + + // Copy all *.so files from stage0/lib to stage0-sysroot/lib + let stage0_lib_dir = builder.out.join(&host).join("stage0/lib"); + if let Ok(files) = fs::read_dir(&stage0_lib_dir) { + for file in files { + let file = t!(file); + let path = file.path(); + if path.is_file() && is_dylib(&file.file_name().into_string().unwrap()) { + builder.copy(&path, &sysroot.join("lib").join(path.file_name().unwrap())); + } + } + } + + // Copy codegen-backends from stage0 + let sysroot_codegen_backends = builder.sysroot_codegen_backends(compiler); + t!(fs::create_dir_all(&sysroot_codegen_backends)); + let stage0_codegen_backends = builder + .out + .join(&host) + .join("stage0/lib/rustlib") + .join(&host) + .join("codegen-backends"); + builder.cp_r(&stage0_codegen_backends, &sysroot_codegen_backends); + } + } +} + +/// Copies sanitizer runtime libraries into target libdir. +fn copy_sanitizers( + builder: &Builder<'_>, + compiler: &Compiler, + target: TargetSelection, +) -> Vec { + let runtimes: Vec = builder.ensure(llvm::Sanitizers { target }); + + if builder.config.dry_run() { + return Vec::new(); + } + + let mut target_deps = Vec::new(); + let libdir = builder.sysroot_libdir(*compiler, target); + + for runtime in &runtimes { + let dst = libdir.join(&runtime.name); + builder.copy(&runtime.path, &dst); + + // The `aarch64-apple-ios-macabi` and `x86_64-apple-ios-macabi` are also supported for + // sanitizers, but they share a sanitizer runtime with `${arch}-apple-darwin`, so we do + // not list them here to rename and sign the runtime library. + if target == "x86_64-apple-darwin" + || target == "aarch64-apple-darwin" + || target == "aarch64-apple-ios" + || target == "aarch64-apple-ios-sim" + || target == "x86_64-apple-ios" + { + // Update the library’s install name to reflect that it has been renamed. + apple_darwin_update_library_name(&dst, &format!("@rpath/{}", &runtime.name)); + // Upon renaming the install name, the code signature of the file will invalidate, + // so we will sign it again. + apple_darwin_sign_file(&dst); + } + + target_deps.push(dst); + } + + target_deps +} + +fn apple_darwin_update_library_name(library_path: &Path, new_name: &str) { + let status = Command::new("install_name_tool") + .arg("-id") + .arg(new_name) + .arg(library_path) + .status() + .expect("failed to execute `install_name_tool`"); + assert!(status.success()); +} + +fn apple_darwin_sign_file(file_path: &Path) { + let status = Command::new("codesign") + .arg("-f") // Force to rewrite the existing signature + .arg("-s") + .arg("-") + .arg(file_path) + .status() + .expect("failed to execute `codesign`"); + assert!(status.success()); +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct StartupObjects { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for StartupObjects { + type Output = Vec<(PathBuf, DependencyType)>; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("library/rtstartup") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(StartupObjects { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + }); + } + + /// Builds and prepare startup objects like rsbegin.o and rsend.o + /// + /// These are primarily used on Windows right now for linking executables/dlls. + /// They don't require any library support as they're just plain old object + /// files, so we just use the nightly snapshot compiler to always build them (as + /// no other compilers are guaranteed to be available). + fn run(self, builder: &Builder<'_>) -> Vec<(PathBuf, DependencyType)> { + let for_compiler = self.compiler; + let target = self.target; + if !target.ends_with("windows-gnu") { + return vec![]; + } + + let mut target_deps = vec![]; + + let src_dir = &builder.src.join("library").join("rtstartup"); + let dst_dir = &builder.native_dir(target).join("rtstartup"); + let sysroot_dir = &builder.sysroot_libdir(for_compiler, target); + t!(fs::create_dir_all(dst_dir)); + + for file in &["rsbegin", "rsend"] { + let src_file = &src_dir.join(file.to_string() + ".rs"); + let dst_file = &dst_dir.join(file.to_string() + ".o"); + if !up_to_date(src_file, dst_file) { + let mut cmd = Command::new(&builder.initial_rustc); + cmd.env("RUSTC_BOOTSTRAP", "1"); + if !builder.local_rebuild { + // a local_rebuild compiler already has stage1 features + cmd.arg("--cfg").arg("bootstrap"); + } + builder.run( + cmd.arg("--target") + .arg(target.rustc_target_arg()) + .arg("--emit=obj") + .arg("-o") + .arg(dst_file) + .arg(src_file), + ); + } + + let target = sysroot_dir.join((*file).to_string() + ".o"); + builder.copy(dst_file, &target); + target_deps.push((target, DependencyType::Target)); + } + + target_deps + } +} + +fn cp_rustc_component_to_ci_sysroot( + builder: &Builder<'_>, + compiler: Compiler, + contents: Vec, +) { + let sysroot = builder.ensure(Sysroot { compiler, force_recompile: false }); + let ci_rustc_dir = builder.config.ci_rustc_dir(); + + for file in contents { + let src = ci_rustc_dir.join(&file); + let dst = sysroot.join(file); + if src.is_dir() { + t!(fs::create_dir_all(dst)); + } else { + builder.copy(&src, &dst); + } + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rustc { + pub target: TargetSelection, + pub compiler: Compiler, + /// Whether to build a subset of crates, rather than the whole compiler. + /// + /// This should only be requested by the user, not used within rustbuild itself. + /// Using it within rustbuild can lead to confusing situation where lints are replayed + /// in two different steps. + crates: Interned>, +} + +impl Rustc { + pub fn new(compiler: Compiler, target: TargetSelection) -> Self { + Self { target, compiler, crates: Default::default() } + } +} + +impl Step for Rustc { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let mut crates = run.builder.in_tree_crates("rustc-main", None); + for (i, krate) in crates.iter().enumerate() { + // We can't allow `build rustc` as an alias for this Step, because that's reserved by `Assemble`. + // Ideally Assemble would use `build compiler` instead, but that seems too confusing to be worth the breaking change. + if krate.name == "rustc-main" { + crates.swap_remove(i); + break; + } + } + run.crates(crates) + } + + fn make_run(run: RunConfig<'_>) { + let crates = run.cargo_crates_in_set(); + run.builder.ensure(Rustc { + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + target: run.target, + crates, + }); + } + + /// Builds the compiler. + /// + /// This will build the compiler for a particular stage of the build using + /// the `compiler` targeting the `target` architecture. The artifacts + /// created will also be linked into the sysroot directory. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + + // NOTE: the ABI of the beta compiler is different from the ABI of the downloaded compiler, + // so its artifacts can't be reused. + if builder.download_rustc() && compiler.stage != 0 { + // Copy the existing artifacts instead of rebuilding them. + // NOTE: this path is only taken for tools linking to rustc-dev (including ui-fulldeps tests). + cp_rustc_component_to_ci_sysroot( + builder, + compiler, + builder.config.ci_rustc_dev_contents(), + ); + return; + } + + builder.ensure(Std::new(compiler, target)); + + if builder.config.keep_stage.contains(&compiler.stage) { + builder.info("WARNING: Using a potentially old librustc. This may not behave well."); + builder.info("WARNING: Use `--keep-stage-std` if you want to rebuild the compiler when it changes"); + builder.ensure(RustcLink::from_rustc(self, compiler)); + return; + } + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(Rustc::new(compiler_to_use, target)); + let msg = if compiler_to_use.host == target { + format!( + "Uplifting rustc (stage{} -> stage{})", + compiler_to_use.stage, + compiler.stage + 1 + ) + } else { + format!( + "Uplifting rustc (stage{}:{} -> stage{}:{})", + compiler_to_use.stage, + compiler_to_use.host, + compiler.stage + 1, + target + ) + }; + builder.info(&msg); + builder.ensure(RustcLink::from_rustc(self, compiler_to_use)); + return; + } + + // Ensure that build scripts and proc macros have a std / libproc_macro to link against. + builder.ensure(Std::new( + builder.compiler(self.compiler.stage, builder.config.build), + builder.config.build, + )); + + let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "build"); + rustc_cargo(builder, &mut cargo, target, compiler.stage); + + if builder.config.rust_profile_use.is_some() + && builder.config.rust_profile_generate.is_some() + { + panic!("Cannot use and generate PGO profiles at the same time"); + } + + // With LLD, we can use ICF (identical code folding) to reduce the executable size + // of librustc_driver/rustc and to improve i-cache utilization. + // + // -Wl,[link options] doesn't work on MSVC. However, /OPT:ICF (technically /OPT:REF,ICF) + // is already on by default in MSVC optimized builds, which is interpreted as --icf=all: + // https://github.com/llvm/llvm-project/blob/3329cec2f79185bafd678f310fafadba2a8c76d2/lld/COFF/Driver.cpp#L1746 + // https://github.com/rust-lang/rust/blob/f22819bcce4abaff7d1246a56eec493418f9f4ee/compiler/rustc_codegen_ssa/src/back/linker.rs#L827 + if builder.config.use_lld && !compiler.host.contains("msvc") { + cargo.rustflag("-Clink-args=-Wl,--icf=all"); + } + + let is_collecting = if let Some(path) = &builder.config.rust_profile_generate { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-generate={path}")); + // Apparently necessary to avoid overflowing the counters during + // a Cargo build profile + cargo.rustflag("-Cllvm-args=-vp-counters-per-site=4"); + true + } else { + false + } + } else if let Some(path) = &builder.config.rust_profile_use { + if compiler.stage == 1 { + cargo.rustflag(&format!("-Cprofile-use={path}")); + cargo.rustflag("-Cllvm-args=-pgo-warn-missing-function"); + true + } else { + false + } + } else { + false + }; + if is_collecting { + // Ensure paths to Rust sources are relative, not absolute. + cargo.rustflag(&format!( + "-Cllvm-args=-static-func-strip-dirname-prefix={}", + builder.config.src.components().count() + )); + } + + // We currently don't support cross-crate LTO in stage0. This also isn't hugely necessary + // and may just be a time sink. + if compiler.stage != 0 { + match builder.config.rust_lto { + RustcLto::Thin | RustcLto::Fat => { + // Since using LTO for optimizing dylibs is currently experimental, + // we need to pass -Zdylib-lto. + cargo.rustflag("-Zdylib-lto"); + // Cargo by default passes `-Cembed-bitcode=no` and doesn't pass `-Clto` when + // compiling dylibs (and their dependencies), even when LTO is enabled for the + // crate. Therefore, we need to override `-Clto` and `-Cembed-bitcode` here. + let lto_type = match builder.config.rust_lto { + RustcLto::Thin => "thin", + RustcLto::Fat => "fat", + _ => unreachable!(), + }; + cargo.rustflag(&format!("-Clto={lto_type}")); + cargo.rustflag("-Cembed-bitcode=yes"); + } + RustcLto::ThinLocal => { /* Do nothing, this is the default */ } + RustcLto::Off => { + cargo.rustflag("-Clto=off"); + } + } + } else if builder.config.rust_lto == RustcLto::Off { + cargo.rustflag("-Clto=off"); + } + + for krate in &*self.crates { + cargo.arg("-p").arg(krate); + } + + if builder.build.config.enable_bolt_settings && compiler.stage == 1 { + // Relocations are required for BOLT to work. + cargo.env("RUSTC_BOLT_LINK_FLAGS", "1"); + } + + let _guard = builder.msg_sysroot_tool( + Kind::Build, + compiler.stage, + format_args!("compiler artifacts{}", crate_description(&self.crates)), + compiler.host, + target, + ); + let stamp = librustc_stamp(builder, compiler, target); + run_cargo( + builder, + cargo, + vec![], + &stamp, + vec![], + false, + true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. + ); + + // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain + // unexpected debuginfo from dependencies, for example from the C++ standard library used in + // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with + // debuginfo (via the debuginfo level of the executables using it): strip this debuginfo + // away after the fact. + if builder.config.rust_debuginfo_level_rustc == DebuginfoLevel::None + && builder.config.rust_debuginfo_level_tools == DebuginfoLevel::None + { + let target_root_dir = stamp.parent().unwrap(); + let rustc_driver = target_root_dir.join("librustc_driver.so"); + strip_debug(builder, target, &rustc_driver); + } + + builder.ensure(RustcLink::from_rustc( + self, + builder.compiler(compiler.stage, builder.config.build), + )); + } +} + +pub fn rustc_cargo(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection, stage: u32) { + cargo + .arg("--features") + .arg(builder.rustc_features(builder.kind)) + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc/Cargo.toml")); + + cargo.rustdocflag("-Zcrate-attr=warn(rust_2018_idioms)"); + + rustc_cargo_env(builder, cargo, target, stage); +} + +pub fn rustc_cargo_env( + builder: &Builder<'_>, + cargo: &mut Cargo, + target: TargetSelection, + stage: u32, +) { + // Set some configuration variables picked up by build scripts and + // the compiler alike + cargo + .env("CFG_RELEASE", builder.rust_release()) + .env("CFG_RELEASE_CHANNEL", &builder.config.channel) + .env("CFG_VERSION", builder.rust_version()); + + if let Some(backend) = builder.config.default_codegen_backend() { + cargo.env("CFG_DEFAULT_CODEGEN_BACKEND", backend); + } + + let libdir_relative = builder.config.libdir_relative().unwrap_or_else(|| Path::new("lib")); + let target_config = builder.config.target_config.get(&target); + + cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); + + if let Some(ref ver_date) = builder.rust_info().commit_date() { + cargo.env("CFG_VER_DATE", ver_date); + } + if let Some(ref ver_hash) = builder.rust_info().sha() { + cargo.env("CFG_VER_HASH", ver_hash); + } + if !builder.unstable_features() { + cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1"); + } + + // Prefer the current target's own default_linker, else a globally + // specified one. + if let Some(s) = target_config.and_then(|c| c.default_linker.as_ref()) { + cargo.env("CFG_DEFAULT_LINKER", s); + } else if let Some(ref s) = builder.config.rustc_default_linker { + cargo.env("CFG_DEFAULT_LINKER", s); + } + + if builder.config.rustc_parallel { + // keep in sync with `bootstrap/lib.rs:Build::rustc_features` + // `cfg` option for rustc, `features` option for cargo, for conditional compilation + cargo.rustflag("--cfg=parallel_compiler"); + cargo.rustdocflag("--cfg=parallel_compiler"); + } + if builder.config.rust_verify_llvm_ir { + cargo.env("RUSTC_VERIFY_LLVM_IR", "1"); + } + + // Note that this is disabled if LLVM itself is disabled or we're in a check + // build. If we are in a check build we still go ahead here presuming we've + // detected that LLVM is already built and good to go which helps prevent + // busting caches (e.g. like #71152). + if builder.config.llvm_enabled() { + let building_is_expensive = + crate::core::build_steps::llvm::prebuilt_llvm_config(builder, target).is_err(); + // `top_stage == stage` might be false for `check --stage 1`, if we are building the stage 1 compiler + let can_skip_build = builder.kind == Kind::Check && builder.top_stage == stage; + let should_skip_build = building_is_expensive && can_skip_build; + if !should_skip_build { + rustc_llvm_env(builder, cargo, target) + } + } +} + +/// Pass down configuration from the LLVM build into the build of +/// rustc_llvm and rustc_codegen_llvm. +fn rustc_llvm_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetSelection) { + let target_config = builder.config.target_config.get(&target); + + if builder.is_rust_llvm(target) { + cargo.env("LLVM_RUSTLLVM", "1"); + } + let llvm::LlvmResult { llvm_config, .. } = builder.ensure(llvm::Llvm { target }); + cargo.env("LLVM_CONFIG", &llvm_config); + if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + cargo.env("CFG_LLVM_ROOT", s); + } + + // Some LLVM linker flags (-L and -l) may be needed to link `rustc_llvm`. Its build script + // expects these to be passed via the `LLVM_LINKER_FLAGS` env variable, separated by + // whitespace. + // + // For example: + // - on windows, when `clang-cl` is used with instrumentation, we need to manually add + // clang's runtime library resource directory so that the profiler runtime library can be + // found. This is to avoid the linker errors about undefined references to + // `__llvm_profile_instrument_memop` when linking `rustc_driver`. + let mut llvm_linker_flags = String::new(); + if builder.config.llvm_profile_generate && target.contains("msvc") { + if let Some(ref clang_cl_path) = builder.config.llvm_clang_cl { + // Add clang's runtime library directory to the search path + let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); + llvm_linker_flags.push_str(&format!("-L{}", clang_rt_dir.display())); + } + } + + // The config can also specify its own llvm linker flags. + if let Some(ref s) = builder.config.llvm_ldflags { + if !llvm_linker_flags.is_empty() { + llvm_linker_flags.push_str(" "); + } + llvm_linker_flags.push_str(s); + } + + // Set the linker flags via the env var that `rustc_llvm`'s build script will read. + if !llvm_linker_flags.is_empty() { + cargo.env("LLVM_LINKER_FLAGS", llvm_linker_flags); + } + + // Building with a static libstdc++ is only supported on linux right now, + // not for MSVC or macOS + if builder.config.llvm_static_stdcpp + && !target.contains("freebsd") + && !target.contains("msvc") + && !target.contains("apple") + && !target.contains("solaris") + { + let file = compiler_file( + builder, + &builder.cxx(target).unwrap(), + target, + CLang::Cxx, + "libstdc++.a", + ); + cargo.env("LLVM_STATIC_STDCPP", file); + } + if builder.llvm_link_shared() { + cargo.env("LLVM_LINK_SHARED", "1"); + } + if builder.config.llvm_use_libcxx { + cargo.env("LLVM_USE_LIBCXX", "1"); + } + if builder.config.llvm_optimize && !builder.config.llvm_release_debuginfo { + cargo.env("LLVM_NDEBUG", "1"); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct RustcLink { + pub compiler: Compiler, + pub target_compiler: Compiler, + pub target: TargetSelection, + /// Not actually used; only present to make sure the cache invalidation is correct. + crates: Interned>, +} + +impl RustcLink { + fn from_rustc(rustc: Rustc, host_compiler: Compiler) -> Self { + Self { + compiler: host_compiler, + target_compiler: rustc.compiler, + target: rustc.target, + crates: rustc.crates, + } + } +} + +impl Step for RustcLink { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Same as `std_link`, only for librustc + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target_compiler = self.target_compiler; + let target = self.target; + add_to_sysroot( + builder, + &builder.sysroot_libdir(target_compiler, target), + &builder.sysroot_libdir(target_compiler, compiler.host), + &librustc_stamp(builder, compiler, target), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CodegenBackend { + pub target: TargetSelection, + pub compiler: Compiler, + pub backend: Interned, +} + +fn needs_codegen_config(run: &RunConfig<'_>) -> bool { + let mut needs_codegen_cfg = false; + for path_set in &run.paths { + needs_codegen_cfg = match path_set { + PathSet::Set(set) => set.iter().any(|p| is_codegen_cfg_needed(p, run)), + PathSet::Suite(suite) => is_codegen_cfg_needed(&suite, run), + } + } + needs_codegen_cfg +} + +pub(crate) const CODEGEN_BACKEND_PREFIX: &str = "rustc_codegen_"; + +fn is_codegen_cfg_needed(path: &TaskPath, run: &RunConfig<'_>) -> bool { + if path.path.to_str().unwrap().contains(&CODEGEN_BACKEND_PREFIX) { + let mut needs_codegen_backend_config = true; + for &backend in &run.builder.config.rust_codegen_backends { + if path + .path + .to_str() + .unwrap() + .ends_with(&(CODEGEN_BACKEND_PREFIX.to_owned() + &backend)) + { + needs_codegen_backend_config = false; + } + } + if needs_codegen_backend_config { + run.builder.info( + "WARNING: no codegen-backends config matched the requested path to build a codegen backend. \ + HELP: add backend to codegen-backends in config.toml.", + ); + return true; + } + } + + return false; +} + +impl Step for CodegenBackend { + type Output = (); + const ONLY_HOSTS: bool = true; + // Only the backends specified in the `codegen-backends` entry of `config.toml` are built. + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift", "compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + if needs_codegen_config(&run) { + return; + } + + for &backend in &run.builder.config.rust_codegen_backends { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + run.builder.ensure(CodegenBackend { + target: run.target, + compiler: run.builder.compiler(run.builder.top_stage, run.build_triple()), + backend, + }); + } + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + let backend = self.backend; + + builder.ensure(Rustc::new(compiler, target)); + + if builder.config.keep_stage.contains(&compiler.stage) { + builder.info( + "WARNING: Using a potentially old codegen backend. \ + This may not behave well.", + ); + // Codegen backends are linked separately from this step today, so we don't do + // anything here. + return; + } + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + if compiler_to_use != compiler { + builder.ensure(CodegenBackend { compiler: compiler_to_use, target, backend }); + return; + } + + let out_dir = builder.cargo_out(compiler, Mode::Codegen, target); + + let mut cargo = builder.cargo(compiler, Mode::Codegen, SourceType::InTree, target, "build"); + cargo + .arg("--manifest-path") + .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); + rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + let tmp_stamp = out_dir.join(".tmp.stamp"); + + let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target); + let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false); + if builder.config.dry_run() { + return; + } + let mut files = files.into_iter().filter(|f| { + let filename = f.file_name().unwrap().to_str().unwrap(); + is_dylib(filename) && filename.contains("rustc_codegen_") + }); + let codegen_backend = match files.next() { + Some(f) => f, + None => panic!("no dylibs built for codegen backend?"), + }; + if let Some(f) = files.next() { + panic!( + "codegen backend built two dylibs:\n{}\n{}", + codegen_backend.display(), + f.display() + ); + } + let stamp = codegen_backend_stamp(builder, compiler, target, backend); + let codegen_backend = codegen_backend.to_str().unwrap(); + t!(fs::write(&stamp, &codegen_backend)); + } +} + +/// Creates the `codegen-backends` folder for a compiler that's about to be +/// assembled as a complete compiler. +/// +/// This will take the codegen artifacts produced by `compiler` and link them +/// into an appropriate location for `target_compiler` to be a functional +/// compiler. +fn copy_codegen_backends_to_sysroot( + builder: &Builder<'_>, + compiler: Compiler, + target_compiler: Compiler, +) { + let target = target_compiler.host; + + // Note that this step is different than all the other `*Link` steps in + // that it's not assembling a bunch of libraries but rather is primarily + // moving the codegen backend into place. The codegen backend of rustc is + // not linked into the main compiler by default but is rather dynamically + // selected at runtime for inclusion. + // + // Here we're looking for the output dylib of the `CodegenBackend` step and + // we're copying that into the `codegen-backends` folder. + let dst = builder.sysroot_codegen_backends(target_compiler); + t!(fs::create_dir_all(&dst), dst); + + if builder.config.dry_run() { + return; + } + + for backend in builder.config.rust_codegen_backends.iter() { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + let stamp = codegen_backend_stamp(builder, compiler, target, *backend); + let dylib = t!(fs::read_to_string(&stamp)); + let file = Path::new(&dylib); + let filename = file.file_name().unwrap().to_str().unwrap(); + // change `librustc_codegen_cranelift-xxxxxx.so` to + // `librustc_codegen_cranelift-release.so` + let target_filename = { + let dash = filename.find('-').unwrap(); + let dot = filename.find('.').unwrap(); + format!("{}-{}{}", &filename[..dash], builder.rust_release(), &filename[dot..]) + }; + builder.copy(&file, &dst.join(target_filename)); + } +} + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { + builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp") +} + +/// Cargo's output path for librustc in a given stage, compiled by a particular +/// compiler for the specified target. +pub fn librustc_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> PathBuf { + builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp") +} + +/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular +/// compiler for the specified target and backend. +fn codegen_backend_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + backend: Interned, +) -> PathBuf { + builder + .cargo_out(compiler, Mode::Codegen, target) + .join(format!(".librustc_codegen_{backend}.stamp")) +} + +pub fn compiler_file( + builder: &Builder<'_>, + compiler: &Path, + target: TargetSelection, + c: CLang, + file: &str, +) -> PathBuf { + if builder.config.dry_run() { + return PathBuf::new(); + } + let mut cmd = Command::new(compiler); + cmd.args(builder.cflags(target, GitRepo::Rustc, c)); + cmd.arg(format!("-print-file-name={file}")); + let out = output(&mut cmd); + PathBuf::from(out.trim()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Sysroot { + pub compiler: Compiler, + /// See [`Std::force_recompile`]. + force_recompile: bool, +} + +impl Sysroot { + pub(crate) fn new(compiler: Compiler) -> Self { + Sysroot { compiler, force_recompile: false } + } +} + +impl Step for Sysroot { + type Output = Interned; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Returns the sysroot for the `compiler` specified that *this build system + /// generates*. + /// + /// That is, the sysroot for the stage0 compiler is not what the compiler + /// thinks it is by default, but it's the same as the default for stages + /// 1-3. + fn run(self, builder: &Builder<'_>) -> Interned { + let compiler = self.compiler; + let host_dir = builder.out.join(&compiler.host.triple); + + let sysroot_dir = |stage| { + if stage == 0 { + host_dir.join("stage0-sysroot") + } else if self.force_recompile && stage == compiler.stage { + host_dir.join(format!("stage{stage}-test-sysroot")) + } else if builder.download_rustc() && compiler.stage != builder.top_stage { + host_dir.join("ci-rustc-sysroot") + } else { + host_dir.join(format!("stage{}", stage)) + } + }; + let sysroot = sysroot_dir(compiler.stage); + + builder.verbose(&format!("Removing sysroot {} to avoid caching bugs", sysroot.display())); + let _ = fs::remove_dir_all(&sysroot); + t!(fs::create_dir_all(&sysroot)); + + // In some cases(see https://github.com/rust-lang/rust/issues/109314), when the stage0 + // compiler relies on more recent version of LLVM than the beta compiler, it may not + // be able to locate the correct LLVM in the sysroot. This situation typically occurs + // when we upgrade LLVM version while the beta compiler continues to use an older version. + // + // Make sure to add the correct version of LLVM into the stage0 sysroot. + if compiler.stage == 0 { + dist::maybe_install_llvm_target(builder, compiler.host, &sysroot); + } + + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.download_rustc() && compiler.stage != 0 { + assert_eq!( + builder.config.build, compiler.host, + "Cross-compiling is not yet supported with `download-rustc`", + ); + + // #102002, cleanup old toolchain folders when using download-rustc so people don't use them by accident. + for stage in 0..=2 { + if stage != compiler.stage { + let dir = sysroot_dir(stage); + if !dir.ends_with("ci-rustc-sysroot") { + let _ = fs::remove_dir_all(dir); + } + } + } + + // Copy the compiler into the correct sysroot. + // NOTE(#108767): We intentionally don't copy `rustc-dev` artifacts until they're requested with `builder.ensure(Rustc)`. + // This fixes an issue where we'd have multiple copies of libc in the sysroot with no way to tell which to load. + // There are a few quirks of bootstrap that interact to make this reliable: + // 1. The order `Step`s are run is hard-coded in `builder.rs` and not configurable. This + // avoids e.g. reordering `test::UiFulldeps` before `test::Ui` and causing the latter to + // fail because of duplicate metadata. + // 2. The sysroot is deleted and recreated between each invocation, so running `x test + // ui-fulldeps && x test ui` can't cause failures. + let mut filtered_files = Vec::new(); + let mut add_filtered_files = |suffix, contents| { + for path in contents { + let path = Path::new(&path); + if path.parent().map_or(false, |parent| parent.ends_with(&suffix)) { + filtered_files.push(path.file_name().unwrap().to_owned()); + } + } + }; + let suffix = format!("lib/rustlib/{}/lib", compiler.host); + add_filtered_files(suffix.as_str(), builder.config.ci_rustc_dev_contents()); + // NOTE: we can't copy std eagerly because `stage2-test-sysroot` needs to have only the + // newly compiled std, not the downloaded std. + add_filtered_files("lib", builder.config.ci_rust_std_contents()); + + let filtered_extensions = [ + OsStr::new("rmeta"), + OsStr::new("rlib"), + // FIXME: this is wrong when compiler.host != build, but we don't support that today + OsStr::new(std::env::consts::DLL_EXTENSION), + ]; + let ci_rustc_dir = builder.config.ci_rustc_dir(); + builder.cp_filtered(&ci_rustc_dir, &sysroot, &|path| { + if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { + return true; + } + if !path.parent().map_or(true, |p| p.ends_with(&suffix)) { + return true; + } + if !filtered_files.iter().all(|f| f != path.file_name().unwrap()) { + builder.verbose_than(1, &format!("ignoring {}", path.display())); + false + } else { + true + } + }); + } + + // Symlink the source root into the same location inside the sysroot, + // where `rust-src` component would go (`$sysroot/lib/rustlib/src/rust`), + // so that any tools relying on `rust-src` also work for local builds, + // and also for translating the virtual `/rustc/$hash` back to the real + // directory (for running tests with `rust.remap-debuginfo = true`). + let sysroot_lib_rustlib_src = sysroot.join("lib/rustlib/src"); + t!(fs::create_dir_all(&sysroot_lib_rustlib_src)); + let sysroot_lib_rustlib_src_rust = sysroot_lib_rustlib_src.join("rust"); + if let Err(e) = symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_src_rust) { + eprintln!( + "WARNING: creating symbolic link `{}` to `{}` failed with {}", + sysroot_lib_rustlib_src_rust.display(), + builder.src.display(), + e, + ); + if builder.config.rust_remap_debuginfo { + eprintln!( + "WARNING: some `tests/ui` tests will fail when lacking `{}`", + sysroot_lib_rustlib_src_rust.display(), + ); + } + } + // Same for the rustc-src component. + let sysroot_lib_rustlib_rustcsrc = sysroot.join("lib/rustlib/rustc-src"); + t!(fs::create_dir_all(&sysroot_lib_rustlib_rustcsrc)); + let sysroot_lib_rustlib_rustcsrc_rust = sysroot_lib_rustlib_rustcsrc.join("rust"); + if let Err(e) = + symlink_dir(&builder.config, &builder.src, &sysroot_lib_rustlib_rustcsrc_rust) + { + eprintln!( + "WARNING: creating symbolic link `{}` to `{}` failed with {}", + sysroot_lib_rustlib_rustcsrc_rust.display(), + builder.src.display(), + e, + ); + } + + INTERNER.intern_path(sysroot) + } +} + +#[derive(Debug, Copy, PartialOrd, Ord, Clone, PartialEq, Eq, Hash)] +pub struct Assemble { + /// The compiler which we will produce in this step. Assemble itself will + /// take care of ensuring that the necessary prerequisites to do so exist, + /// that is, this target can be a stage2 compiler and Assemble will build + /// previous stages for you. + pub target_compiler: Compiler, +} + +impl Step for Assemble { + type Output = Compiler; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("compiler/rustc").path("compiler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Assemble { + target_compiler: run.builder.compiler(run.builder.top_stage + 1, run.target), + }); + } + + /// Prepare a new compiler from the artifacts in `stage` + /// + /// This will assemble a compiler in `build/$host/stage$stage`. The compiler + /// must have been previously produced by the `stage - 1` builder.build + /// compiler. + fn run(self, builder: &Builder<'_>) -> Compiler { + let target_compiler = self.target_compiler; + + if target_compiler.stage == 0 { + assert_eq!( + builder.config.build, target_compiler.host, + "Cannot obtain compiler for non-native build triple at stage 0" + ); + // The stage 0 compiler for the build triple is always pre-built. + return target_compiler; + } + + // Get the compiler that we'll use to bootstrap ourselves. + // + // Note that this is where the recursive nature of the bootstrap + // happens, as this will request the previous stage's compiler on + // downwards to stage 0. + // + // Also note that we're building a compiler for the host platform. We + // only assume that we can run `build` artifacts, which means that to + // produce some other architecture compiler we need to start from + // `build` to get there. + // + // FIXME: It may be faster if we build just a stage 1 compiler and then + // use that to bootstrap this compiler forward. + let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); + + // If we're downloading a compiler from CI, we can use the same compiler for all stages other than 0. + if builder.download_rustc() { + let sysroot = + builder.ensure(Sysroot { compiler: target_compiler, force_recompile: false }); + // Ensure that `libLLVM.so` ends up in the newly created target directory, + // so that tools using `rustc_private` can use it. + dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); + // Lower stages use `ci-rustc-sysroot`, not stageN + if target_compiler.stage == builder.top_stage { + builder.info(&format!("Creating a sysroot for stage{stage} compiler (use `rustup toolchain link 'name' build/host/stage{stage}`)", stage=target_compiler.stage)); + } + return target_compiler; + } + + // Build the libraries for this compiler to link to (i.e., the libraries + // it uses at runtime). NOTE: Crates the target compiler compiles don't + // link to these. (FIXME: Is that correct? It seems to be correct most + // of the time but I think we do link to these for stage2/bin compilers + // when not performing a full bootstrap). + builder.ensure(Rustc::new(build_compiler, target_compiler.host)); + + // FIXME: For now patch over problems noted in #90244 by early returning here, even though + // we've not properly assembled the target sysroot. A full fix is pending further investigation, + // for now full bootstrap usage is rare enough that this is OK. + if target_compiler.stage >= 3 && !builder.config.full_bootstrap { + return target_compiler; + } + + for &backend in builder.config.rust_codegen_backends.iter() { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + builder.ensure(CodegenBackend { + compiler: build_compiler, + target: target_compiler.host, + backend, + }); + } + + let lld_install = if builder.config.lld_enabled { + Some(builder.ensure(llvm::Lld { target: target_compiler.host })) + } else { + None + }; + + let stage = target_compiler.stage; + let host = target_compiler.host; + let (host_info, dir_name) = if build_compiler.host == host { + ("".into(), "host".into()) + } else { + (format!(" ({host})"), host.to_string()) + }; + // NOTE: "Creating a sysroot" is somewhat inconsistent with our internal terminology, since + // sysroots can temporarily be empty until we put the compiler inside. However, + // `ensure(Sysroot)` isn't really something that's user facing, so there shouldn't be any + // ambiguity. + let msg = format!( + "Creating a sysroot for stage{stage} compiler{host_info} (use `rustup toolchain link 'name' build/{dir_name}/stage{stage}`)" + ); + builder.info(&msg); + + // Link in all dylibs to the libdir + let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); + let proc_macros = builder + .read_stamp_file(&stamp) + .into_iter() + .filter_map(|(path, dependency_type)| { + if dependency_type == DependencyType::Host { + Some(path.file_name().unwrap().to_owned().into_string().unwrap()) + } else { + None + } + }) + .collect::>(); + + let sysroot = builder.sysroot(target_compiler); + let rustc_libdir = builder.rustc_libdir(target_compiler); + t!(fs::create_dir_all(&rustc_libdir)); + let src_libdir = builder.sysroot_libdir(build_compiler, host); + for f in builder.read_dir(&src_libdir) { + let filename = f.file_name().into_string().unwrap(); + if (is_dylib(&filename) || is_debug_info(&filename)) && !proc_macros.contains(&filename) + { + builder.copy(&f.path(), &rustc_libdir.join(&filename)); + } + } + + copy_codegen_backends_to_sysroot(builder, build_compiler, target_compiler); + + // We prepend this bin directory to the user PATH when linking Rust binaries. To + // avoid shadowing the system LLD we rename the LLD we provide to `rust-lld`. + let libdir = builder.sysroot_libdir(target_compiler, target_compiler.host); + let libdir_bin = libdir.parent().unwrap().join("bin"); + t!(fs::create_dir_all(&libdir_bin)); + if let Some(lld_install) = lld_install { + let src_exe = exe("lld", target_compiler.host); + let dst_exe = exe("rust-lld", target_compiler.host); + builder.copy(&lld_install.join("bin").join(&src_exe), &libdir_bin.join(&dst_exe)); + let self_contained_lld_dir = libdir_bin.join("gcc-ld"); + t!(fs::create_dir(&self_contained_lld_dir)); + let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { + compiler: build_compiler, + target: target_compiler.host, + }); + for name in crate::LLD_FILE_NAMES { + builder.copy( + &lld_wrapper_exe, + &self_contained_lld_dir.join(exe(name, target_compiler.host)), + ); + } + } + + if builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { + let llvm::LlvmResult { llvm_config, .. } = + builder.ensure(llvm::Llvm { target: target_compiler.host }); + if !builder.config.dry_run() { + let llvm_bin_dir = output(Command::new(llvm_config).arg("--bindir")); + let llvm_bin_dir = Path::new(llvm_bin_dir.trim()); + + // Since we've already built the LLVM tools, install them to the sysroot. + // This is the equivalent of installing the `llvm-tools-preview` component via + // rustup, and lets developers use a locally built toolchain to + // build projects that expect llvm tools to be present in the sysroot + // (e.g. the `bootimage` crate). + for tool in LLVM_TOOLS { + let tool_exe = exe(tool, target_compiler.host); + let src_path = llvm_bin_dir.join(&tool_exe); + // When using `download-ci-llvm`, some of the tools + // may not exist, so skip trying to copy them. + if src_path.exists() { + builder.copy(&src_path, &libdir_bin.join(&tool_exe)); + } + } + } + } + + // Ensure that `libLLVM.so` ends up in the newly build compiler directory, + // so that it can be found when the newly built `rustc` is run. + dist::maybe_install_llvm_runtime(builder, target_compiler.host, &sysroot); + dist::maybe_install_llvm_target(builder, target_compiler.host, &sysroot); + + // Link the compiler binary itself into place + let out_dir = builder.cargo_out(build_compiler, Mode::Rustc, host); + let rustc = out_dir.join(exe("rustc-main", host)); + let bindir = sysroot.join("bin"); + t!(fs::create_dir_all(&bindir)); + let compiler = builder.rustc(target_compiler); + builder.copy(&rustc, &compiler); + + target_compiler + } +} + +/// Link some files into a rustc sysroot. +/// +/// For a particular stage this will link the file listed in `stamp` into the +/// `sysroot_dst` provided. +pub fn add_to_sysroot( + builder: &Builder<'_>, + sysroot_dst: &Path, + sysroot_host_dst: &Path, + stamp: &Path, +) { + let self_contained_dst = &sysroot_dst.join("self-contained"); + t!(fs::create_dir_all(&sysroot_dst)); + t!(fs::create_dir_all(&sysroot_host_dst)); + t!(fs::create_dir_all(&self_contained_dst)); + for (path, dependency_type) in builder.read_stamp_file(stamp) { + let dst = match dependency_type { + DependencyType::Host => sysroot_host_dst, + DependencyType::Target => sysroot_dst, + DependencyType::TargetSelfContained => self_contained_dst, + }; + builder.copy(&path, &dst.join(path.file_name().unwrap())); + } +} + +pub fn run_cargo( + builder: &Builder<'_>, + cargo: Cargo, + tail_args: Vec, + stamp: &Path, + additional_target_deps: Vec<(PathBuf, DependencyType)>, + is_check: bool, + rlib_only_metadata: bool, +) -> Vec { + if builder.config.dry_run() { + return Vec::new(); + } + + // `target_root_dir` looks like $dir/$target/release + let target_root_dir = stamp.parent().unwrap(); + // `target_deps_dir` looks like $dir/$target/release/deps + let target_deps_dir = target_root_dir.join("deps"); + // `host_root_dir` looks like $dir/release + let host_root_dir = target_root_dir + .parent() + .unwrap() // chop off `release` + .parent() + .unwrap() // chop off `$target` + .join(target_root_dir.file_name().unwrap()); + + // Spawn Cargo slurping up its JSON output. We'll start building up the + // `deps` array of all files it generated along with a `toplevel` array of + // files we need to probe for later. + let mut deps = Vec::new(); + let mut toplevel = Vec::new(); + let ok = stream_cargo(builder, cargo, tail_args, &mut |msg| { + let (filenames, crate_types) = match msg { + CargoMessage::CompilerArtifact { + filenames, + target: CargoTarget { crate_types }, + .. + } => (filenames, crate_types), + _ => return, + }; + for filename in filenames { + // Skip files like executables + let mut keep = false; + if filename.ends_with(".lib") + || filename.ends_with(".a") + || is_debug_info(&filename) + || is_dylib(&filename) + { + // Always keep native libraries, rust dylibs and debuginfo + keep = true; + } + if is_check && filename.ends_with(".rmeta") { + // During check builds we need to keep crate metadata + keep = true; + } else if rlib_only_metadata { + if filename.contains("jemalloc_sys") + || filename.contains("rustc_smir") + || filename.contains("stable_mir") + { + // jemalloc_sys and rustc_smir are not linked into librustc_driver.so, + // so we need to distribute them as rlib to be able to use them. + keep |= filename.ends_with(".rlib"); + } else { + // Distribute the rest of the rustc crates as rmeta files only to reduce + // the tarball sizes by about 50%. The object files are linked into + // librustc_driver.so, so it is still possible to link against them. + keep |= filename.ends_with(".rmeta"); + } + } else { + // In all other cases keep all rlibs + keep |= filename.ends_with(".rlib"); + } + + if !keep { + continue; + } + + let filename = Path::new(&*filename); + + // If this was an output file in the "host dir" we don't actually + // worry about it, it's not relevant for us + if filename.starts_with(&host_root_dir) { + // Unless it's a proc macro used in the compiler + if crate_types.iter().any(|t| t == "proc-macro") { + deps.push((filename.to_path_buf(), DependencyType::Host)); + } + continue; + } + + // If this was output in the `deps` dir then this is a precise file + // name (hash included) so we start tracking it. + if filename.starts_with(&target_deps_dir) { + deps.push((filename.to_path_buf(), DependencyType::Target)); + continue; + } + + // Otherwise this was a "top level artifact" which right now doesn't + // have a hash in the name, but there's a version of this file in + // the `deps` folder which *does* have a hash in the name. That's + // the one we'll want to we'll probe for it later. + // + // We do not use `Path::file_stem` or `Path::extension` here, + // because some generated files may have multiple extensions e.g. + // `std-.dll.lib` on Windows. The aforementioned methods only + // split the file name by the last extension (`.lib`) while we need + // to split by all extensions (`.dll.lib`). + let expected_len = t!(filename.metadata()).len(); + let filename = filename.file_name().unwrap().to_str().unwrap(); + let mut parts = filename.splitn(2, '.'); + let file_stem = parts.next().unwrap().to_owned(); + let extension = parts.next().unwrap().to_owned(); + + toplevel.push((file_stem, extension, expected_len)); + } + }); + + if !ok { + crate::exit!(1); + } + + // Ok now we need to actually find all the files listed in `toplevel`. We've + // got a list of prefix/extensions and we basically just need to find the + // most recent file in the `deps` folder corresponding to each one. + let contents = t!(target_deps_dir.read_dir()) + .map(|e| t!(e)) + .map(|e| (e.path(), e.file_name().into_string().unwrap(), t!(e.metadata()))) + .collect::>(); + for (prefix, extension, expected_len) in toplevel { + let candidates = contents.iter().filter(|&&(_, ref filename, ref meta)| { + meta.len() == expected_len + && filename + .strip_prefix(&prefix[..]) + .map(|s| s.starts_with('-') && s.ends_with(&extension[..])) + .unwrap_or(false) + }); + let max = candidates.max_by_key(|&&(_, _, ref metadata)| { + metadata.modified().expect("mtime should be available on all relevant OSes") + }); + let path_to_add = match max { + Some(triple) => triple.0.to_str().unwrap(), + None => panic!("no output generated for {prefix:?} {extension:?}"), + }; + if is_dylib(path_to_add) { + let candidate = format!("{path_to_add}.lib"); + let candidate = PathBuf::from(candidate); + if candidate.exists() { + deps.push((candidate, DependencyType::Target)); + } + } + deps.push((path_to_add.into(), DependencyType::Target)); + } + + deps.extend(additional_target_deps); + deps.sort(); + let mut new_contents = Vec::new(); + for (dep, dependency_type) in deps.iter() { + new_contents.extend(match *dependency_type { + DependencyType::Host => b"h", + DependencyType::Target => b"t", + DependencyType::TargetSelfContained => b"s", + }); + new_contents.extend(dep.to_str().unwrap().as_bytes()); + new_contents.extend(b"\0"); + } + t!(fs::write(&stamp, &new_contents)); + deps.into_iter().map(|(d, _)| d).collect() +} + +pub fn stream_cargo( + builder: &Builder<'_>, + cargo: Cargo, + tail_args: Vec, + cb: &mut dyn FnMut(CargoMessage<'_>), +) -> bool { + let mut cargo = Command::from(cargo); + if builder.config.dry_run() { + return true; + } + // Instruct Cargo to give us json messages on stdout, critically leaving + // stderr as piped so we can get those pretty colors. + let mut message_format = if builder.config.json_output { + String::from("json") + } else { + String::from("json-render-diagnostics") + }; + if let Some(s) = &builder.config.rustc_error_format { + message_format.push_str(",json-diagnostic-"); + message_format.push_str(s); + } + cargo.arg("--message-format").arg(message_format).stdout(Stdio::piped()); + + for arg in tail_args { + cargo.arg(arg); + } + + builder.verbose(&format!("running: {cargo:?}")); + let mut child = match cargo.spawn() { + Ok(child) => child, + Err(e) => panic!("failed to execute command: {cargo:?}\nERROR: {e}"), + }; + + // Spawn Cargo slurping up its JSON output. We'll start building up the + // `deps` array of all files it generated along with a `toplevel` array of + // files we need to probe for later. + let stdout = BufReader::new(child.stdout.take().unwrap()); + for line in stdout.lines() { + let line = t!(line); + match serde_json::from_str::>(&line) { + Ok(msg) => { + if builder.config.json_output { + // Forward JSON to stdout. + println!("{line}"); + } + cb(msg) + } + // If this was informational, just print it out and continue + Err(_) => println!("{line}"), + } + } + + // Make sure Cargo actually succeeded after we read all of its stdout. + let status = t!(child.wait()); + if builder.is_verbose() && !status.success() { + eprintln!( + "command did not execute successfully: {cargo:?}\n\ + expected success, got: {status}" + ); + } + status.success() +} + +#[derive(Deserialize)] +pub struct CargoTarget<'a> { + crate_types: Vec>, +} + +#[derive(Deserialize)] +#[serde(tag = "reason", rename_all = "kebab-case")] +pub enum CargoMessage<'a> { + CompilerArtifact { + package_id: Cow<'a, str>, + features: Vec>, + filenames: Vec>, + target: CargoTarget<'a>, + }, + BuildScriptExecuted { + package_id: Cow<'a, str>, + }, + BuildFinished { + success: bool, + }, +} + +pub fn strip_debug(builder: &Builder<'_>, target: TargetSelection, path: &Path) { + // FIXME: to make things simpler for now, limit this to the host and target where we know + // `strip -g` is both available and will fix the issue, i.e. on a x64 linux host that is not + // cross-compiling. Expand this to other appropriate targets in the future. + if target != "x86_64-unknown-linux-gnu" || target != builder.config.build || !path.exists() { + return; + } + + let previous_mtime = FileTime::from_last_modification_time(&path.metadata().unwrap()); + // NOTE: `output` will propagate any errors here. + output(Command::new("strip").arg("--strip-debug").arg(path)); + + // After running `strip`, we have to set the file modification time to what it was before, + // otherwise we risk Cargo invalidating its fingerprint and rebuilding the world next time + // bootstrap is invoked. + // + // An example of this is if we run this on librustc_driver.so. In the first invocation: + // - Cargo will build librustc_driver.so (mtime of 1) + // - Cargo will build rustc-main (mtime of 2) + // - Bootstrap will strip librustc_driver.so (changing the mtime to 3). + // + // In the second invocation of bootstrap, Cargo will see that the mtime of librustc_driver.so + // is greater than the mtime of rustc-main, and will rebuild rustc-main. That will then cause + // everything else (standard library, future stages...) to be rebuilt. + t!(filetime::set_file_mtime(path, previous_mtime)); +} diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs new file mode 100644 index 000000000..c485481b9 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -0,0 +1,2366 @@ +//! Implementation of the various distribution aspects of the compiler. +//! +//! This module is responsible for creating tarballs of the standard library, +//! compiler, and documentation. This ends up being what we distribute to +//! everyone as well. +//! +//! No tarball is actually created literally in this file, but rather we shell +//! out to `rust-installer` still. This may one day be replaced with bits and +//! pieces of `rustup.rs`! + +use std::collections::HashSet; +use std::env; +use std::ffi::OsStr; +use std::fs; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use object::read::archive::ArchiveFile; +use object::BinaryFormat; + +use crate::core::build_steps::compile; +use crate::core::build_steps::doc::DocumentationFormat; +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::{self, Tool}; +use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::channel; +use crate::utils::helpers::{exe, is_dylib, output, t, target_supports_cranelift_backend, timeit}; +use crate::utils::tarball::{GeneratedTarball, OverlayKind, Tarball}; +use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; + +pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { + format!("{}-{}", component, builder.rust_package_vers()) +} + +pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf { + builder.out.join("dist") +} + +pub fn tmpdir(builder: &Builder<'_>) -> PathBuf { + builder.out.join("tmp/dist") +} + +fn should_build_extended_tool(builder: &Builder<'_>, tool: &str) -> bool { + if !builder.config.extended { + return false; + } + builder.config.tools.as_ref().map_or(true, |tools| tools.contains(tool)) +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Docs { + pub host: TargetSelection, +} + +impl Step for Docs { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.docs; + run.alias("rust-docs").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Docs { host: run.target }); + } + + /// Builds the `rust-docs` installer component. + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + builder.default_doc(&[]); + + let dest = "share/doc/rust/html"; + + let mut tarball = Tarball::new(builder, "rust-docs", &host.triple); + tarball.set_product_name("Rust Documentation"); + tarball.add_bulk_dir(&builder.doc_out(host), dest); + tarball.add_file(&builder.src.join("src/doc/robots.txt"), dest, 0o644); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct JsonDocs { + pub host: TargetSelection, +} + +impl Step for JsonDocs { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.docs; + run.alias("rust-docs-json").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(JsonDocs { host: run.target }); + } + + /// Builds the `rust-docs-json` installer component. + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + builder.ensure(crate::core::build_steps::doc::Std::new( + builder.top_stage, + host, + builder, + DocumentationFormat::JSON, + )); + + let dest = "share/doc/rust/json"; + + let mut tarball = Tarball::new(builder, "rust-docs-json", &host.triple); + tarball.set_product_name("Rust Documentation In JSON Format"); + tarball.is_preview(true); + tarball.add_bulk_dir(&builder.json_doc_out(host), dest); + Some(tarball.generate()) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcDocs { + pub host: TargetSelection, +} + +impl Step for RustcDocs { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("rustc-docs").default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcDocs { host: run.target }); + } + + /// Builds the `rustc-docs` installer component. + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + builder.default_doc(&[]); + + let mut tarball = Tarball::new(builder, "rustc-docs", &host.triple); + tarball.set_product_name("Rustc Documentation"); + tarball.add_bulk_dir(&builder.compiler_doc_out(host), "share/doc/rust/html/rustc"); + Some(tarball.generate()) + } +} + +fn find_files(files: &[&str], path: &[PathBuf]) -> Vec { + let mut found = Vec::with_capacity(files.len()); + + for file in files { + let file_path = path.iter().map(|dir| dir.join(file)).find(|p| p.exists()); + + if let Some(file_path) = file_path { + found.push(file_path); + } else { + panic!("Could not find '{file}' in {path:?}"); + } + } + + found +} + +fn make_win_dist( + rust_root: &Path, + plat_root: &Path, + target: TargetSelection, + builder: &Builder<'_>, +) { + if builder.config.dry_run() { + return; + } + + //Ask gcc where it keeps its stuff + let mut cmd = Command::new(builder.cc(target)); + cmd.arg("-print-search-dirs"); + let gcc_out = output(&mut cmd); + + let mut bin_path: Vec<_> = env::split_paths(&env::var_os("PATH").unwrap_or_default()).collect(); + let mut lib_path = Vec::new(); + + for line in gcc_out.lines() { + let idx = line.find(':').unwrap(); + let key = &line[..idx]; + let trim_chars: &[_] = &[' ', '=']; + let value = env::split_paths(line[(idx + 1)..].trim_start_matches(trim_chars)); + + if key == "programs" { + bin_path.extend(value); + } else if key == "libraries" { + lib_path.extend(value); + } + } + + let compiler = if target == "i686-pc-windows-gnu" { + "i686-w64-mingw32-gcc.exe" + } else if target == "x86_64-pc-windows-gnu" { + "x86_64-w64-mingw32-gcc.exe" + } else { + "gcc.exe" + }; + let target_tools = [compiler, "ld.exe", "dlltool.exe", "libwinpthread-1.dll"]; + let mut rustc_dlls = vec!["libwinpthread-1.dll"]; + if target.starts_with("i686-") { + rustc_dlls.push("libgcc_s_dw2-1.dll"); + } else { + rustc_dlls.push("libgcc_s_seh-1.dll"); + } + + // Libraries necessary to link the windows-gnu toolchains. + // System libraries will be preferred if they are available (see #67429). + let target_libs = [ + //MinGW libs + "libgcc.a", + "libgcc_eh.a", + "libgcc_s.a", + "libm.a", + "libmingw32.a", + "libmingwex.a", + "libstdc++.a", + "libiconv.a", + "libmoldname.a", + "libpthread.a", + //Windows import libs + //This should contain only the set of libraries necessary to link the standard library. + "libadvapi32.a", + "libbcrypt.a", + "libcomctl32.a", + "libcomdlg32.a", + "libcredui.a", + "libcrypt32.a", + "libdbghelp.a", + "libgdi32.a", + "libimagehlp.a", + "libiphlpapi.a", + "libkernel32.a", + "libmsimg32.a", + "libmsvcrt.a", + "libntdll.a", + "libodbc32.a", + "libole32.a", + "liboleaut32.a", + "libopengl32.a", + "libpsapi.a", + "librpcrt4.a", + "libsecur32.a", + "libsetupapi.a", + "libshell32.a", + "libsynchronization.a", + "libuser32.a", + "libuserenv.a", + "libuuid.a", + "libwinhttp.a", + "libwinmm.a", + "libwinspool.a", + "libws2_32.a", + "libwsock32.a", + ]; + + //Find mingw artifacts we want to bundle + let target_tools = find_files(&target_tools, &bin_path); + let rustc_dlls = find_files(&rustc_dlls, &bin_path); + let target_libs = find_files(&target_libs, &lib_path); + + // Copy runtime dlls next to rustc.exe + let dist_bin_dir = rust_root.join("bin/"); + fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed"); + for src in rustc_dlls { + builder.copy_to_folder(&src, &dist_bin_dir); + } + + //Copy platform tools to platform-specific bin directory + let target_bin_dir = plat_root + .join("lib") + .join("rustlib") + .join(target.triple) + .join("bin") + .join("self-contained"); + fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed"); + for src in target_tools { + builder.copy_to_folder(&src, &target_bin_dir); + } + + // Warn windows-gnu users that the bundled GCC cannot compile C files + builder.create( + &target_bin_dir.join("GCC-WARNING.txt"), + "gcc.exe contained in this folder cannot be used for compiling C files - it is only \ + used as a linker. In order to be able to compile projects containing C code use \ + the GCC provided by MinGW or Cygwin.", + ); + + //Copy platform libs to platform-specific lib directory + let target_lib_dir = plat_root + .join("lib") + .join("rustlib") + .join(target.triple) + .join("lib") + .join("self-contained"); + fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed"); + for src in target_libs { + builder.copy_to_folder(&src, &target_lib_dir); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Mingw { + pub host: TargetSelection, +} + +impl Step for Mingw { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-mingw") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Mingw { host: run.target }); + } + + /// Builds the `rust-mingw` installer component. + /// + /// This contains all the bits and pieces to run the MinGW Windows targets + /// without any extra installed software (e.g., we bundle gcc, libraries, etc). + fn run(self, builder: &Builder<'_>) -> Option { + let host = self.host; + if !host.ends_with("pc-windows-gnu") || !builder.config.dist_include_mingw_linker { + return None; + } + + let mut tarball = Tarball::new(builder, "rust-mingw", &host.triple); + tarball.set_product_name("Rust MinGW"); + + // The first argument is a "temporary directory" which is just + // thrown away (this contains the runtime DLLs included in the rustc package + // above) and the second argument is where to place all the MinGW components + // (which is what we want). + make_win_dist(&tmpdir(builder), tarball.image_dir(), host, &builder); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rustc { + pub compiler: Compiler, +} + +impl Step for Rustc { + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rustc") + } + + fn make_run(run: RunConfig<'_>) { + run.builder + .ensure(Rustc { compiler: run.builder.compiler(run.builder.top_stage, run.target) }); + } + + /// Creates the `rustc` installer component. + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let compiler = self.compiler; + let host = self.compiler.host; + + let tarball = Tarball::new(builder, "rustc", &host.triple); + + // Prepare the rustc "image", what will actually end up getting installed + prepare_image(builder, compiler, tarball.image_dir()); + + // On MinGW we've got a few runtime DLL dependencies that we need to + // include. The first argument to this script is where to put these DLLs + // (the image we're creating), and the second argument is a junk directory + // to ignore all other MinGW stuff the script creates. + // + // On 32-bit MinGW we're always including a DLL which needs some extra + // licenses to distribute. On 64-bit MinGW we don't actually distribute + // anything requiring us to distribute a license, but it's likely the + // install will *also* include the rust-mingw package, which also needs + // licenses, so to be safe we just include it here in all MinGW packages. + if host.ends_with("pc-windows-gnu") && builder.config.dist_include_mingw_linker { + make_win_dist(tarball.image_dir(), &tmpdir(builder), host, builder); + tarball.add_dir(builder.src.join("src/etc/third-party"), "share/doc"); + } + + return tarball.generate(); + + fn prepare_image(builder: &Builder<'_>, compiler: Compiler, image: &Path) { + let host = compiler.host; + let src = builder.sysroot(compiler); + + // Copy rustc/rustdoc binaries + t!(fs::create_dir_all(image.join("bin"))); + builder.cp_r(&src.join("bin"), &image.join("bin")); + + if builder + .config + .tools + .as_ref() + .map_or(true, |tools| tools.iter().any(|tool| tool == "rustdoc")) + { + let rustdoc = builder.rustdoc(compiler); + builder.install(&rustdoc, &image.join("bin"), 0o755); + } + + if let Some(ra_proc_macro_srv) = builder.ensure_if_default( + tool::RustAnalyzerProcMacroSrv { + compiler: builder.compiler_for( + compiler.stage, + builder.config.build, + compiler.host, + ), + target: compiler.host, + }, + builder.kind, + ) { + builder.install(&ra_proc_macro_srv, &image.join("libexec"), 0o755); + } + + let libdir_relative = builder.libdir_relative(compiler); + + // Copy runtime DLLs needed by the compiler + if libdir_relative.to_str() != Some("bin") { + let libdir = builder.rustc_libdir(compiler); + for entry in builder.read_dir(&libdir) { + let name = entry.file_name(); + if let Some(s) = name.to_str() { + if is_dylib(s) { + // Don't use custom libdir here because ^lib/ will be resolved again + // with installer + builder.install(&entry.path(), &image.join("lib"), 0o644); + } + } + } + } + + // Copy libLLVM.so to the lib dir as well, if needed. While not + // technically needed by rustc itself it's needed by lots of other + // components like the llvm tools and LLD. LLD is included below and + // tools/LLDB come later, so let's just throw it in the rustc + // component for now. + maybe_install_llvm_runtime(builder, host, image); + + let dst_dir = image.join("lib/rustlib").join(&*host.triple).join("bin"); + t!(fs::create_dir_all(&dst_dir)); + + // Copy over lld if it's there + if builder.config.lld_enabled { + let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin"); + let rust_lld = exe("rust-lld", compiler.host); + builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); + let self_contained_lld_src_dir = src_dir.join("gcc-ld"); + let self_contained_lld_dst_dir = dst_dir.join("gcc-ld"); + t!(fs::create_dir(&self_contained_lld_dst_dir)); + for name in crate::LLD_FILE_NAMES { + let exe_name = exe(name, compiler.host); + builder.copy( + &self_contained_lld_src_dir.join(&exe_name), + &self_contained_lld_dst_dir.join(&exe_name), + ); + } + } + + // Man pages + t!(fs::create_dir_all(image.join("share/man/man1"))); + let man_src = builder.src.join("src/doc/man"); + let man_dst = image.join("share/man/man1"); + + // don't use our `bootstrap::{copy, cp_r}`, because those try + // to hardlink, and we don't want to edit the source templates + for file_entry in builder.read_dir(&man_src) { + let page_src = file_entry.path(); + let page_dst = man_dst.join(file_entry.file_name()); + let src_text = t!(std::fs::read_to_string(&page_src)); + let new_text = src_text.replace("", &builder.version); + t!(std::fs::write(&page_dst, &new_text)); + t!(fs::copy(&page_src, &page_dst)); + } + + // Debugger scripts + builder + .ensure(DebuggerScripts { sysroot: INTERNER.intern_path(image.to_owned()), host }); + + // Misc license info + let cp = |file: &str| { + builder.install(&builder.src.join(file), &image.join("share/doc/rust"), 0o644); + }; + cp("COPYRIGHT"); + cp("LICENSE-APACHE"); + cp("LICENSE-MIT"); + cp("README.md"); + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct DebuggerScripts { + pub sysroot: Interned, + pub host: TargetSelection, +} + +impl Step for DebuggerScripts { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Copies debugger scripts for `target` into the `sysroot` specified. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let sysroot = self.sysroot; + let dst = sysroot.join("lib/rustlib/etc"); + t!(fs::create_dir_all(&dst)); + let cp_debugger_script = |file: &str| { + builder.install(&builder.src.join("src/etc/").join(file), &dst, 0o644); + }; + if host.contains("windows-msvc") { + // windbg debugger scripts + builder.install( + &builder.src.join("src/etc/rust-windbg.cmd"), + &sysroot.join("bin"), + 0o755, + ); + + cp_debugger_script("natvis/intrinsic.natvis"); + cp_debugger_script("natvis/liballoc.natvis"); + cp_debugger_script("natvis/libcore.natvis"); + cp_debugger_script("natvis/libstd.natvis"); + } else { + cp_debugger_script("rust_types.py"); + + // gdb debugger scripts + builder.install(&builder.src.join("src/etc/rust-gdb"), &sysroot.join("bin"), 0o755); + builder.install(&builder.src.join("src/etc/rust-gdbgui"), &sysroot.join("bin"), 0o755); + + cp_debugger_script("gdb_load_rust_pretty_printers.py"); + cp_debugger_script("gdb_lookup.py"); + cp_debugger_script("gdb_providers.py"); + + // lldb debugger scripts + builder.install(&builder.src.join("src/etc/rust-lldb"), &sysroot.join("bin"), 0o755); + + cp_debugger_script("lldb_lookup.py"); + cp_debugger_script("lldb_providers.py"); + cp_debugger_script("lldb_commands") + } + } +} + +fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { + // The only true set of target libraries came from the build triple, so + // let's reduce redundant work by only producing archives from that host. + if compiler.host != builder.config.build { + builder.info("\tskipping, not a build host"); + true + } else { + false + } +} + +/// Check that all objects in rlibs for UEFI targets are COFF. This +/// ensures that the C compiler isn't producing ELF objects, which would +/// not link correctly with the COFF objects. +fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp: &Path) { + if !target.ends_with("-uefi") { + return; + } + + for (path, _) in builder.read_stamp_file(stamp) { + if path.extension() != Some(OsStr::new("rlib")) { + continue; + } + + let data = t!(fs::read(&path)); + let data = data.as_slice(); + let archive = t!(ArchiveFile::parse(data)); + for member in archive.members() { + let member = t!(member); + let member_data = t!(member.data(data)); + + let is_coff = match object::File::parse(member_data) { + Ok(member_file) => member_file.format() == BinaryFormat::Coff, + Err(_) => false, + }; + + if !is_coff { + let member_name = String::from_utf8_lossy(member.name()); + panic!("member {} in {} is not COFF", member_name, path.display()); + } + } + } +} + +/// Copy stamped files into an image's `target/lib` directory. +fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) { + let dst = image.join("lib/rustlib").join(target.triple).join("lib"); + let self_contained_dst = dst.join("self-contained"); + t!(fs::create_dir_all(&dst)); + t!(fs::create_dir_all(&self_contained_dst)); + for (path, dependency_type) in builder.read_stamp_file(stamp) { + if dependency_type == DependencyType::TargetSelfContained { + builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap())); + } else if dependency_type == DependencyType::Target || builder.config.build == target { + builder.copy(&path, &dst.join(path.file_name().unwrap())); + } + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Std { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Std { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-std") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Std { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + if skip_host_target_lib(builder, compiler) { + return None; + } + + builder.ensure(compile::Std::new(compiler, target)); + + let mut tarball = Tarball::new(builder, "rust-std", &target.triple); + tarball.include_target_in_component_name(true); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::libstd_stamp(builder, compiler_to_use, target); + verify_uefi_rlib_format(builder, target, &stamp); + copy_target_libs(builder, target, &tarball.image_dir(), &stamp); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcDev { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustcDev { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rustc-dev") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcDev { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + if skip_host_target_lib(builder, compiler) { + return None; + } + + builder.ensure(compile::Rustc::new(compiler, target)); + + let tarball = Tarball::new(builder, "rustc-dev", &target.triple); + + let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); + let stamp = compile::librustc_stamp(builder, compiler_to_use, target); + copy_target_libs(builder, target, tarball.image_dir(), &stamp); + + let src_files = &["Cargo.lock"]; + // This is the reduced set of paths which will become the rustc-dev component + // (essentially the compiler crates and all of their path dependencies). + copy_src_dirs( + builder, + &builder.src, + &["compiler"], + &[], + &tarball.image_dir().join("lib/rustlib/rustc-src/rust"), + ); + for file in src_files { + tarball.add_file(builder.src.join(file), "lib/rustlib/rustc-src/rust", 0o644); + } + + Some(tarball.generate()) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Analysis { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Analysis { + type Output = Option; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "analysis"); + run.alias("rust-analysis").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Analysis { + // Find the actual compiler (handling the full bootstrap option) which + // produced the save-analysis data because that data isn't copied + // through the sysroot uplifting. + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + /// Creates a tarball of (degenerate) save-analysis metadata, if available. + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + if compiler.host != builder.config.build { + return None; + } + + let src = builder + .stage_out(compiler, Mode::Std) + .join(target.triple) + .join(builder.cargo_dir()) + .join("deps") + .join("save-analysis"); + + // Write a file indicating that this component has been removed. + t!(std::fs::create_dir_all(&src)); + let mut removed = src.clone(); + removed.push("removed.json"); + let mut f = t!(std::fs::File::create(removed)); + t!(write!(f, r#"{{ "warning": "The `rust-analysis` component has been removed." }}"#)); + + let mut tarball = Tarball::new(builder, "rust-analysis", &target.triple); + tarball.include_target_in_component_name(true); + tarball.add_dir(src, format!("lib/rustlib/{}/analysis", target.triple)); + Some(tarball.generate()) + } +} + +/// Use the `builder` to make a filtered copy of `base`/X for X in (`src_dirs` - `exclude_dirs`) to +/// `dst_dir`. +fn copy_src_dirs( + builder: &Builder<'_>, + base: &Path, + src_dirs: &[&str], + exclude_dirs: &[&str], + dst_dir: &Path, +) { + fn filter_fn(exclude_dirs: &[&str], dir: &str, path: &Path) -> bool { + let spath = match path.to_str() { + Some(path) => path, + None => return false, + }; + if spath.ends_with('~') || spath.ends_with(".pyc") { + return false; + } + + const LLVM_PROJECTS: &[&str] = &[ + "llvm-project/clang", + "llvm-project\\clang", + "llvm-project/libunwind", + "llvm-project\\libunwind", + "llvm-project/lld", + "llvm-project\\lld", + "llvm-project/lldb", + "llvm-project\\lldb", + "llvm-project/llvm", + "llvm-project\\llvm", + "llvm-project/compiler-rt", + "llvm-project\\compiler-rt", + "llvm-project/cmake", + "llvm-project\\cmake", + "llvm-project/runtimes", + "llvm-project\\runtimes", + ]; + if spath.contains("llvm-project") + && !spath.ends_with("llvm-project") + && !LLVM_PROJECTS.iter().any(|path| spath.contains(path)) + { + return false; + } + + const LLVM_TEST: &[&str] = &["llvm-project/llvm/test", "llvm-project\\llvm\\test"]; + if LLVM_TEST.iter().any(|path| spath.contains(path)) + && (spath.ends_with(".ll") || spath.ends_with(".td") || spath.ends_with(".s")) + { + return false; + } + + let full_path = Path::new(dir).join(path); + if exclude_dirs.iter().any(|excl| full_path == Path::new(excl)) { + return false; + } + + let excludes = [ + "CVS", + "RCS", + "SCCS", + ".git", + ".gitignore", + ".gitmodules", + ".gitattributes", + ".cvsignore", + ".svn", + ".arch-ids", + "{arch}", + "=RELEASE-ID", + "=meta-update", + "=update", + ".bzr", + ".bzrignore", + ".bzrtags", + ".hg", + ".hgignore", + ".hgrags", + "_darcs", + ]; + !path.iter().map(|s| s.to_str().unwrap()).any(|s| excludes.contains(&s)) + } + + // Copy the directories using our filter + for item in src_dirs { + let dst = &dst_dir.join(item); + t!(fs::create_dir_all(dst)); + builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path)); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Src; + +impl Step for Src { + /// The output path of the src installer tarball + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-src") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Src); + } + + /// Creates the `rust-src` installer component + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + if !builder.config.dry_run() { + builder.update_submodule(&Path::new("src/llvm-project")); + } + + let tarball = Tarball::new_targetless(builder, "rust-src"); + + // A lot of tools expect the rust-src component to be entirely in this directory, so if you + // change that (e.g. by adding another directory `lib/rustlib/src/foo` or + // `lib/rustlib/src/rust/foo`), you will need to go around hunting for implicit assumptions + // and fix them... + // + // NOTE: if you update the paths here, you also should update the "virtual" path + // translation code in `imported_source_files` in `src/librustc_metadata/rmeta/decoder.rs` + let dst_src = tarball.image_dir().join("lib/rustlib/src/rust"); + + let src_files = ["Cargo.lock"]; + // This is the reduced set of paths which will become the rust-src component + // (essentially libstd and all of its path dependencies). + copy_src_dirs( + builder, + &builder.src, + &["library", "src/llvm-project/libunwind"], + &[ + // not needed and contains symlinks which rustup currently + // chokes on when unpacking. + "library/backtrace/crates", + // these are 30MB combined and aren't necessary for building + // the standard library. + "library/stdarch/Cargo.toml", + "library/stdarch/crates/stdarch-verify", + "library/stdarch/crates/intrinsic-test", + ], + &dst_src, + ); + for file in src_files.iter() { + builder.copy(&builder.src.join(file), &dst_src.join(file)); + } + + tarball.generate() + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct PlainSourceTarball; + +impl Step for PlainSourceTarball { + /// Produces the location of the tarball generated + type Output = GeneratedTarball; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("rustc-src").default_condition(builder.config.rust_dist_src) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(PlainSourceTarball); + } + + /// Creates the plain source tarball + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + // NOTE: This is a strange component in a lot of ways. It uses `src` as the target, which + // means neither rustup nor rustup-toolchain-install-master know how to download it. + // It also contains symbolic links, unlike other any other dist tarball. + // It's used for distros building rustc from source in a pre-vendored environment. + let mut tarball = Tarball::new(builder, "rustc", "src"); + tarball.permit_symlinks(true); + let plain_dst_src = tarball.image_dir(); + + // This is the set of root paths which will become part of the source package + let src_files = [ + "COPYRIGHT", + "LICENSE-APACHE", + "LICENSE-MIT", + "CONTRIBUTING.md", + "README.md", + "RELEASES.md", + "configure", + "x.py", + "config.example.toml", + "Cargo.toml", + "Cargo.lock", + ".gitmodules", + ]; + let src_dirs = ["src", "compiler", "library", "tests"]; + + copy_src_dirs(builder, &builder.src, &src_dirs, &[], &plain_dst_src); + + // Copy the files normally + for item in &src_files { + builder.copy(&builder.src.join(item), &plain_dst_src.join(item)); + } + + // Create the version file + builder.create(&plain_dst_src.join("version"), &builder.rust_version()); + if let Some(info) = builder.rust_info().info() { + channel::write_commit_hash_file(&plain_dst_src, &info.sha); + channel::write_commit_info_file(&plain_dst_src, info); + } + + // If we're building from git or tarball sources, we need to vendor + // a complete distribution. + if builder.rust_info().is_managed_git_subrepository() + || builder.rust_info().is_from_tarball() + { + if builder.rust_info().is_managed_git_subrepository() { + // Ensure we have the submodules checked out. + builder.update_submodule(Path::new("src/tools/cargo")); + } + + // Vendor all Cargo dependencies + let mut cmd = Command::new(&builder.initial_cargo); + cmd.arg("vendor") + .arg("--sync") + .arg(builder.src.join("./src/tools/cargo/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./src/tools/rust-analyzer/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./compiler/rustc_codegen_cranelift/Cargo.toml")) + .arg("--sync") + .arg(builder.src.join("./src/bootstrap/Cargo.toml")) + // Will read the libstd Cargo.toml + // which uses the unstable `public-dependency` feature. + .env("RUSTC_BOOTSTRAP", "1") + .current_dir(&plain_dst_src); + + let config = if !builder.config.dry_run() { + t!(String::from_utf8(t!(cmd.output()).stdout)) + } else { + String::new() + }; + + let cargo_config_dir = plain_dst_src.join(".cargo"); + builder.create_dir(&cargo_config_dir); + builder.create(&cargo_config_dir.join("config.toml"), &config); + } + + tarball.bare() + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Cargo { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Cargo { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "cargo"); + run.alias("cargo").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let cargo = builder.ensure(tool::Cargo { compiler, target }); + let src = builder.src.join("src/tools/cargo"); + let etc = src.join("src/etc"); + + // Prepare the image directory + let mut tarball = Tarball::new(builder, "cargo", &target.triple); + tarball.set_overlay(OverlayKind::Cargo); + + tarball.add_file(&cargo, "bin", 0o755); + tarball.add_file(etc.join("_cargo"), "share/zsh/site-functions", 0o644); + tarball.add_renamed_file(etc.join("cargo.bashcomp.sh"), "etc/bash_completion.d", "cargo"); + tarball.add_dir(etc.join("man"), "share/man/man1"); + tarball.add_legal_and_readme_to("share/doc/cargo"); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rls { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Rls { + type Output = Option; + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "rls"); + run.alias("rls").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rls { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let rls = builder + .ensure(tool::Rls { compiler, target, extra_features: Vec::new() }) + .expect("rls expected to build"); + + let mut tarball = Tarball::new(builder, "rls", &target.triple); + tarball.set_overlay(OverlayKind::RLS); + tarball.is_preview(true); + tarball.add_file(rls, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rls"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "rust-analyzer"); + run.alias("rust-analyzer").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let rust_analyzer = builder + .ensure(tool::RustAnalyzer { compiler, target }) + .expect("rust-analyzer always builds"); + + let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple); + tarball.set_overlay(OverlayKind::RustAnalyzer); + tarball.is_preview(true); + tarball.add_file(rust_analyzer, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-analyzer"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Clippy { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Clippy { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "clippy"); + run.alias("clippy").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Clippy { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + // Prepare the image directory + // We expect clippy to build, because we've exited this step above if tool + // state for clippy isn't testing. + let clippy = builder + .ensure(tool::Clippy { compiler, target, extra_features: Vec::new() }) + .expect("clippy expected to build - essential tool"); + let cargoclippy = builder + .ensure(tool::CargoClippy { compiler, target, extra_features: Vec::new() }) + .expect("clippy expected to build - essential tool"); + + let mut tarball = Tarball::new(builder, "clippy", &target.triple); + tarball.set_overlay(OverlayKind::Clippy); + tarball.is_preview(true); + tarball.add_file(clippy, "bin", 0o755); + tarball.add_file(cargoclippy, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/clippy"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Miri { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Miri { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "miri"); + run.alias("miri").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Miri { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + // This prevents miri from being built for "dist" or "install" + // on the stable/beta channels. It is a nightly-only tool and should + // not be included. + if !builder.build.unstable_features() { + return None; + } + let compiler = self.compiler; + let target = self.target; + + let miri = builder.ensure(tool::Miri { compiler, target, extra_features: Vec::new() })?; + let cargomiri = + builder.ensure(tool::CargoMiri { compiler, target, extra_features: Vec::new() })?; + + let mut tarball = Tarball::new(builder, "miri", &target.triple); + tarball.set_overlay(OverlayKind::Miri); + tarball.is_preview(true); + tarball.add_file(miri, "bin", 0o755); + tarball.add_file(cargomiri, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/miri"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct CodegenBackend { + pub compiler: Compiler, + pub backend: Interned, +} + +impl Step for CodegenBackend { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("compiler/rustc_codegen_cranelift") + } + + fn make_run(run: RunConfig<'_>) { + for &backend in &run.builder.config.rust_codegen_backends { + if backend == "llvm" { + continue; // Already built as part of rustc + } + + run.builder.ensure(CodegenBackend { + compiler: run.builder.compiler(run.builder.top_stage, run.target), + backend, + }); + } + } + + fn run(self, builder: &Builder<'_>) -> Option { + if builder.config.dry_run() { + return None; + } + + // This prevents rustc_codegen_cranelift from being built for "dist" + // or "install" on the stable/beta channels. It is not yet stable and + // should not be included. + if !builder.build.unstable_features() { + return None; + } + + if !builder.config.rust_codegen_backends.contains(&self.backend) { + return None; + } + + if self.backend == "cranelift" { + if !target_supports_cranelift_backend(self.compiler.host) { + builder.info("target not supported by rustc_codegen_cranelift. skipping"); + return None; + } + + if self.compiler.host.contains("windows") { + builder.info( + "dist currently disabled for windows by rustc_codegen_cranelift. skipping", + ); + return None; + } + } + + let compiler = self.compiler; + let backend = self.backend; + + let mut tarball = + Tarball::new(builder, &format!("rustc-codegen-{}", backend), &compiler.host.triple); + if backend == "cranelift" { + tarball.set_overlay(OverlayKind::RustcCodegenCranelift); + } else { + panic!("Unknown backend rustc_codegen_{}", backend); + } + tarball.is_preview(true); + tarball.add_legal_and_readme_to(format!("share/doc/rustc_codegen_{}", backend)); + + let src = builder.sysroot(compiler); + let backends_src = builder.sysroot_codegen_backends(compiler); + let backends_rel = backends_src + .strip_prefix(&src) + .unwrap() + .strip_prefix(builder.sysroot_libdir_relative(compiler)) + .unwrap(); + // Don't use custom libdir here because ^lib/ will be resolved again with installer + let backends_dst = PathBuf::from("lib").join(&backends_rel); + + let backend_name = format!("rustc_codegen_{}", backend); + let mut found_backend = false; + for backend in fs::read_dir(&backends_src).unwrap() { + let file_name = backend.unwrap().file_name(); + if file_name.to_str().unwrap().contains(&backend_name) { + tarball.add_file(backends_src.join(file_name), &backends_dst, 0o644); + found_backend = true; + } + } + assert!(found_backend); + + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rustfmt { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Rustfmt { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "rustfmt"); + run.alias("rustfmt").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustfmt { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + let rustfmt = builder + .ensure(tool::Rustfmt { compiler, target, extra_features: Vec::new() }) + .expect("rustfmt expected to build - essential tool"); + let cargofmt = builder + .ensure(tool::Cargofmt { compiler, target, extra_features: Vec::new() }) + .expect("cargo fmt expected to build - essential tool"); + let mut tarball = Tarball::new(builder, "rustfmt", &target.triple); + tarball.set_overlay(OverlayKind::Rustfmt); + tarball.is_preview(true); + tarball.add_file(rustfmt, "bin", 0o755); + tarball.add_file(cargofmt, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rustfmt"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustDemangler { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustDemangler { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // While other tools use `should_build_extended_tool` to decide whether to be run by + // default or not, `rust-demangler` must be build when *either* it's enabled as a tool like + // the other ones or if `profiler = true`. Because we don't know the target at this stage + // we run the step by default when only `extended = true`, and decide whether to actually + // run it or not later. + let default = run.builder.config.extended; + run.alias("rust-demangler").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDemangler { + compiler: run.builder.compiler_for( + run.builder.top_stage, + run.builder.config.build, + run.target, + ), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + + // Only build this extended tool if explicitly included in `tools`, or if `profiler = true` + let condition = should_build_extended_tool(builder, "rust-demangler") + || builder.config.profiler_enabled(target); + if builder.config.extended && !condition { + return None; + } + + let rust_demangler = builder + .ensure(tool::RustDemangler { compiler, target, extra_features: Vec::new() }) + .expect("rust-demangler expected to build - in-tree tool"); + + // Prepare the image directory + let mut tarball = Tarball::new(builder, "rust-demangler", &target.triple); + tarball.set_overlay(OverlayKind::RustDemangler); + tarball.is_preview(true); + tarball.add_file(&rust_demangler, "bin", 0o755); + tarball.add_legal_and_readme_to("share/doc/rust-demangler"); + Some(tarball.generate()) + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Extended { + stage: u32, + host: TargetSelection, + target: TargetSelection, +} + +impl Step for Extended { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.alias("extended").default_condition(builder.config.extended) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Extended { + stage: run.builder.top_stage, + host: run.builder.config.build, + target: run.target, + }); + } + + /// Creates a combined installer for the specified target in the provided stage. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let stage = self.stage; + let compiler = builder.compiler_for(self.stage, self.host, self.target); + + builder.info(&format!("Dist extended stage{} ({})", compiler.stage, target)); + + let mut tarballs = Vec::new(); + let mut built_tools = HashSet::new(); + macro_rules! add_component { + ($name:expr => $step:expr) => { + if let Some(tarball) = builder.ensure_if_default($step, Kind::Dist) { + tarballs.push(tarball); + built_tools.insert($name); + } + }; + } + + // When rust-std package split from rustc, we needed to ensure that during + // upgrades rustc was upgraded before rust-std. To avoid rustc clobbering + // the std files during uninstall. To do this ensure that rustc comes + // before rust-std in the list below. + tarballs.push(builder.ensure(Rustc { compiler: builder.compiler(stage, target) })); + tarballs.push(builder.ensure(Std { compiler, target }).expect("missing std")); + + if target.ends_with("windows-gnu") { + tarballs.push(builder.ensure(Mingw { host: target }).expect("missing mingw")); + } + + add_component!("rust-docs" => Docs { host: target }); + add_component!("rust-json-docs" => JsonDocs { host: target }); + add_component!("rust-demangler"=> RustDemangler { compiler, target }); + add_component!("cargo" => Cargo { compiler, target }); + add_component!("rustfmt" => Rustfmt { compiler, target }); + add_component!("rls" => Rls { compiler, target }); + add_component!("rust-analyzer" => RustAnalyzer { compiler, target }); + add_component!("llvm-components" => LlvmTools { target }); + add_component!("clippy" => Clippy { compiler, target }); + add_component!("miri" => Miri { compiler, target }); + add_component!("analysis" => Analysis { compiler, target }); + add_component!("rustc-codegen-cranelift" => CodegenBackend { + compiler: builder.compiler(stage, target), + backend: INTERNER.intern_str("cranelift"), + }); + + let etc = builder.src.join("src/etc/installer"); + + // Avoid producing tarballs during a dry run. + if builder.config.dry_run() { + return; + } + + let tarball = Tarball::new(builder, "rust", &target.triple); + let generated = tarball.combine(&tarballs); + + let tmp = tmpdir(builder).join("combined-tarball"); + let work = generated.work_dir(); + + let mut license = String::new(); + license += &builder.read(&builder.src.join("COPYRIGHT")); + license += &builder.read(&builder.src.join("LICENSE-APACHE")); + license += &builder.read(&builder.src.join("LICENSE-MIT")); + license.push('\n'); + license.push('\n'); + + let rtf = r"{\rtf1\ansi\deff0{\fonttbl{\f0\fnil\fcharset0 Arial;}}\nowwrap\fs18"; + let mut rtf = rtf.to_string(); + rtf.push('\n'); + for line in license.lines() { + rtf.push_str(line); + rtf.push_str("\\line "); + } + rtf.push('}'); + + fn filter(contents: &str, marker: &str) -> String { + let start = format!("tool-{marker}-start"); + let end = format!("tool-{marker}-end"); + let mut lines = Vec::new(); + let mut omitted = false; + for line in contents.lines() { + if line.contains(&start) { + omitted = true; + } else if line.contains(&end) { + omitted = false; + } else if !omitted { + lines.push(line); + } + } + + lines.join("\n") + } + + let xform = |p: &Path| { + let mut contents = t!(fs::read_to_string(p)); + for tool in &["rust-demangler", "miri", "rust-docs"] { + if !built_tools.contains(tool) { + contents = filter(&contents, tool); + } + } + let ret = tmp.join(p.file_name().unwrap()); + t!(fs::write(&ret, &contents)); + ret + }; + + if target.contains("apple-darwin") { + builder.info("building pkg installer"); + let pkg = tmp.join("pkg"); + let _ = fs::remove_dir_all(&pkg); + + let pkgbuild = |component: &str| { + let mut cmd = Command::new("pkgbuild"); + cmd.arg("--identifier") + .arg(format!("org.rust-lang.{}", component)) + .arg("--scripts") + .arg(pkg.join(component)) + .arg("--nopayload") + .arg(pkg.join(component).with_extension("pkg")); + builder.run(&mut cmd); + }; + + let prepare = |name: &str| { + builder.create_dir(&pkg.join(name)); + builder.cp_r( + &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)), + &pkg.join(name), + ); + builder.install(&etc.join("pkg/postinstall"), &pkg.join(name), 0o755); + pkgbuild(name); + }; + prepare("rustc"); + prepare("cargo"); + prepare("rust-std"); + prepare("rust-analysis"); + prepare("clippy"); + prepare("rust-analyzer"); + for tool in &["rust-docs", "rust-demangler", "miri", "rustc-codegen-cranelift"] { + if built_tools.contains(tool) { + prepare(tool); + } + } + // create an 'uninstall' package + builder.install(&etc.join("pkg/postinstall"), &pkg.join("uninstall"), 0o755); + pkgbuild("uninstall"); + + builder.create_dir(&pkg.join("res")); + builder.create(&pkg.join("res/LICENSE.txt"), &license); + builder.install(&etc.join("gfx/rust-logo.png"), &pkg.join("res"), 0o644); + let mut cmd = Command::new("productbuild"); + cmd.arg("--distribution") + .arg(xform(&etc.join("pkg/Distribution.xml"))) + .arg("--resources") + .arg(pkg.join("res")) + .arg(distdir(builder).join(format!( + "{}-{}.pkg", + pkgname(builder, "rust"), + target.triple + ))) + .arg("--package-path") + .arg(&pkg); + let _time = timeit(builder); + builder.run(&mut cmd); + } + + if target.contains("windows") { + let exe = tmp.join("exe"); + let _ = fs::remove_dir_all(&exe); + + let prepare = |name: &str| { + builder.create_dir(&exe.join(name)); + let dir = if name == "rust-std" || name == "rust-analysis" { + format!("{}-{}", name, target.triple) + } else if name == "rust-analyzer" { + "rust-analyzer-preview".to_string() + } else if name == "clippy" { + "clippy-preview".to_string() + } else if name == "rust-demangler" { + "rust-demangler-preview".to_string() + } else if name == "miri" { + "miri-preview".to_string() + } else if name == "rustc-codegen-cranelift" { + // FIXME add installer support for cg_clif once it is ready to be distributed on + // windows. + unreachable!("cg_clif shouldn't be built for windows"); + } else { + name.to_string() + }; + builder.cp_r( + &work.join(&format!("{}-{}", pkgname(builder, name), target.triple)).join(dir), + &exe.join(name), + ); + builder.remove(&exe.join(name).join("manifest.in")); + }; + prepare("rustc"); + prepare("cargo"); + prepare("rust-analysis"); + prepare("rust-std"); + for tool in &["clippy", "rust-analyzer", "rust-docs", "rust-demangler", "miri"] { + if built_tools.contains(tool) { + prepare(tool); + } + } + if target.ends_with("windows-gnu") { + prepare("rust-mingw"); + } + + builder.install(&etc.join("gfx/rust-logo.ico"), &exe, 0o644); + + // Generate msi installer + let wix_path = env::var_os("WIX") + .expect("`WIX` environment variable must be set for generating MSI installer(s)."); + let wix = PathBuf::from(wix_path); + let heat = wix.join("bin/heat.exe"); + let candle = wix.join("bin/candle.exe"); + let light = wix.join("bin/light.exe"); + + let heat_flags = ["-nologo", "-gg", "-sfrag", "-srd", "-sreg"]; + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rustc") + .args(&heat_flags) + .arg("-cg") + .arg("RustcGroup") + .arg("-dr") + .arg("Rustc") + .arg("-var") + .arg("var.RustcDir") + .arg("-out") + .arg(exe.join("RustcGroup.wxs")), + ); + if built_tools.contains("rust-docs") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-docs") + .args(&heat_flags) + .arg("-cg") + .arg("DocsGroup") + .arg("-dr") + .arg("Docs") + .arg("-var") + .arg("var.DocsDir") + .arg("-out") + .arg(exe.join("DocsGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/squash-components.xsl")), + ); + } + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("cargo") + .args(&heat_flags) + .arg("-cg") + .arg("CargoGroup") + .arg("-dr") + .arg("Cargo") + .arg("-var") + .arg("var.CargoDir") + .arg("-out") + .arg(exe.join("CargoGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-std") + .args(&heat_flags) + .arg("-cg") + .arg("StdGroup") + .arg("-dr") + .arg("Std") + .arg("-var") + .arg("var.StdDir") + .arg("-out") + .arg(exe.join("StdGroup.wxs")), + ); + if built_tools.contains("rust-analyzer") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-analyzer") + .args(&heat_flags) + .arg("-cg") + .arg("RustAnalyzerGroup") + .arg("-dr") + .arg("RustAnalyzer") + .arg("-var") + .arg("var.RustAnalyzerDir") + .arg("-out") + .arg(exe.join("RustAnalyzerGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + if built_tools.contains("clippy") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("clippy") + .args(&heat_flags) + .arg("-cg") + .arg("ClippyGroup") + .arg("-dr") + .arg("Clippy") + .arg("-var") + .arg("var.ClippyDir") + .arg("-out") + .arg(exe.join("ClippyGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + if built_tools.contains("rust-demangler") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-demangler") + .args(&heat_flags) + .arg("-cg") + .arg("RustDemanglerGroup") + .arg("-dr") + .arg("RustDemangler") + .arg("-var") + .arg("var.RustDemanglerDir") + .arg("-out") + .arg(exe.join("RustDemanglerGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + if built_tools.contains("miri") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("miri") + .args(&heat_flags) + .arg("-cg") + .arg("MiriGroup") + .arg("-dr") + .arg("Miri") + .arg("-var") + .arg("var.MiriDir") + .arg("-out") + .arg(exe.join("MiriGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + } + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-analysis") + .args(&heat_flags) + .arg("-cg") + .arg("AnalysisGroup") + .arg("-dr") + .arg("Analysis") + .arg("-var") + .arg("var.AnalysisDir") + .arg("-out") + .arg(exe.join("AnalysisGroup.wxs")) + .arg("-t") + .arg(etc.join("msi/remove-duplicates.xsl")), + ); + if target.ends_with("windows-gnu") { + builder.run( + Command::new(&heat) + .current_dir(&exe) + .arg("dir") + .arg("rust-mingw") + .args(&heat_flags) + .arg("-cg") + .arg("GccGroup") + .arg("-dr") + .arg("Gcc") + .arg("-var") + .arg("var.GccDir") + .arg("-out") + .arg(exe.join("GccGroup.wxs")), + ); + } + + let candle = |input: &Path| { + let output = exe.join(input.file_stem().unwrap()).with_extension("wixobj"); + let arch = if target.contains("x86_64") { "x64" } else { "x86" }; + let mut cmd = Command::new(&candle); + cmd.current_dir(&exe) + .arg("-nologo") + .arg("-dRustcDir=rustc") + .arg("-dCargoDir=cargo") + .arg("-dStdDir=rust-std") + .arg("-dAnalysisDir=rust-analysis") + .arg("-arch") + .arg(&arch) + .arg("-out") + .arg(&output) + .arg(&input); + add_env(builder, &mut cmd, target); + + if built_tools.contains("clippy") { + cmd.arg("-dClippyDir=clippy"); + } + if built_tools.contains("rust-docs") { + cmd.arg("-dDocsDir=rust-docs"); + } + if built_tools.contains("rust-demangler") { + cmd.arg("-dRustDemanglerDir=rust-demangler"); + } + if built_tools.contains("rust-analyzer") { + cmd.arg("-dRustAnalyzerDir=rust-analyzer"); + } + if built_tools.contains("miri") { + cmd.arg("-dMiriDir=miri"); + } + if target.ends_with("windows-gnu") { + cmd.arg("-dGccDir=rust-mingw"); + } + builder.run(&mut cmd); + }; + candle(&xform(&etc.join("msi/rust.wxs"))); + candle(&etc.join("msi/ui.wxs")); + candle(&etc.join("msi/rustwelcomedlg.wxs")); + candle("RustcGroup.wxs".as_ref()); + if built_tools.contains("rust-docs") { + candle("DocsGroup.wxs".as_ref()); + } + candle("CargoGroup.wxs".as_ref()); + candle("StdGroup.wxs".as_ref()); + if built_tools.contains("clippy") { + candle("ClippyGroup.wxs".as_ref()); + } + if built_tools.contains("miri") { + candle("MiriGroup.wxs".as_ref()); + } + if built_tools.contains("rust-demangler") { + candle("RustDemanglerGroup.wxs".as_ref()); + } + if built_tools.contains("rust-analyzer") { + candle("RustAnalyzerGroup.wxs".as_ref()); + } + candle("AnalysisGroup.wxs".as_ref()); + + if target.ends_with("windows-gnu") { + candle("GccGroup.wxs".as_ref()); + } + + builder.create(&exe.join("LICENSE.rtf"), &rtf); + builder.install(&etc.join("gfx/banner.bmp"), &exe, 0o644); + builder.install(&etc.join("gfx/dialogbg.bmp"), &exe, 0o644); + + builder.info(&format!("building `msi` installer with {light:?}")); + let filename = format!("{}-{}.msi", pkgname(builder, "rust"), target.triple); + let mut cmd = Command::new(&light); + cmd.arg("-nologo") + .arg("-ext") + .arg("WixUIExtension") + .arg("-ext") + .arg("WixUtilExtension") + .arg("-out") + .arg(exe.join(&filename)) + .arg("rust.wixobj") + .arg("ui.wixobj") + .arg("rustwelcomedlg.wixobj") + .arg("RustcGroup.wixobj") + .arg("CargoGroup.wixobj") + .arg("StdGroup.wixobj") + .arg("AnalysisGroup.wixobj") + .current_dir(&exe); + + if built_tools.contains("clippy") { + cmd.arg("ClippyGroup.wixobj"); + } + if built_tools.contains("miri") { + cmd.arg("MiriGroup.wixobj"); + } + if built_tools.contains("rust-analyzer") { + cmd.arg("RustAnalyzerGroup.wixobj"); + } + if built_tools.contains("rust-demangler") { + cmd.arg("RustDemanglerGroup.wixobj"); + } + if built_tools.contains("rust-docs") { + cmd.arg("DocsGroup.wixobj"); + } + + if target.ends_with("windows-gnu") { + cmd.arg("GccGroup.wixobj"); + } + // ICE57 wrongly complains about the shortcuts + cmd.arg("-sice:ICE57"); + + let _time = timeit(builder); + builder.run(&mut cmd); + + if !builder.config.dry_run() { + t!(fs::rename(exe.join(&filename), distdir(builder).join(&filename))); + } + } + } +} + +fn add_env(builder: &Builder<'_>, cmd: &mut Command, target: TargetSelection) { + let mut parts = builder.version.split('.'); + cmd.env("CFG_RELEASE_INFO", builder.rust_version()) + .env("CFG_RELEASE_NUM", &builder.version) + .env("CFG_RELEASE", builder.rust_release()) + .env("CFG_VER_MAJOR", parts.next().unwrap()) + .env("CFG_VER_MINOR", parts.next().unwrap()) + .env("CFG_VER_PATCH", parts.next().unwrap()) + .env("CFG_VER_BUILD", "0") // just needed to build + .env("CFG_PACKAGE_VERS", builder.rust_package_vers()) + .env("CFG_PACKAGE_NAME", pkgname(builder, "rust")) + .env("CFG_BUILD", target.triple) + .env("CFG_CHANNEL", &builder.config.channel); + + if target.contains("windows-gnullvm") { + cmd.env("CFG_MINGW", "1").env("CFG_ABI", "LLVM"); + } else if target.contains("windows-gnu") { + cmd.env("CFG_MINGW", "1").env("CFG_ABI", "GNU"); + } else { + cmd.env("CFG_MINGW", "0").env("CFG_ABI", "MSVC"); + } +} + +fn install_llvm_file(builder: &Builder<'_>, source: &Path, destination: &Path) { + if builder.config.dry_run() { + return; + } + + builder.install(&source, destination, 0o644); +} + +/// Maybe add LLVM object files to the given destination lib-dir. Allows either static or dynamic linking. +/// +/// Returns whether the files were actually copied. +fn maybe_install_llvm(builder: &Builder<'_>, target: TargetSelection, dst_libdir: &Path) -> bool { + if let Some(config) = builder.config.target_config.get(&target) { + if config.llvm_config.is_some() && !builder.config.llvm_from_ci { + // If the LLVM was externally provided, then we don't currently copy + // artifacts into the sysroot. This is not necessarily the right + // choice (in particular, it will require the LLVM dylib to be in + // the linker's load path at runtime), but the common use case for + // external LLVMs is distribution provided LLVMs, and in that case + // they're usually in the standard search path (e.g., /usr/lib) and + // copying them here is going to cause problems as we may end up + // with the wrong files and isn't what distributions want. + // + // This behavior may be revisited in the future though. + // + // If the LLVM is coming from ourselves (just from CI) though, we + // still want to install it, as it otherwise won't be available. + return false; + } + } + + // On macOS, rustc (and LLVM tools) link to an unversioned libLLVM.dylib + // instead of libLLVM-11-rust-....dylib, as on linux. It's not entirely + // clear why this is the case, though. llvm-config will emit the versioned + // paths and we don't want those in the sysroot (as we're expecting + // unversioned paths). + if target.contains("apple-darwin") && builder.llvm_link_shared() { + let src_libdir = builder.llvm_out(target).join("lib"); + let llvm_dylib_path = src_libdir.join("libLLVM.dylib"); + if llvm_dylib_path.exists() { + builder.install(&llvm_dylib_path, dst_libdir, 0o644); + } + !builder.config.dry_run() + } else if let Ok(llvm::LlvmResult { llvm_config, .. }) = + llvm::prebuilt_llvm_config(builder, target) + { + let mut cmd = Command::new(llvm_config); + cmd.arg("--libfiles"); + builder.verbose(&format!("running {cmd:?}")); + let files = if builder.config.dry_run() { "".into() } else { output(&mut cmd) }; + let build_llvm_out = &builder.llvm_out(builder.config.build); + let target_llvm_out = &builder.llvm_out(target); + for file in files.trim_end().split(' ') { + // If we're not using a custom LLVM, make sure we package for the target. + let file = if let Ok(relative_path) = Path::new(file).strip_prefix(build_llvm_out) { + target_llvm_out.join(relative_path) + } else { + PathBuf::from(file) + }; + install_llvm_file(builder, &file, dst_libdir); + } + !builder.config.dry_run() + } else { + false + } +} + +/// Maybe add libLLVM.so to the target lib-dir for linking. +pub fn maybe_install_llvm_target(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { + let dst_libdir = sysroot.join("lib/rustlib").join(&*target.triple).join("lib"); + // We do not need to copy LLVM files into the sysroot if it is not + // dynamically linked; it is already included into librustc_llvm + // statically. + if builder.llvm_link_shared() { + maybe_install_llvm(builder, target, &dst_libdir); + } +} + +/// Maybe add libLLVM.so to the runtime lib-dir for rustc itself. +pub fn maybe_install_llvm_runtime(builder: &Builder<'_>, target: TargetSelection, sysroot: &Path) { + let dst_libdir = + sysroot.join(builder.sysroot_libdir_relative(Compiler { stage: 1, host: target })); + // We do not need to copy LLVM files into the sysroot if it is not + // dynamically linked; it is already included into librustc_llvm + // statically. + if builder.llvm_link_shared() { + maybe_install_llvm(builder, target, &dst_libdir); + } +} + +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct LlvmTools { + pub target: TargetSelection, +} + +impl Step for LlvmTools { + type Output = Option; + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = should_build_extended_tool(&run.builder, "llvm-tools"); + // FIXME: allow using the names of the tools themselves? + run.alias("llvm-tools").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(LlvmTools { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + + /* run only if llvm-config isn't used */ + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref _s) = config.llvm_config { + builder.info(&format!("Skipping LlvmTools ({target}): external LLVM")); + return None; + } + } + + builder.ensure(crate::core::build_steps::llvm::Llvm { target }); + + let mut tarball = Tarball::new(builder, "llvm-tools", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); + tarball.is_preview(true); + + // Prepare the image directory + let src_bindir = builder.llvm_out(target).join("bin"); + let dst_bindir = format!("lib/rustlib/{}/bin", target.triple); + for tool in LLVM_TOOLS { + let exe = src_bindir.join(exe(tool, target)); + tarball.add_file(&exe, &dst_bindir, 0o755); + } + + // Copy libLLVM.so to the target lib dir as well, so the RPATH like + // `$ORIGIN/../lib` can find it. It may also be used as a dependency + // of `rustc-dev` to support the inherited `-lLLVM` when using the + // compiler libraries. + maybe_install_llvm_target(builder, target, tarball.image_dir()); + + Some(tarball.generate()) + } +} + +// Tarball intended for internal consumption to ease rustc/std development. +// +// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct RustDev { + pub target: TargetSelection, +} + +impl Step for RustDev { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("rust-dev") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDev { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + + /* run only if llvm-config isn't used */ + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref _s) = config.llvm_config { + builder.info(&format!("Skipping RustDev ({target}): external LLVM")); + return None; + } + } + + let mut tarball = Tarball::new(builder, "rust-dev", &target.triple); + tarball.set_overlay(OverlayKind::LLVM); + + builder.ensure(crate::core::build_steps::llvm::Llvm { target }); + + // We want to package `lld` to use it with `download-ci-llvm`. + builder.ensure(crate::core::build_steps::llvm::Lld { target }); + + let src_bindir = builder.llvm_out(target).join("bin"); + // If updating this, you likely want to change + // src/bootstrap/download-ci-llvm-stamp as well, otherwise local users + // will not pick up the extra file until LLVM gets bumped. + // We should include all the build artifacts obtained from a source build, + // so that you can use the downloadable LLVM as if you’ve just run a full source build. + if src_bindir.exists() { + for entry in walkdir::WalkDir::new(&src_bindir) { + let entry = t!(entry); + if entry.file_type().is_file() && !entry.path_is_symlink() { + let name = entry.file_name().to_str().unwrap(); + tarball.add_file(src_bindir.join(name), "bin", 0o755); + } + } + } + + // We don't build LLD on some platforms, so only add it if it exists + let lld_path = builder.lld_out(target).join("bin").join(exe("lld", target)); + if lld_path.exists() { + tarball.add_file(lld_path, "bin", 0o755); + } + + tarball.add_file(&builder.llvm_filecheck(target), "bin", 0o755); + + // Copy the include directory as well; needed mostly to build + // librustc_llvm properly (e.g., llvm-config.h is in here). But also + // just broadly useful to be able to link against the bundled LLVM. + tarball.add_dir(&builder.llvm_out(target).join("include"), "include"); + + // Copy libLLVM.so to the target lib dir as well, so the RPATH like + // `$ORIGIN/../lib` can find it. It may also be used as a dependency + // of `rustc-dev` to support the inherited `-lLLVM` when using the + // compiler libraries. + let dst_libdir = tarball.image_dir().join("lib"); + maybe_install_llvm(builder, target, &dst_libdir); + let link_type = if builder.llvm_link_shared() { "dynamic" } else { "static" }; + t!(std::fs::write(tarball.image_dir().join("link-type.txt"), link_type), dst_libdir); + + Some(tarball.generate()) + } +} + +// Tarball intended for internal consumption to ease rustc/std development. +// +// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct Bootstrap { + pub target: TargetSelection, +} + +impl Step for Bootstrap { + type Output = Option; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("bootstrap") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Bootstrap { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let target = self.target; + + let tarball = Tarball::new(builder, "bootstrap", &target.triple); + + let bootstrap_outdir = &builder.bootstrap_out; + for file in &["bootstrap", "rustc", "rustdoc", "sccache-plus-cl"] { + tarball.add_file(bootstrap_outdir.join(exe(file, target)), "bootstrap/bin", 0o755); + } + + Some(tarball.generate()) + } +} + +/// Tarball containing a prebuilt version of the build-manifest tool, intended to be used by the +/// release process to avoid cloning the monorepo and building stuff. +/// +/// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BuildManifest { + pub target: TargetSelection, +} + +impl Step for BuildManifest { + type Output = GeneratedTarball; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("build-manifest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BuildManifest { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> GeneratedTarball { + let build_manifest = builder.tool_exe(Tool::BuildManifest); + + let tarball = Tarball::new(builder, "build-manifest", &self.target.triple); + tarball.add_file(&build_manifest, "bin", 0o755); + tarball.generate() + } +} + +/// Tarball containing artifacts necessary to reproduce the build of rustc. +/// +/// Currently this is the PGO profile data. +/// +/// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct ReproducibleArtifacts { + pub target: TargetSelection, +} + +impl Step for ReproducibleArtifacts { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("reproducible-artifacts") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ReproducibleArtifacts { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let mut added_anything = false; + let tarball = Tarball::new(builder, "reproducible-artifacts", &self.target.triple); + if let Some(path) = builder.config.rust_profile_use.as_ref() { + tarball.add_file(path, ".", 0o644); + added_anything = true; + } + if let Some(path) = builder.config.llvm_profile_use.as_ref() { + tarball.add_file(path, ".", 0o644); + added_anything = true; + } + for profile in &builder.config.reproducible_artifacts { + tarball.add_file(profile, ".", 0o644); + added_anything = true; + } + if added_anything { Some(tarball.generate()) } else { None } + } +} diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs new file mode 100644 index 000000000..e3fd942fc --- /dev/null +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -0,0 +1,1094 @@ +//! Documentation generation for rustbuilder. +//! +//! This module implements generation for all bits and pieces of documentation +//! for the Rust project. This notably includes suites like the rust book, the +//! nomicon, rust by example, standalone documentation, etc. +//! +//! Everything here is basically just a shim around calling either `rustbook` or +//! `rustdoc`. + +use std::fs; +use std::path::{Path, PathBuf}; + +use crate::core::build_steps::compile; +use crate::core::build_steps::tool::{self, prepare_tool_cargo, SourceType, Tool}; +use crate::core::builder::crate_description; +use crate::core::builder::{Alias, Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::helpers::{dir_is_empty, symlink_dir, t, up_to_date}; +use crate::Mode; + +macro_rules! submodule_helper { + ($path:expr, submodule) => { + $path + }; + ($path:expr, submodule = $submodule:literal) => { + $submodule + }; +} + +macro_rules! book { + ($($name:ident, $path:expr, $book_name:expr $(, submodule $(= $submodule:literal)? )? ;)+) => { + $( + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $name { + target: TargetSelection, + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path($path).default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) { + $( + let path = Path::new(submodule_helper!( $path, submodule $( = $submodule )? )); + builder.update_submodule(&path); + )? + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str($book_name), + src: INTERNER.intern_path(builder.src.join($path)), + parent: Some(self), + }) + } + } + )+ + } +} + +// NOTE: When adding a book here, make sure to ALSO build the book by +// adding a build step in `src/bootstrap/builder.rs`! +// NOTE: Make sure to add the corresponding submodule when adding a new book. +// FIXME: Make checking for a submodule automatic somehow (maybe by having a list of all submodules +// and checking against it?). +book!( + CargoBook, "src/tools/cargo/src/doc", "cargo", submodule = "src/tools/cargo"; + ClippyBook, "src/tools/clippy/book", "clippy"; + EditionGuide, "src/doc/edition-guide", "edition-guide", submodule; + EmbeddedBook, "src/doc/embedded-book", "embedded-book", submodule; + Nomicon, "src/doc/nomicon", "nomicon", submodule; + Reference, "src/doc/reference", "reference", submodule; + RustByExample, "src/doc/rust-by-example", "rust-by-example", submodule; + RustdocBook, "src/doc/rustdoc", "rustdoc"; + StyleGuide, "src/doc/style-guide", "style-guide"; +); + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct UnstableBook { + target: TargetSelection, +} + +impl Step for UnstableBook { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/unstable-book").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(UnstableBook { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(UnstableBookGen { target: self.target }); + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str("unstable-book"), + src: INTERNER.intern_path(builder.md_doc_out(self.target).join("unstable-book")), + parent: Some(self), + }) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +struct RustbookSrc { + target: TargetSelection, + name: Interned, + src: Interned, + parent: Option

, +} + +impl Step for RustbookSrc

{ + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Invoke `rustbook` for `target` for the doc book `name` from the `src` path. + /// + /// This will not actually generate any documentation if the documentation has + /// already been generated. + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let name = self.name; + let src = self.src; + let out = builder.doc_out(target); + t!(fs::create_dir_all(&out)); + + let out = out.join(name); + let index = out.join("index.html"); + let rustbook = builder.tool_exe(Tool::Rustbook); + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + + if !builder.config.dry_run() && !(up_to_date(&src, &index) || up_to_date(&rustbook, &index)) + { + builder.info(&format!("Rustbook ({target}) - {name}")); + let _ = fs::remove_dir_all(&out); + + builder.run(rustbook_cmd.arg("build").arg(&src).arg("-d").arg(out)); + } + + if self.parent.is_some() { + builder.maybe_open_in_browser::

(index) + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct TheBook { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for TheBook { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/book").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(TheBook { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Builds the book and associated stuff. + /// + /// We need to build: + /// + /// * Book + /// * Older edition redirects + /// * Version info and CSS + /// * Index page + /// * Redirect pages + fn run(self, builder: &Builder<'_>) { + let relative_path = Path::new("src").join("doc").join("book"); + builder.update_submodule(&relative_path); + + let compiler = self.compiler; + let target = self.target; + + let absolute_path = builder.src.join(&relative_path); + let redirect_path = absolute_path.join("redirects"); + if !absolute_path.exists() + || !redirect_path.exists() + || dir_is_empty(&absolute_path) + || dir_is_empty(&redirect_path) + { + eprintln!("Please checkout submodule: {}", relative_path.display()); + crate::exit!(1); + } + // build book + builder.ensure(RustbookSrc { + target, + name: INTERNER.intern_str("book"), + src: INTERNER.intern_path(absolute_path.clone()), + parent: Some(self), + }); + + // building older edition redirects + for edition in &["first-edition", "second-edition", "2018-edition"] { + builder.ensure(RustbookSrc { + target, + name: INTERNER.intern_string(format!("book/{edition}")), + src: INTERNER.intern_path(absolute_path.join(edition)), + // There should only be one book that is marked as the parent for each target, so + // treat the other editions as not having a parent. + parent: Option::::None, + }); + } + + // build the version info page and CSS + let shared_assets = builder.ensure(SharedAssets { target }); + + // build the command first so we don't nest GHA groups + builder.rustdoc_cmd(compiler); + + // build the redirect pages + let _guard = builder.msg_doc(compiler, "book redirect pages", target); + for file in t!(fs::read_dir(redirect_path)) { + let file = t!(file); + let path = file.path(); + let path = path.to_str().unwrap(); + + invoke_rustdoc(builder, compiler, &shared_assets, target, path); + } + } +} + +fn invoke_rustdoc( + builder: &Builder<'_>, + compiler: Compiler, + shared_assets: &SharedAssetsPaths, + target: TargetSelection, + markdown: &str, +) { + let out = builder.doc_out(target); + + let path = builder.src.join("src/doc").join(markdown); + + let header = builder.src.join("src/doc/redirect.inc"); + let footer = builder.src.join("src/doc/footer.inc"); + + let mut cmd = builder.rustdoc_cmd(compiler); + + let out = out.join("book"); + + cmd.arg("--html-after-content") + .arg(&footer) + .arg("--html-before-content") + .arg(&shared_assets.version_info) + .arg("--html-in-header") + .arg(&header) + .arg("--markdown-no-toc") + .arg("--markdown-playground-url") + .arg("https://play.rust-lang.org/") + .arg("-o") + .arg(&out) + .arg(&path) + .arg("--markdown-css") + .arg("../rust.css"); + + if !builder.config.docs_minification { + cmd.arg("-Z").arg("unstable-options").arg("--disable-minification"); + } + + builder.run(&mut cmd); +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Standalone { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for Standalone { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc").alias("standalone").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Standalone { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Generates all standalone documentation as compiled by the rustdoc in `stage` + /// for the `target` into `out`. + /// + /// This will list all of `src/doc` looking for markdown files and appropriately + /// perform transformations like substituting `VERSION`, `SHORT_HASH`, and + /// `STAMP` along with providing the various header/footer HTML we've customized. + /// + /// In the end, this is just a glorified wrapper around rustdoc! + fn run(self, builder: &Builder<'_>) { + let target = self.target; + let compiler = self.compiler; + let _guard = builder.msg_doc(compiler, "standalone", target); + let out = builder.doc_out(target); + t!(fs::create_dir_all(&out)); + + let version_info = builder.ensure(SharedAssets { target: self.target }).version_info; + + let favicon = builder.src.join("src/doc/favicon.inc"); + let footer = builder.src.join("src/doc/footer.inc"); + let full_toc = builder.src.join("src/doc/full-toc.inc"); + + for file in t!(fs::read_dir(builder.src.join("src/doc"))) { + let file = t!(file); + let path = file.path(); + let filename = path.file_name().unwrap().to_str().unwrap(); + if !filename.ends_with(".md") || filename == "README.md" { + continue; + } + + let html = out.join(filename).with_extension("html"); + let rustdoc = builder.rustdoc(compiler); + if up_to_date(&path, &html) + && up_to_date(&footer, &html) + && up_to_date(&favicon, &html) + && up_to_date(&full_toc, &html) + && (builder.config.dry_run() || up_to_date(&version_info, &html)) + && (builder.config.dry_run() || up_to_date(&rustdoc, &html)) + { + continue; + } + + let mut cmd = builder.rustdoc_cmd(compiler); + // Needed for --index-page flag + cmd.arg("-Z").arg("unstable-options"); + + cmd.arg("--html-after-content") + .arg(&footer) + .arg("--html-before-content") + .arg(&version_info) + .arg("--html-in-header") + .arg(&favicon) + .arg("--markdown-no-toc") + .arg("--index-page") + .arg(&builder.src.join("src/doc/index.md")) + .arg("--markdown-playground-url") + .arg("https://play.rust-lang.org/") + .arg("-o") + .arg(&out) + .arg(&path); + + if !builder.config.docs_minification { + cmd.arg("--disable-minification"); + } + + if filename == "not_found.md" { + cmd.arg("--markdown-css").arg("https://doc.rust-lang.org/rust.css"); + } else { + cmd.arg("--markdown-css").arg("rust.css"); + } + builder.run(&mut cmd); + } + + // We open doc/index.html as the default if invoked as `x.py doc --open` + // with no particular explicit doc requested (e.g. library/core). + if builder.paths.is_empty() || builder.was_invoked_explicitly::(Kind::Doc) { + let index = out.join("index.html"); + builder.open_in_browser(&index); + } + } +} + +#[derive(Debug, Clone)] +pub struct SharedAssetsPaths { + pub version_info: PathBuf, +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct SharedAssets { + target: TargetSelection, +} + +impl Step for SharedAssets { + type Output = SharedAssetsPaths; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + // Other tasks depend on this, no need to execute it on its own + run.never() + } + + // Generate shared resources used by other pieces of documentation. + fn run(self, builder: &Builder<'_>) -> Self::Output { + let out = builder.doc_out(self.target); + + let version_input = builder.src.join("src").join("doc").join("version_info.html.template"); + let version_info = out.join("version_info.html"); + if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) { + let info = t!(fs::read_to_string(&version_input)) + .replace("VERSION", &builder.rust_release()) + .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or("")) + .replace("STAMP", builder.rust_info().sha().unwrap_or("")); + t!(fs::write(&version_info, &info)); + } + + builder.copy(&builder.src.join("src").join("doc").join("rust.css"), &out.join("rust.css")); + + SharedAssetsPaths { version_info } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub struct Std { + pub stage: u32, + pub target: TargetSelection, + pub format: DocumentationFormat, + crates: Interned>, +} + +impl Std { + pub(crate) fn new( + stage: u32, + target: TargetSelection, + builder: &Builder<'_>, + format: DocumentationFormat, + ) -> Self { + let crates = builder + .in_tree_crates("sysroot", Some(target)) + .into_iter() + .map(|krate| krate.name.to_string()) + .collect(); + Std { stage, target, format, crates: INTERNER.intern_list(crates) } + } +} + +impl Step for Std { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.crate_or_deps("sysroot").path("library").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Std { + stage: run.builder.top_stage, + target: run.target, + format: if run.builder.config.cmd.json() { + DocumentationFormat::JSON + } else { + DocumentationFormat::HTML + }, + crates: run.make_run_crates(Alias::Library), + }); + } + + /// Compile all standard library documentation. + /// + /// This will generate all documentation for the standard library and its + /// dependencies. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let target = self.target; + let out = match self.format { + DocumentationFormat::HTML => builder.doc_out(target), + DocumentationFormat::JSON => builder.json_doc_out(target), + }; + + t!(fs::create_dir_all(&out)); + + if self.format == DocumentationFormat::HTML { + builder.ensure(SharedAssets { target: self.target }); + } + + let index_page = builder + .src + .join("src/doc/index.md") + .into_os_string() + .into_string() + .expect("non-utf8 paths are unsupported"); + let mut extra_args = match self.format { + DocumentationFormat::HTML => { + vec!["--markdown-css", "rust.css", "--markdown-no-toc", "--index-page", &index_page] + } + DocumentationFormat::JSON => vec!["--output-format", "json"], + }; + + if !builder.config.docs_minification { + extra_args.push("--disable-minification"); + } + + doc_std(builder, self.format, stage, target, &out, &extra_args, &self.crates); + + // Don't open if the format is json + if let DocumentationFormat::JSON = self.format { + return; + } + + if builder.paths.iter().any(|path| path.ends_with("library")) { + // For `x.py doc library --open`, open `std` by default. + let index = out.join("std").join("index.html"); + builder.open_in_browser(index); + } else { + for requested_crate in &*self.crates { + if STD_PUBLIC_CRATES.iter().any(|&k| k == requested_crate) { + let index = out.join(requested_crate).join("index.html"); + builder.open_in_browser(index); + break; + } + } + } + } +} + +/// Name of the crates that are visible to consumers of the standard library. +/// Documentation for internal crates is handled by the rustc step, so internal crates will show +/// up there. +/// +/// Order here is important! +/// Crates need to be processed starting from the leaves, otherwise rustdoc will not +/// create correct links between crates because rustdoc depends on the +/// existence of the output directories to know if it should be a local +/// or remote link. +const STD_PUBLIC_CRATES: [&str; 5] = ["core", "alloc", "std", "proc_macro", "test"]; + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub enum DocumentationFormat { + HTML, + JSON, +} + +impl DocumentationFormat { + fn as_str(&self) -> &str { + match self { + DocumentationFormat::HTML => "HTML", + DocumentationFormat::JSON => "JSON", + } + } +} + +/// Build the documentation for public standard library crates. +fn doc_std( + builder: &Builder<'_>, + format: DocumentationFormat, + stage: u32, + target: TargetSelection, + out: &Path, + extra_args: &[&str], + requested_crates: &[String], +) { + if builder.no_std(target) == Some(true) { + panic!( + "building std documentation for no_std target {target} is not supported\n\ + Set `docs = false` in the config to disable documentation, or pass `--skip library`." + ); + } + + let compiler = builder.compiler(stage, builder.config.build); + + let target_doc_dir_name = if format == DocumentationFormat::JSON { "json-doc" } else { "doc" }; + let target_dir = + builder.stage_out(compiler, Mode::Std).join(target.triple).join(target_doc_dir_name); + + // This is directory where the compiler will place the output of the command. + // We will then copy the files from this directory into the final `out` directory, the specified + // as a function parameter. + let out_dir = target_dir.join(target.triple).join("doc"); + + let mut cargo = builder.cargo(compiler, Mode::Std, SourceType::InTree, target, "doc"); + compile::std_cargo(builder, target, compiler.stage, &mut cargo); + cargo + .arg("--no-deps") + .arg("--target-dir") + .arg(&*target_dir.to_string_lossy()) + .arg("-Zskip-rustdoc-fingerprint") + .rustdocflag("-Z") + .rustdocflag("unstable-options") + .rustdocflag("--resource-suffix") + .rustdocflag(&builder.version); + for arg in extra_args { + cargo.rustdocflag(arg); + } + + if builder.config.library_docs_private_items { + cargo.rustdocflag("--document-private-items").rustdocflag("--document-hidden-items"); + } + + for krate in requested_crates { + if krate == "sysroot" { + // The sysroot crate is an implementation detail, don't include it in public docs. + continue; + } + cargo.arg("-p").arg(krate); + } + + let description = + format!("library{} in {} format", crate_description(&requested_crates), format.as_str()); + let _guard = builder.msg_doc(compiler, &description, target); + + builder.run(&mut cargo.into()); + builder.cp_r(&out_dir, &out); +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Rustc { + pub stage: u32, + pub target: TargetSelection, + crates: Interned>, +} + +impl Rustc { + pub(crate) fn new(stage: u32, target: TargetSelection, builder: &Builder<'_>) -> Self { + let crates = builder + .in_tree_crates("rustc-main", Some(target)) + .into_iter() + .map(|krate| krate.name.to_string()) + .collect(); + Self { stage, target, crates: INTERNER.intern_list(crates) } + } +} + +impl Step for Rustc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.crate_or_deps("rustc-main") + .path("compiler") + .default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustc { + stage: run.builder.top_stage, + target: run.target, + crates: run.make_run_crates(Alias::Compiler), + }); + } + + /// Generates compiler documentation. + /// + /// This will generate all documentation for compiler and dependencies. + /// Compiler documentation is distributed separately, so we make sure + /// we do not merge it with the other documentation from std, test and + /// proc_macros. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let target = self.target; + + // This is the intended out directory for compiler documentation. + let out = builder.compiler_doc_out(target); + t!(fs::create_dir_all(&out)); + + // Build the standard library, so that proc-macros can use it. + // (Normally, only the metadata would be necessary, but proc-macros are special since they run at compile-time.) + let compiler = builder.compiler(stage, builder.config.build); + builder.ensure(compile::Std::new(compiler, builder.config.build)); + + let _guard = builder.msg_sysroot_tool( + Kind::Doc, + stage, + &format!("compiler{}", crate_description(&self.crates)), + compiler.host, + target, + ); + + // Build cargo command. + let mut cargo = builder.cargo(compiler, Mode::Rustc, SourceType::InTree, target, "doc"); + cargo.rustdocflag("--document-private-items"); + // Since we always pass --document-private-items, there's no need to warn about linking to private items. + cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); + cargo.rustdocflag("--enable-index-page"); + cargo.rustdocflag("-Zunstable-options"); + cargo.rustdocflag("-Znormalize-docs"); + cargo.rustdocflag("--show-type-layout"); + cargo.rustdocflag("--generate-link-to-definition"); + compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); + cargo.arg("-Zunstable-options"); + cargo.arg("-Zskip-rustdoc-fingerprint"); + + // Only include compiler crates, no dependencies of those, such as `libc`. + // Do link to dependencies on `docs.rs` however using `rustdoc-map`. + cargo.arg("--no-deps"); + cargo.arg("-Zrustdoc-map"); + + // FIXME: `-Zrustdoc-map` does not yet correctly work for transitive dependencies, + // once this is no longer an issue the special case for `ena` can be removed. + cargo.rustdocflag("--extern-html-root-url"); + cargo.rustdocflag("ena=https://docs.rs/ena/latest/"); + + let mut to_open = None; + + let out_dir = builder.stage_out(compiler, Mode::Rustc).join(target.triple).join("doc"); + for krate in &*self.crates { + // Create all crate output directories first to make sure rustdoc uses + // relative links. + // FIXME: Cargo should probably do this itself. + let dir_name = krate.replace("-", "_"); + t!(fs::create_dir_all(out_dir.join(&*dir_name))); + cargo.arg("-p").arg(krate); + if to_open.is_none() { + to_open = Some(dir_name); + } + } + + // This uses a shared directory so that librustdoc documentation gets + // correctly built and merged with the rustc documentation. + // + // This is needed because rustdoc is built in a different directory from + // rustc. rustdoc needs to be able to see everything, for example when + // merging the search index, or generating local (relative) links. + symlink_dir_force(&builder.config, &out, &out_dir); + // Cargo puts proc macros in `target/doc` even if you pass `--target` + // explicitly (https://github.com/rust-lang/cargo/issues/7677). + let proc_macro_out_dir = builder.stage_out(compiler, Mode::Rustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + + builder.run(&mut cargo.into()); + + if !builder.config.dry_run() { + // Sanity check on linked compiler crates + for krate in &*self.crates { + let dir_name = krate.replace("-", "_"); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); + } + } + + if builder.paths.iter().any(|path| path.ends_with("compiler")) { + // For `x.py doc compiler --open`, open `rustc_middle` by default. + let index = out.join("rustc_middle").join("index.html"); + builder.open_in_browser(index); + } else if let Some(krate) = to_open { + // Let's open the first crate documentation page: + let index = out.join(krate).join("index.html"); + builder.open_in_browser(index); + } + } +} + +macro_rules! tool_doc { + ( + $tool: ident, + $should_run: literal, + $path: literal, + $(rustc_tool = $rustc_tool:literal, )? + $(in_tree = $in_tree:literal ,)? + $(is_library = $is_library:expr,)? + $(crates = $crates:expr)? + ) => { + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $tool { + target: TargetSelection, + } + + impl Step for $tool { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.crate_or_deps($should_run).default_condition(builder.config.compiler_docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($tool { target: run.target }); + } + + /// Generates compiler documentation. + /// + /// This will generate all documentation for compiler and dependencies. + /// Compiler documentation is distributed separately, so we make sure + /// we do not merge it with the other documentation from std, test and + /// proc_macros. This is largely just a wrapper around `cargo doc`. + fn run(self, builder: &Builder<'_>) { + let stage = builder.top_stage; + let target = self.target; + + // This is the intended out directory for compiler documentation. + let out = builder.compiler_doc_out(target); + t!(fs::create_dir_all(&out)); + + let compiler = builder.compiler(stage, builder.config.build); + builder.ensure(compile::Std::new(compiler, target)); + + if true $(&& $rustc_tool)? { + // Build rustc docs so that we generate relative links. + builder.ensure(Rustc::new(stage, target, builder)); + + // Rustdoc needs the rustc sysroot available to build. + // FIXME: is there a way to only ensure `check::Rustc` here? Last time I tried it failed + // with strange errors, but only on a full bors test ... + builder.ensure(compile::Rustc::new(compiler, target)); + } + + let source_type = if true $(&& $in_tree)? { + SourceType::InTree + } else { + SourceType::Submodule + }; + + // Build cargo command. + let mut cargo = prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + "doc", + $path, + source_type, + &[], + ); + + cargo.arg("-Zskip-rustdoc-fingerprint"); + // Only include compiler crates, no dependencies of those, such as `libc`. + cargo.arg("--no-deps"); + + if false $(|| $is_library)? { + cargo.arg("--lib"); + } + + $(for krate in $crates { + cargo.arg("-p").arg(krate); + })? + + cargo.rustdocflag("--document-private-items"); + // Since we always pass --document-private-items, there's no need to warn about linking to private items. + cargo.rustdocflag("-Arustdoc::private-intra-doc-links"); + cargo.rustdocflag("--enable-index-page"); + cargo.rustdocflag("--show-type-layout"); + cargo.rustdocflag("--generate-link-to-definition"); + cargo.rustdocflag("-Zunstable-options"); + + let out_dir = builder.stage_out(compiler, Mode::ToolRustc).join(target.triple).join("doc"); + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + t!(fs::create_dir_all(out_dir.join(&*dir_name))); + })? + + // Symlink compiler docs to the output directory of rustdoc documentation. + symlink_dir_force(&builder.config, &out, &out_dir); + let proc_macro_out_dir = builder.stage_out(compiler, Mode::ToolRustc).join("doc"); + symlink_dir_force(&builder.config, &out, &proc_macro_out_dir); + + let _guard = builder.msg_doc(compiler, stringify!($tool).to_lowercase(), target); + builder.run(&mut cargo.into()); + + if !builder.config.dry_run() { + // Sanity check on linked doc directories + $(for krate in $crates { + let dir_name = krate.replace("-", "_"); + // Making sure the directory exists and is not empty. + assert!(out.join(&*dir_name).read_dir().unwrap().next().is_some()); + })? + } + } + } + } +} + +tool_doc!(Rustdoc, "rustdoc-tool", "src/tools/rustdoc", crates = ["rustdoc", "rustdoc-json-types"]); +tool_doc!( + Rustfmt, + "rustfmt-nightly", + "src/tools/rustfmt", + crates = ["rustfmt-nightly", "rustfmt-config_proc_macro"] +); +tool_doc!(Clippy, "clippy", "src/tools/clippy", crates = ["clippy_config", "clippy_utils"]); +tool_doc!(Miri, "miri", "src/tools/miri", crates = ["miri"]); +tool_doc!( + Cargo, + "cargo", + "src/tools/cargo", + rustc_tool = false, + in_tree = false, + crates = [ + "cargo", + "cargo-platform", + "cargo-util", + "crates-io", + "cargo-test-macro", + "cargo-test-support", + "cargo-credential", + "mdman", + // FIXME: this trips a license check in tidy. + // "resolver-tests", + ] +); +tool_doc!(Tidy, "tidy", "src/tools/tidy", rustc_tool = false, crates = ["tidy"]); +tool_doc!( + Bootstrap, + "bootstrap", + "src/bootstrap", + rustc_tool = false, + is_library = true, + crates = ["bootstrap"] +); + +#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct ErrorIndex { + pub target: TargetSelection, +} + +impl Step for ErrorIndex { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/error_index_generator").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + let target = run.target; + run.builder.ensure(ErrorIndex { target }); + } + + /// Generates the HTML rendered error-index by running the + /// `error_index_generator` tool. + fn run(self, builder: &Builder<'_>) { + builder.info(&format!("Documenting error index ({})", self.target)); + let out = builder.doc_out(self.target); + t!(fs::create_dir_all(&out)); + let mut index = tool::ErrorIndex::command(builder); + index.arg("html"); + index.arg(out); + index.arg(&builder.version); + + builder.run(&mut index); + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct UnstableBookGen { + target: TargetSelection, +} + +impl Step for UnstableBookGen { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/unstable-book-gen").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(UnstableBookGen { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let target = self.target; + + builder.info(&format!("Generating unstable book md files ({target})")); + let out = builder.md_doc_out(target).join("unstable-book"); + builder.create_dir(&out); + builder.remove_dir(&out); + let mut cmd = builder.tool_cmd(Tool::UnstableBookGen); + cmd.arg(builder.src.join("library")); + cmd.arg(builder.src.join("compiler")); + cmd.arg(builder.src.join("src")); + cmd.arg(out); + + builder.run(&mut cmd); + } +} + +fn symlink_dir_force(config: &Config, original: &Path, link: &Path) { + if config.dry_run() { + return; + } + if let Ok(m) = fs::symlink_metadata(link) { + if m.file_type().is_dir() { + t!(fs::remove_dir_all(link)); + } else { + // handle directory junctions on windows by falling back to + // `remove_dir`. + t!(fs::remove_file(link).or_else(|_| fs::remove_dir(link))); + } + } + + t!( + symlink_dir(config, original, link), + format!("failed to create link from {} -> {}", link.display(), original.display()) + ); +} + +#[derive(Ord, PartialOrd, Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustcBook { + pub compiler: Compiler, + pub target: TargetSelection, + pub validate: bool, +} + +impl Step for RustcBook { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/doc/rustc").default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcBook { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + validate: false, + }); + } + + /// Builds the rustc book. + /// + /// The lints are auto-generated by a tool, and then merged into the book + /// in the "md-doc" directory in the build output directory. Then + /// "rustbook" is used to convert it to HTML. + fn run(self, builder: &Builder<'_>) { + let out_base = builder.md_doc_out(self.target).join("rustc"); + t!(fs::create_dir_all(&out_base)); + let out_listing = out_base.join("src/lints"); + builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base); + builder.info(&format!("Generating lint docs ({})", self.target)); + + let rustc = builder.rustc(self.compiler); + // The tool runs `rustc` for extracting output examples, so it needs a + // functional sysroot. + builder.ensure(compile::Std::new(self.compiler, self.target)); + let mut cmd = builder.tool_cmd(Tool::LintDocs); + cmd.arg("--src"); + cmd.arg(builder.src.join("compiler")); + cmd.arg("--out"); + cmd.arg(&out_listing); + cmd.arg("--rustc"); + cmd.arg(&rustc); + cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg()); + if builder.is_verbose() { + cmd.arg("--verbose"); + } + if self.validate { + cmd.arg("--validate"); + } + // We need to validate nightly features, even on the stable channel. + // Set this unconditionally as the stage0 compiler may be being used to + // document. + cmd.env("RUSTC_BOOTSTRAP", "1"); + + // If the lib directories are in an unusual location (changed in + // config.toml), then this needs to explicitly update the dylib search + // path. + builder.add_rustc_lib_path(self.compiler, &mut cmd); + let doc_generator_guard = builder.msg( + Kind::Run, + self.compiler.stage, + "lint-docs", + self.compiler.host, + self.target, + ); + builder.run(&mut cmd); + drop(doc_generator_guard); + + // Run rustbook/mdbook to generate the HTML pages. + builder.ensure(RustbookSrc { + target: self.target, + name: INTERNER.intern_str("rustc"), + src: INTERNER.intern_path(out_base), + parent: Some(self), + }); + } +} diff --git a/src/bootstrap/src/core/build_steps/format.rs b/src/bootstrap/src/core/build_steps/format.rs new file mode 100644 index 000000000..86f1d925f --- /dev/null +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -0,0 +1,322 @@ +//! Runs rustfmt on the repository. + +use crate::core::builder::Builder; +use crate::utils::helpers::{output, program_out_of_date, t}; +use build_helper::ci::CiEnv; +use build_helper::git::get_git_modified_files; +use ignore::WalkBuilder; +use std::collections::VecDeque; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::mpsc::SyncSender; + +fn rustfmt(src: &Path, rustfmt: &Path, paths: &[PathBuf], check: bool) -> impl FnMut(bool) -> bool { + let mut cmd = Command::new(&rustfmt); + // avoid the submodule config paths from coming into play, + // we only allow a single global config for the workspace for now + cmd.arg("--config-path").arg(&src.canonicalize().unwrap()); + cmd.arg("--edition").arg("2021"); + cmd.arg("--unstable-features"); + cmd.arg("--skip-children"); + if check { + cmd.arg("--check"); + } + cmd.args(paths); + let cmd_debug = format!("{cmd:?}"); + let mut cmd = cmd.spawn().expect("running rustfmt"); + // poor man's async: return a closure that'll wait for rustfmt's completion + move |block: bool| -> bool { + if !block { + match cmd.try_wait() { + Ok(Some(_)) => {} + _ => return false, + } + } + let status = cmd.wait().unwrap(); + if !status.success() { + eprintln!( + "Running `{}` failed.\nIf you're running `tidy`, \ + try again with `--bless`. Or, if you just want to format \ + code, run `./x.py fmt` instead.", + cmd_debug, + ); + crate::exit!(1); + } + true + } +} + +fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, PathBuf)> { + let stamp_file = build.out.join("rustfmt.stamp"); + + let mut cmd = Command::new(match build.initial_rustfmt() { + Some(p) => p, + None => return None, + }); + cmd.arg("--version"); + let output = match cmd.output() { + Ok(status) => status, + Err(_) => return None, + }; + if !output.status.success() { + return None; + } + Some((String::from_utf8(output.stdout).unwrap(), stamp_file)) +} + +/// Return whether the format cache can be reused. +fn verify_rustfmt_version(build: &Builder<'_>) -> bool { + let Some((version, stamp_file)) = get_rustfmt_version(build) else { + return false; + }; + !program_out_of_date(&stamp_file, &version) +} + +/// Updates the last rustfmt version used +fn update_rustfmt_version(build: &Builder<'_>) { + let Some((version, stamp_file)) = get_rustfmt_version(build) else { + return; + }; + t!(std::fs::write(stamp_file, version)) +} + +/// Returns the Rust files modified between the `merge-base` of HEAD and +/// rust-lang/master and what is now on the disk. +/// +/// Returns `None` if all files should be formatted. +fn get_modified_rs_files(build: &Builder<'_>) -> Result>, String> { + if !verify_rustfmt_version(build) { + return Ok(None); + } + + get_git_modified_files(&build.config.git_config(), Some(&build.config.src), &vec!["rs"]) +} + +#[derive(serde_derive::Deserialize)] +struct RustfmtConfig { + ignore: Vec, +} + +pub fn format(build: &Builder<'_>, check: bool, paths: &[PathBuf]) { + if build.config.dry_run() { + return; + } + let mut builder = ignore::types::TypesBuilder::new(); + builder.add_defaults(); + builder.select("rust"); + let matcher = builder.build().unwrap(); + let rustfmt_config = build.src.join("rustfmt.toml"); + if !rustfmt_config.exists() { + eprintln!("Not running formatting checks; rustfmt.toml does not exist."); + eprintln!("This may happen in distributed tarballs."); + return; + } + let rustfmt_config = t!(std::fs::read_to_string(&rustfmt_config)); + let rustfmt_config: RustfmtConfig = t!(toml::from_str(&rustfmt_config)); + let mut fmt_override = ignore::overrides::OverrideBuilder::new(&build.src); + for ignore in rustfmt_config.ignore { + fmt_override.add(&format!("!{ignore}")).expect(&ignore); + } + let git_available = match Command::new("git") + .arg("--version") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + }; + + if git_available { + let in_working_tree = match build + .config + .git() + .arg("rev-parse") + .arg("--is-inside-work-tree") + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .status() + { + Ok(status) => status.success(), + Err(_) => false, + }; + if in_working_tree { + let untracked_paths_output = output( + build.config.git().arg("status").arg("--porcelain").arg("--untracked-files=normal"), + ); + let untracked_paths = untracked_paths_output + .lines() + .filter(|entry| entry.starts_with("??")) + .map(|entry| { + entry.split(' ').nth(1).expect("every git status entry should list a path") + }); + let mut untracked_count = 0; + for untracked_path in untracked_paths { + println!("skip untracked path {untracked_path} during rustfmt invocations"); + // The leading `/` makes it an exact match against the + // repository root, rather than a glob. Without that, if you + // have `foo.rs` in the repository root it will also match + // against anything like `compiler/rustc_foo/src/foo.rs`, + // preventing the latter from being formatted. + untracked_count += 1; + fmt_override.add(&format!("!/{untracked_path}")).expect(&untracked_path); + } + // Only check modified files locally to speed up runtime. + // We still check all files in CI to avoid bugs in `get_modified_rs_files` letting regressions slip through; + // we also care about CI time less since this is still very fast compared to building the compiler. + if !CiEnv::is_ci() && paths.is_empty() { + match get_modified_rs_files(build) { + Ok(Some(files)) => { + if files.len() <= 10 { + for file in &files { + println!("formatting modified file {file}"); + } + } else { + let pluralized = |count| if count > 1 { "files" } else { "file" }; + let untracked_msg = if untracked_count == 0 { + "".to_string() + } else { + format!( + ", skipped {} untracked {}", + untracked_count, + pluralized(untracked_count), + ) + }; + println!( + "formatting {} modified {}{}", + files.len(), + pluralized(files.len()), + untracked_msg + ); + } + for file in files { + fmt_override.add(&format!("/{file}")).expect(&file); + } + } + Ok(None) => {} + Err(err) => { + println!( + "WARN: Something went wrong when running git commands:\n{err}\n\ + Falling back to formatting all files." + ); + } + } + } + } else { + println!("Not in git tree. Skipping git-aware format checks"); + } + } else { + println!("Could not find usable git. Skipping git-aware format checks"); + } + + let fmt_override = fmt_override.build().unwrap(); + + let rustfmt_path = build.initial_rustfmt().unwrap_or_else(|| { + eprintln!("./x.py fmt is not supported on this channel"); + crate::exit!(1); + }); + assert!(rustfmt_path.exists(), "{}", rustfmt_path.display()); + let src = build.src.clone(); + let (tx, rx): (SyncSender, _) = std::sync::mpsc::sync_channel(128); + let walker = match paths.get(0) { + Some(first) => { + let find_shortcut_candidates = |p: &PathBuf| { + let mut candidates = Vec::new(); + for candidate in WalkBuilder::new(src.clone()).max_depth(Some(3)).build() { + if let Ok(entry) = candidate { + if let Some(dir_name) = p.file_name() { + if entry.path().is_dir() && entry.file_name() == dir_name { + candidates.push(entry.into_path()); + } + } + } + } + candidates + }; + + // Only try to look for shortcut candidates for single component paths like + // `std` and not for e.g. relative paths like `../library/std`. + let should_look_for_shortcut_dir = |p: &PathBuf| p.components().count() == 1; + + let mut walker = if should_look_for_shortcut_dir(first) { + if let [single_candidate] = &find_shortcut_candidates(first)[..] { + WalkBuilder::new(single_candidate) + } else { + WalkBuilder::new(first) + } + } else { + WalkBuilder::new(src.join(first)) + }; + + for path in &paths[1..] { + if should_look_for_shortcut_dir(path) { + if let [single_candidate] = &find_shortcut_candidates(path)[..] { + walker.add(single_candidate); + } else { + walker.add(path); + } + } else { + walker.add(src.join(path)); + } + } + + walker + } + None => WalkBuilder::new(src.clone()), + } + .types(matcher) + .overrides(fmt_override) + .build_parallel(); + + // there is a lot of blocking involved in spawning a child process and reading files to format. + // spawn more processes than available concurrency to keep the CPU busy + let max_processes = build.jobs() as usize * 2; + + // spawn child processes on a separate thread so we can batch entries we have received from ignore + let thread = std::thread::spawn(move || { + let mut children = VecDeque::new(); + while let Ok(path) = rx.recv() { + // try getting a few more paths from the channel to amortize the overhead of spawning processes + let paths: Vec<_> = rx.try_iter().take(7).chain(std::iter::once(path)).collect(); + + let child = rustfmt(&src, &rustfmt_path, paths.as_slice(), check); + children.push_back(child); + + // poll completion before waiting + for i in (0..children.len()).rev() { + if children[i](false) { + children.swap_remove_back(i); + break; + } + } + + if children.len() >= max_processes { + // await oldest child + children.pop_front().unwrap()(true); + } + } + + // await remaining children + for mut child in children { + child(true); + } + }); + + walker.run(|| { + let tx = tx.clone(); + Box::new(move |entry| { + let entry = t!(entry); + if entry.file_type().map_or(false, |t| t.is_file()) { + t!(tx.send(entry.into_path())); + } + ignore::WalkState::Continue + }) + }); + + drop(tx); + + thread.join().unwrap(); + if !check { + update_rustfmt_version(build); + } +} diff --git a/src/bootstrap/src/core/build_steps/install.rs b/src/bootstrap/src/core/build_steps/install.rs new file mode 100644 index 000000000..6b4a8f597 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/install.rs @@ -0,0 +1,332 @@ +//! Implementation of the install aspects of the compiler. +//! +//! This module is responsible for installing the standard library, +//! compiler, and documentation. + +use std::env; +use std::fs; +use std::path::{Component, Path, PathBuf}; +use std::process::Command; + +use crate::core::build_steps::dist; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::helpers::t; +use crate::utils::tarball::GeneratedTarball; +use crate::INTERNER; +use crate::{Compiler, Kind}; + +#[cfg(target_os = "illumos")] +const SHELL: &str = "bash"; +#[cfg(not(target_os = "illumos"))] +const SHELL: &str = "sh"; + +// We have to run a few shell scripts, which choke quite a bit on both `\` +// characters and on `C:\` paths, so normalize both of them away. +fn sanitize_sh(path: &Path) -> String { + let path = path.to_str().unwrap().replace("\\", "/"); + return change_drive(unc_to_lfs(&path)).unwrap_or(path); + + fn unc_to_lfs(s: &str) -> &str { + s.strip_prefix("//?/").unwrap_or(s) + } + + fn change_drive(s: &str) -> Option { + let mut ch = s.chars(); + let drive = ch.next().unwrap_or('C'); + if ch.next() != Some(':') { + return None; + } + if ch.next() != Some('/') { + return None; + } + Some(format!("/{}/{}", drive, &s[drive.len_utf8() + 2..])) + } +} + +fn is_dir_writable_for_user(dir: &PathBuf) -> bool { + let tmp = dir.join(".tmp"); + match fs::create_dir_all(&tmp) { + Ok(_) => { + fs::remove_dir_all(tmp).unwrap(); + true + } + Err(e) => { + if e.kind() == std::io::ErrorKind::PermissionDenied { + false + } else { + panic!("Failed the write access check for the current user. {}", e); + } + } + } +} + +fn install_sh( + builder: &Builder<'_>, + package: &str, + stage: u32, + host: Option, + tarball: &GeneratedTarball, +) { + let _guard = builder.msg(Kind::Install, stage, package, host, host); + + let prefix = default_path(&builder.config.prefix, "/usr/local"); + let sysconfdir = prefix.join(default_path(&builder.config.sysconfdir, "/etc")); + let destdir_env = env::var_os("DESTDIR").map(PathBuf::from); + + // Sanity checks on the write access of user. + // + // When the `DESTDIR` environment variable is present, there is no point to + // check write access for `prefix` and `sysconfdir` individually, as they + // are combined with the path from the `DESTDIR` environment variable. In + // this case, we only need to check the `DESTDIR` path, disregarding the + // `prefix` and `sysconfdir` paths. + if let Some(destdir) = &destdir_env { + assert!(is_dir_writable_for_user(destdir), "User doesn't have write access on DESTDIR."); + } else { + assert!( + is_dir_writable_for_user(&prefix), + "User doesn't have write access on `install.prefix` path in the `config.toml`.", + ); + assert!( + is_dir_writable_for_user(&sysconfdir), + "User doesn't have write access on `install.sysconfdir` path in `config.toml`." + ); + } + + let datadir = prefix.join(default_path(&builder.config.datadir, "share")); + let docdir = prefix.join(default_path(&builder.config.docdir, "share/doc/rust")); + let mandir = prefix.join(default_path(&builder.config.mandir, "share/man")); + let libdir = prefix.join(default_path(&builder.config.libdir, "lib")); + let bindir = prefix.join(&builder.config.bindir); // Default in config.rs + + let empty_dir = builder.out.join("tmp/empty_dir"); + t!(fs::create_dir_all(&empty_dir)); + + let mut cmd = Command::new(SHELL); + cmd.current_dir(&empty_dir) + .arg(sanitize_sh(&tarball.decompressed_output().join("install.sh"))) + .arg(format!("--prefix={}", prepare_dir(&destdir_env, prefix))) + .arg(format!("--sysconfdir={}", prepare_dir(&destdir_env, sysconfdir))) + .arg(format!("--datadir={}", prepare_dir(&destdir_env, datadir))) + .arg(format!("--docdir={}", prepare_dir(&destdir_env, docdir))) + .arg(format!("--bindir={}", prepare_dir(&destdir_env, bindir))) + .arg(format!("--libdir={}", prepare_dir(&destdir_env, libdir))) + .arg(format!("--mandir={}", prepare_dir(&destdir_env, mandir))) + .arg("--disable-ldconfig"); + builder.run(&mut cmd); + t!(fs::remove_dir_all(&empty_dir)); +} + +fn default_path(config: &Option, default: &str) -> PathBuf { + config.as_ref().cloned().unwrap_or_else(|| PathBuf::from(default)) +} + +fn prepare_dir(destdir_env: &Option, mut path: PathBuf) -> String { + // The DESTDIR environment variable is a standard way to install software in a subdirectory + // while keeping the original directory structure, even if the prefix or other directories + // contain absolute paths. + // + // More information on the environment variable is available here: + // https://www.gnu.org/prep/standards/html_node/DESTDIR.html + if let Some(destdir) = destdir_env { + let without_destdir = path.clone(); + path = destdir.clone(); + // Custom .join() which ignores disk roots. + for part in without_destdir.components() { + if let Component::Normal(s) = part { + path.push(s) + } + } + } + + // The installation command is not executed from the current directory, but from a temporary + // directory. To prevent relative paths from breaking this converts relative paths to absolute + // paths. std::fs::canonicalize is not used as that requires the path to actually be present. + if path.is_relative() { + path = std::env::current_dir().expect("failed to get the current directory").join(path); + assert!(path.is_absolute(), "could not make the path relative"); + } + + sanitize_sh(&path) +} + +macro_rules! install { + (($sel:ident, $builder:ident, $_config:ident), + $($name:ident, + $condition_name: ident = $path_or_alias: literal, + $default_cond:expr, + only_hosts: $only_hosts:expr, + $run_item:block $(, $c:ident)*;)+) => { + $( + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl $name { + #[allow(dead_code)] + fn should_build(config: &Config) -> bool { + config.extended && config.tools.as_ref() + .map_or(true, |t| t.contains($path_or_alias)) + } + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = $only_hosts; + $(const $c: bool = true;)* + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let $_config = &run.builder.config; + run.$condition_name($path_or_alias).default_condition($default_cond) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run($sel, $builder: &Builder<'_>) { + $run_item + } + })+ + } +} + +install!((self, builder, _config), + Docs, path = "src/doc", _config.docs, only_hosts: false, { + let tarball = builder.ensure(dist::Docs { host: self.target }).expect("missing docs"); + install_sh(builder, "docs", self.compiler.stage, Some(self.target), &tarball); + }; + Std, path = "library/std", true, only_hosts: false, { + for target in &builder.targets { + // `expect` should be safe, only None when host != build, but this + // only runs when host == build + let tarball = builder.ensure(dist::Std { + compiler: self.compiler, + target: *target + }).expect("missing std"); + install_sh(builder, "std", self.compiler.stage, Some(*target), &tarball); + } + }; + Cargo, alias = "cargo", Self::should_build(_config), only_hosts: true, { + let tarball = builder + .ensure(dist::Cargo { compiler: self.compiler, target: self.target }) + .expect("missing cargo"); + install_sh(builder, "cargo", self.compiler.stage, Some(self.target), &tarball); + }; + RustAnalyzer, alias = "rust-analyzer", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = + builder.ensure(dist::RustAnalyzer { compiler: self.compiler, target: self.target }) + { + install_sh(builder, "rust-analyzer", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install rust-analyzer stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + Clippy, alias = "clippy", Self::should_build(_config), only_hosts: true, { + let tarball = builder + .ensure(dist::Clippy { compiler: self.compiler, target: self.target }) + .expect("missing clippy"); + install_sh(builder, "clippy", self.compiler.stage, Some(self.target), &tarball); + }; + Miri, alias = "miri", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::Miri { compiler: self.compiler, target: self.target }) { + install_sh(builder, "miri", self.compiler.stage, Some(self.target), &tarball); + } else { + // Miri is only available on nightly + builder.info( + &format!("skipping Install miri stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + LlvmTools, alias = "llvm-tools", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::LlvmTools { target: self.target }) { + install_sh(builder, "llvm-tools", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping llvm-tools stage{} ({}): external LLVM", self.compiler.stage, self.target), + ); + } + }; + Rustfmt, alias = "rustfmt", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::Rustfmt { + compiler: self.compiler, + target: self.target + }) { + install_sh(builder, "rustfmt", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install Rustfmt stage{} ({})", self.compiler.stage, self.target), + ); + } + }; + RustDemangler, alias = "rust-demangler", Self::should_build(_config), only_hosts: true, { + // NOTE: Even though `should_build` may return true for `extended` default tools, + // dist::RustDemangler may still return None, unless the target-dependent `profiler` config + // is also true, or the `tools` array explicitly includes "rust-demangler". + if let Some(tarball) = builder.ensure(dist::RustDemangler { + compiler: self.compiler, + target: self.target + }) { + install_sh(builder, "rust-demangler", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install RustDemangler stage{} ({})", + self.compiler.stage, self.target), + ); + } + }; + Rustc, path = "compiler/rustc", true, only_hosts: true, { + let tarball = builder.ensure(dist::Rustc { + compiler: builder.compiler(builder.top_stage, self.target), + }); + install_sh(builder, "rustc", self.compiler.stage, Some(self.target), &tarball); + }; + RustcCodegenCranelift, alias = "rustc-codegen-cranelift", Self::should_build(_config), only_hosts: true, { + if let Some(tarball) = builder.ensure(dist::CodegenBackend { + compiler: self.compiler, + backend: INTERNER.intern_str("cranelift"), + }) { + install_sh(builder, "rustc-codegen-cranelift", self.compiler.stage, Some(self.target), &tarball); + } else { + builder.info( + &format!("skipping Install CodegenBackend(\"cranelift\") stage{} ({})", + self.compiler.stage, self.target), + ); + } + }; +); + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Src { + pub stage: u32, +} + +impl Step for Src { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let config = &run.builder.config; + let cond = config.extended && config.tools.as_ref().map_or(true, |t| t.contains("src")); + run.path("src").default_condition(cond) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Src { stage: run.builder.top_stage }); + } + + fn run(self, builder: &Builder<'_>) { + let tarball = builder.ensure(dist::Src); + install_sh(builder, "src", self.stage, None, &tarball); + } +} diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs new file mode 100644 index 000000000..a1f6fac8a --- /dev/null +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -0,0 +1,1366 @@ +//! Compilation of native dependencies like LLVM. +//! +//! Native projects like LLVM unfortunately aren't suited just yet for +//! compilation in build scripts that Cargo has. This is because the +//! compilation takes a *very* long time but also because we don't want to +//! compile LLVM 3 times as part of a normal bootstrap (we want it cached). +//! +//! LLVM and compiler-rt are essentially just wired up to everything else to +//! ensure that they're always in place if needed. + +use std::env; +use std::env::consts::EXE_EXTENSION; +use std::ffi::{OsStr, OsString}; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::Command; + +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::{Config, TargetSelection}; +use crate::utils::channel; +use crate::utils::helpers::{self, exe, get_clang_cl_resource_dir, output, t, up_to_date}; +use crate::{CLang, GitRepo, Kind}; + +use build_helper::ci::CiEnv; +use build_helper::git::get_git_merge_base; + +#[derive(Clone)] +pub struct LlvmResult { + /// Path to llvm-config binary. + /// NB: This is always the host llvm-config! + pub llvm_config: PathBuf, + /// Path to LLVM cmake directory for the target. + pub llvm_cmake_dir: PathBuf, +} + +pub struct Meta { + stamp: HashStamp, + res: LlvmResult, + out_dir: PathBuf, + root: String, +} + +// Linker flags to pass to LLVM's CMake invocation. +#[derive(Debug, Clone, Default)] +struct LdFlags { + // CMAKE_EXE_LINKER_FLAGS + exe: OsString, + // CMAKE_SHARED_LINKER_FLAGS + shared: OsString, + // CMAKE_MODULE_LINKER_FLAGS + module: OsString, +} + +impl LdFlags { + fn push_all(&mut self, s: impl AsRef) { + let s = s.as_ref(); + self.exe.push(" "); + self.exe.push(s); + self.shared.push(" "); + self.shared.push(s); + self.module.push(" "); + self.module.push(s); + } +} + +/// This returns whether we've already previously built LLVM. +/// +/// It's used to avoid busting caches during x.py check -- if we've already built +/// LLVM, it's fine for us to not try to avoid doing so. +/// +/// This will return the llvm-config if it can get it (but it will not build it +/// if not). +pub fn prebuilt_llvm_config( + builder: &Builder<'_>, + target: TargetSelection, +) -> Result { + builder.config.maybe_download_ci_llvm(); + + // If we're using a custom LLVM bail out here, but we can only use a + // custom LLVM for the build triple. + if let Some(config) = builder.config.target_config.get(&target) { + if let Some(ref s) = config.llvm_config { + check_llvm_version(builder, s); + let llvm_config = s.to_path_buf(); + let mut llvm_cmake_dir = llvm_config.clone(); + llvm_cmake_dir.pop(); + llvm_cmake_dir.pop(); + llvm_cmake_dir.push("lib"); + llvm_cmake_dir.push("cmake"); + llvm_cmake_dir.push("llvm"); + return Ok(LlvmResult { llvm_config, llvm_cmake_dir }); + } + } + + let root = "src/llvm-project/llvm"; + let out_dir = builder.llvm_out(target); + + let mut llvm_config_ret_dir = builder.llvm_out(builder.config.build); + if !builder.config.build.contains("msvc") || builder.ninja() { + llvm_config_ret_dir.push("build"); + } + llvm_config_ret_dir.push("bin"); + let build_llvm_config = llvm_config_ret_dir.join(exe("llvm-config", builder.config.build)); + let llvm_cmake_dir = out_dir.join("lib/cmake/llvm"); + let res = LlvmResult { llvm_config: build_llvm_config, llvm_cmake_dir }; + + let stamp = out_dir.join("llvm-finished-building"); + let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); + + if stamp.is_done() { + if stamp.hash.is_none() { + builder.info( + "Could not determine the LLVM submodule commit hash. \ + Assuming that an LLVM rebuild is not necessary.", + ); + builder.info(&format!( + "To force LLVM to rebuild, remove the file `{}`", + stamp.path.display() + )); + } + return Ok(res); + } + + Err(Meta { stamp, res, out_dir, root: root.into() }) +} + +/// This retrieves the LLVM sha we *want* to use, according to git history. +pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { + let llvm_sha = if is_git { + // We proceed in 2 steps. First we get the closest commit that is actually upstream. Then we + // walk back further to the last bors merge commit that actually changed LLVM. The first + // step will fail on CI because only the `auto` branch exists; we just fall back to `HEAD` + // in that case. + let closest_upstream = get_git_merge_base(&config.git_config(), Some(&config.src)) + .unwrap_or_else(|_| "HEAD".into()); + let mut rev_list = config.git(); + rev_list.args(&[ + PathBuf::from("rev-list"), + format!("--author={}", config.stage0_metadata.config.git_merge_commit_email).into(), + "-n1".into(), + "--first-parent".into(), + closest_upstream.into(), + "--".into(), + config.src.join("src/llvm-project"), + config.src.join("src/bootstrap/download-ci-llvm-stamp"), + // the LLVM shared object file is named `LLVM-12-rust-{version}-nightly` + config.src.join("src/version"), + ]); + output(&mut rev_list).trim().to_owned() + } else if let Some(info) = channel::read_commit_info_file(&config.src) { + info.sha.trim().to_owned() + } else { + "".to_owned() + }; + + if llvm_sha.is_empty() { + eprintln!("error: could not find commit hash for downloading LLVM"); + eprintln!("HELP: maybe your repository history is too shallow?"); + eprintln!("HELP: consider disabling `download-ci-llvm`"); + eprintln!("HELP: or fetch enough history to include one upstream commit"); + panic!(); + } + + llvm_sha +} + +/// Returns whether the CI-found LLVM is currently usable. +/// +/// This checks both the build triple platform to confirm we're usable at all, +/// and then verifies if the current HEAD matches the detected LLVM SHA head, +/// in which case LLVM is indicated as not available. +pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { + // This is currently all tier 1 targets and tier 2 targets with host tools + // (since others may not have CI artifacts) + // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 + let supported_platforms = [ + // tier 1 + ("aarch64-unknown-linux-gnu", false), + ("i686-pc-windows-gnu", false), + ("i686-pc-windows-msvc", false), + ("i686-unknown-linux-gnu", false), + ("x86_64-unknown-linux-gnu", true), + ("x86_64-apple-darwin", true), + ("x86_64-pc-windows-gnu", true), + ("x86_64-pc-windows-msvc", true), + // tier 2 with host tools + ("aarch64-apple-darwin", false), + ("aarch64-pc-windows-msvc", false), + ("aarch64-unknown-linux-musl", false), + ("arm-unknown-linux-gnueabi", false), + ("arm-unknown-linux-gnueabihf", false), + ("armv7-unknown-linux-gnueabihf", false), + ("loongarch64-unknown-linux-gnu", false), + ("mips-unknown-linux-gnu", false), + ("mips64-unknown-linux-gnuabi64", false), + ("mips64el-unknown-linux-gnuabi64", false), + ("mipsel-unknown-linux-gnu", false), + ("powerpc-unknown-linux-gnu", false), + ("powerpc64-unknown-linux-gnu", false), + ("powerpc64le-unknown-linux-gnu", false), + ("riscv64gc-unknown-linux-gnu", false), + ("s390x-unknown-linux-gnu", false), + ("x86_64-unknown-freebsd", false), + ("x86_64-unknown-illumos", false), + ("x86_64-unknown-linux-musl", false), + ("x86_64-unknown-netbsd", false), + ]; + + if !supported_platforms.contains(&(&*config.build.triple, asserts)) + && (asserts || !supported_platforms.contains(&(&*config.build.triple, true))) + { + return false; + } + + if is_ci_llvm_modified(config) { + eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); + return false; + } + + true +} + +/// Returns true if we're running in CI with modified LLVM (and thus can't download it) +pub(crate) fn is_ci_llvm_modified(config: &Config) -> bool { + CiEnv::is_ci() && config.rust_info.is_managed_git_subrepository() && { + // We assume we have access to git, so it's okay to unconditionally pass + // `true` here. + let llvm_sha = detect_llvm_sha(config, true); + let head_sha = output(config.git().arg("rev-parse").arg("HEAD")); + let head_sha = head_sha.trim(); + llvm_sha == head_sha + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Llvm { + pub target: TargetSelection, +} + +impl Step for Llvm { + type Output = LlvmResult; + + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project").path("src/llvm-project/llvm") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Llvm { target: run.target }); + } + + /// Compile LLVM for `target`. + fn run(self, builder: &Builder<'_>) -> LlvmResult { + let target = self.target; + let target_native = if self.target.starts_with("riscv") { + // RISC-V target triples in Rust is not named the same as C compiler target triples. + // This converts Rust RISC-V target triples to C compiler triples. + let idx = target.triple.find('-').unwrap(); + + format!("riscv{}{}", &target.triple[5..7], &target.triple[idx..]) + } else if self.target.starts_with("powerpc") && self.target.ends_with("freebsd") { + // FreeBSD 13 had incompatible ABI changes on all PowerPC platforms. + // Set the version suffix to 13.0 so the correct target details are used. + format!("{}{}", self.target, "13.0") + } else { + target.to_string() + }; + + let Meta { stamp, res, out_dir, root } = match prebuilt_llvm_config(builder, target) { + Ok(p) => return p, + Err(m) => m, + }; + + builder.update_submodule(&Path::new("src").join("llvm-project")); + if builder.llvm_link_shared() && target.contains("windows") { + panic!("shared linking to LLVM is not currently supported on {}", target.triple); + } + + let _guard = builder.msg_unstaged(Kind::Build, "LLVM", target); + t!(stamp.remove()); + let _time = helpers::timeit(&builder); + t!(fs::create_dir_all(&out_dir)); + + // https://llvm.org/docs/CMake.html + let mut cfg = cmake::Config::new(builder.src.join(root)); + let mut ldflags = LdFlags::default(); + + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + + // NOTE: remember to also update `config.example.toml` when changing the + // defaults! + let llvm_targets = match &builder.config.llvm_targets { + Some(s) => s, + None => { + "AArch64;ARM;BPF;Hexagon;LoongArch;MSP430;Mips;NVPTX;PowerPC;RISCV;\ + Sparc;SystemZ;WebAssembly;X86" + } + }; + + let llvm_exp_targets = match builder.config.llvm_experimental_targets { + Some(ref s) => s, + None => "AVR;M68k;CSKY", + }; + + let assertions = if builder.config.llvm_assertions { "ON" } else { "OFF" }; + let plugins = if builder.config.llvm_plugins { "ON" } else { "OFF" }; + let enable_tests = if builder.config.llvm_tests { "ON" } else { "OFF" }; + let enable_warnings = if builder.config.llvm_enable_warnings { "ON" } else { "OFF" }; + + cfg.out_dir(&out_dir) + .profile(profile) + .define("LLVM_ENABLE_ASSERTIONS", assertions) + .define("LLVM_UNREACHABLE_OPTIMIZE", "OFF") + .define("LLVM_ENABLE_PLUGINS", plugins) + .define("LLVM_TARGETS_TO_BUILD", llvm_targets) + .define("LLVM_EXPERIMENTAL_TARGETS_TO_BUILD", llvm_exp_targets) + .define("LLVM_INCLUDE_EXAMPLES", "OFF") + .define("LLVM_INCLUDE_DOCS", "OFF") + .define("LLVM_INCLUDE_BENCHMARKS", "OFF") + .define("LLVM_INCLUDE_TESTS", enable_tests) + .define("LLVM_ENABLE_TERMINFO", "OFF") + .define("LLVM_ENABLE_LIBEDIT", "OFF") + .define("LLVM_ENABLE_BINDINGS", "OFF") + .define("LLVM_ENABLE_Z3_SOLVER", "OFF") + .define("LLVM_PARALLEL_COMPILE_JOBS", builder.jobs().to_string()) + .define("LLVM_TARGET_ARCH", target_native.split('-').next().unwrap()) + .define("LLVM_DEFAULT_TARGET_TRIPLE", target_native) + .define("LLVM_ENABLE_WARNINGS", enable_warnings); + + // Parts of our test suite rely on the `FileCheck` tool, which is built by default in + // `build/$TARGET/llvm/build/bin` is but *not* then installed to `build/$TARGET/llvm/bin`. + // This flag makes sure `FileCheck` is copied in the final binaries directory. + cfg.define("LLVM_INSTALL_UTILS", "ON"); + + if builder.config.llvm_profile_generate { + cfg.define("LLVM_BUILD_INSTRUMENTED", "IR"); + if let Ok(llvm_profile_dir) = std::env::var("LLVM_PROFILE_DIR") { + cfg.define("LLVM_PROFILE_DATA_DIR", llvm_profile_dir); + } + cfg.define("LLVM_BUILD_RUNTIME", "No"); + } + if let Some(path) = builder.config.llvm_profile_use.as_ref() { + cfg.define("LLVM_PROFDATA_FILE", &path); + } + + // Disable zstd to avoid a dependency on libzstd.so. + cfg.define("LLVM_ENABLE_ZSTD", "OFF"); + + if !target.contains("windows") { + cfg.define("LLVM_ENABLE_ZLIB", "ON"); + } else { + cfg.define("LLVM_ENABLE_ZLIB", "OFF"); + } + + // Are we compiling for iOS/tvOS/watchOS? + if target.contains("apple-ios") + || target.contains("apple-tvos") + || target.contains("apple-watchos") + { + // These two defines prevent CMake from automatically trying to add a MacOSX sysroot, which leads to a compiler error. + cfg.define("CMAKE_OSX_SYSROOT", "/"); + cfg.define("CMAKE_OSX_DEPLOYMENT_TARGET", ""); + // Prevent cmake from adding -bundle to CFLAGS automatically, which leads to a compiler error because "-bitcode_bundle" also gets added. + cfg.define("LLVM_ENABLE_PLUGINS", "OFF"); + // Zlib fails to link properly, leading to a compiler error. + cfg.define("LLVM_ENABLE_ZLIB", "OFF"); + } + + // This setting makes the LLVM tools link to the dynamic LLVM library, + // which saves both memory during parallel links and overall disk space + // for the tools. We don't do this on every platform as it doesn't work + // equally well everywhere. + if builder.llvm_link_shared() { + cfg.define("LLVM_LINK_LLVM_DYLIB", "ON"); + } + + if (target.starts_with("riscv") || target.starts_with("csky")) + && !target.contains("freebsd") + && !target.contains("openbsd") + && !target.contains("netbsd") + { + // RISC-V and CSKY GCC erroneously requires linking against + // `libatomic` when using 1-byte and 2-byte C++ + // atomics but the LLVM build system check cannot + // detect this. Therefore it is set manually here. + // Some BSD uses Clang as its system compiler and + // provides no libatomic in its base system so does + // not want this. + ldflags.exe.push(" -latomic"); + ldflags.shared.push(" -latomic"); + } + + if target.starts_with("mips") && target.contains("netbsd") { + // LLVM wants 64-bit atomics, while mipsel is 32-bit only, so needs -latomic + ldflags.exe.push(" -latomic"); + ldflags.shared.push(" -latomic"); + } + + if target.contains("msvc") { + cfg.define("LLVM_USE_CRT_DEBUG", "MT"); + cfg.define("LLVM_USE_CRT_RELEASE", "MT"); + cfg.define("LLVM_USE_CRT_RELWITHDEBINFO", "MT"); + cfg.static_crt(true); + } + + if target.starts_with("i686") { + cfg.define("LLVM_BUILD_32_BITS", "ON"); + } + + let mut enabled_llvm_projects = Vec::new(); + + if helpers::forcing_clang_based_tests() { + enabled_llvm_projects.push("clang"); + enabled_llvm_projects.push("compiler-rt"); + } + + if builder.config.llvm_polly { + enabled_llvm_projects.push("polly"); + } + + if builder.config.llvm_clang { + enabled_llvm_projects.push("clang"); + } + + // We want libxml to be disabled. + // See https://github.com/rust-lang/rust/pull/50104 + cfg.define("LLVM_ENABLE_LIBXML2", "OFF"); + + if !enabled_llvm_projects.is_empty() { + enabled_llvm_projects.sort(); + enabled_llvm_projects.dedup(); + cfg.define("LLVM_ENABLE_PROJECTS", enabled_llvm_projects.join(";")); + } + + if let Some(num_linkers) = builder.config.llvm_link_jobs { + if num_linkers > 0 { + cfg.define("LLVM_PARALLEL_LINK_JOBS", num_linkers.to_string()); + } + } + + // https://llvm.org/docs/HowToCrossCompileLLVM.html + if target != builder.config.build { + let LlvmResult { llvm_config, .. } = + builder.ensure(Llvm { target: builder.config.build }); + if !builder.config.dry_run() { + let llvm_bindir = output(Command::new(&llvm_config).arg("--bindir")); + let host_bin = Path::new(llvm_bindir.trim()); + cfg.define( + "LLVM_TABLEGEN", + host_bin.join("llvm-tblgen").with_extension(EXE_EXTENSION), + ); + // LLVM_NM is required for cross compiling using MSVC + cfg.define("LLVM_NM", host_bin.join("llvm-nm").with_extension(EXE_EXTENSION)); + } + cfg.define("LLVM_CONFIG_PATH", llvm_config); + if builder.config.llvm_clang { + let build_bin = builder.llvm_out(builder.config.build).join("build").join("bin"); + let clang_tblgen = build_bin.join("clang-tblgen").with_extension(EXE_EXTENSION); + if !builder.config.dry_run() && !clang_tblgen.exists() { + panic!("unable to find {}", clang_tblgen.display()); + } + cfg.define("CLANG_TABLEGEN", clang_tblgen); + } + } + + let llvm_version_suffix = if let Some(ref suffix) = builder.config.llvm_version_suffix { + // Allow version-suffix="" to not define a version suffix at all. + if !suffix.is_empty() { Some(suffix.to_string()) } else { None } + } else if builder.config.channel == "dev" { + // Changes to a version suffix require a complete rebuild of the LLVM. + // To avoid rebuilds during a time of version bump, don't include rustc + // release number on the dev channel. + Some("-rust-dev".to_string()) + } else { + Some(format!("-rust-{}-{}", builder.version, builder.config.channel)) + }; + if let Some(ref suffix) = llvm_version_suffix { + cfg.define("LLVM_VERSION_SUFFIX", suffix); + } + + configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); + configure_llvm(builder, target, &mut cfg); + + for (key, val) in &builder.config.llvm_build_config { + cfg.define(key, val); + } + + if builder.config.dry_run() { + return res; + } + + cfg.build(); + + // Helper to find the name of LLVM's shared library on darwin and linux. + let find_llvm_lib_name = |extension| { + let mut cmd = Command::new(&res.llvm_config); + let version = output(cmd.arg("--version")); + let major = version.split('.').next().unwrap(); + + match &llvm_version_suffix { + Some(version_suffix) => format!("libLLVM-{major}{version_suffix}.{extension}"), + None => format!("libLLVM-{major}.{extension}"), + } + }; + + // When building LLVM with LLVM_LINK_LLVM_DYLIB for macOS, an unversioned + // libLLVM.dylib will be built. However, llvm-config will still look + // for a versioned path like libLLVM-14.dylib. Manually create a symbolic + // link to make llvm-config happy. + if builder.llvm_link_shared() && target.contains("apple-darwin") { + let lib_name = find_llvm_lib_name("dylib"); + let lib_llvm = out_dir.join("build").join("lib").join(lib_name); + if !lib_llvm.exists() { + t!(builder.symlink_file("libLLVM.dylib", &lib_llvm)); + } + } + + // When building LLVM as a shared library on linux, it can contain unexpected debuginfo: + // some can come from the C++ standard library. Unless we're explicitly requesting LLVM to + // be built with debuginfo, strip it away after the fact, to make dist artifacts smaller. + if builder.llvm_link_shared() + && builder.config.llvm_optimize + && !builder.config.llvm_release_debuginfo + { + // Find the name of the LLVM shared library that we just built. + let lib_name = find_llvm_lib_name("so"); + + // If the shared library exists in LLVM's `/build/lib/` or `/lib/` folders, strip its + // debuginfo. + crate::core::build_steps::compile::strip_debug( + builder, + target, + &out_dir.join("lib").join(&lib_name), + ); + crate::core::build_steps::compile::strip_debug( + builder, + target, + &out_dir.join("build").join("lib").join(&lib_name), + ); + } + + t!(stamp.write()); + + res + } +} + +fn check_llvm_version(builder: &Builder<'_>, llvm_config: &Path) { + if builder.config.dry_run() { + return; + } + + let mut cmd = Command::new(llvm_config); + let version = output(cmd.arg("--version")); + let mut parts = version.split('.').take(2).filter_map(|s| s.parse::().ok()); + if let (Some(major), Some(_minor)) = (parts.next(), parts.next()) { + if major >= 15 { + return; + } + } + panic!("\n\nbad LLVM version: {version}, need >=15.0\n\n") +} + +fn configure_cmake( + builder: &Builder<'_>, + target: TargetSelection, + cfg: &mut cmake::Config, + use_compiler_launcher: bool, + mut ldflags: LdFlags, + extra_compiler_flags: &[&str], +) { + // Do not print installation messages for up-to-date files. + // LLVM and LLD builds can produce a lot of those and hit CI limits on log size. + cfg.define("CMAKE_INSTALL_MESSAGE", "LAZY"); + + // Do not allow the user's value of DESTDIR to influence where + // LLVM will install itself. LLVM must always be installed in our + // own build directories. + cfg.env("DESTDIR", ""); + + if builder.ninja() { + cfg.generator("Ninja"); + } + cfg.target(&target.triple).host(&builder.config.build.triple); + + if target != builder.config.build { + cfg.define("CMAKE_CROSSCOMPILING", "True"); + + if target.contains("netbsd") { + cfg.define("CMAKE_SYSTEM_NAME", "NetBSD"); + } else if target.contains("dragonfly") { + cfg.define("CMAKE_SYSTEM_NAME", "DragonFly"); + } else if target.contains("freebsd") { + cfg.define("CMAKE_SYSTEM_NAME", "FreeBSD"); + } else if target.contains("windows") { + cfg.define("CMAKE_SYSTEM_NAME", "Windows"); + } else if target.contains("haiku") { + cfg.define("CMAKE_SYSTEM_NAME", "Haiku"); + } else if target.contains("solaris") || target.contains("illumos") { + cfg.define("CMAKE_SYSTEM_NAME", "SunOS"); + } else if target.contains("linux") { + cfg.define("CMAKE_SYSTEM_NAME", "Linux"); + } else { + builder.info(&format!( + "could not determine CMAKE_SYSTEM_NAME from the target `{target}`, build may fail", + )); + } + + // When cross-compiling we should also set CMAKE_SYSTEM_VERSION, but in + // that case like CMake we cannot easily determine system version either. + // + // Since, the LLVM itself makes rather limited use of version checks in + // CMakeFiles (and then only in tests), and so far no issues have been + // reported, the system version is currently left unset. + + if target.contains("darwin") { + // Make sure that CMake does not build universal binaries on macOS. + // Explicitly specify the one single target architecture. + if target.starts_with("aarch64") { + // macOS uses a different name for building arm64 + cfg.define("CMAKE_OSX_ARCHITECTURES", "arm64"); + } else if target.starts_with("i686") { + // macOS uses a different name for building i386 + cfg.define("CMAKE_OSX_ARCHITECTURES", "i386"); + } else { + cfg.define("CMAKE_OSX_ARCHITECTURES", target.triple.split('-').next().unwrap()); + } + } + } + + let sanitize_cc = |cc: &Path| { + if target.contains("msvc") { + OsString::from(cc.to_str().unwrap().replace("\\", "/")) + } else { + cc.as_os_str().to_owned() + } + }; + + // MSVC with CMake uses msbuild by default which doesn't respect these + // vars that we'd otherwise configure. In that case we just skip this + // entirely. + if target.contains("msvc") && !builder.ninja() { + return; + } + + let (cc, cxx) = match builder.config.llvm_clang_cl { + Some(ref cl) => (cl.into(), cl.into()), + None => (builder.cc(target), builder.cxx(target).unwrap()), + }; + + // Handle msvc + ninja + ccache specially (this is what the bots use) + if target.contains("msvc") && builder.ninja() && builder.config.ccache.is_some() { + let mut wrap_cc = env::current_exe().expect("failed to get cwd"); + wrap_cc.set_file_name("sccache-plus-cl.exe"); + + cfg.define("CMAKE_C_COMPILER", sanitize_cc(&wrap_cc)) + .define("CMAKE_CXX_COMPILER", sanitize_cc(&wrap_cc)); + cfg.env("SCCACHE_PATH", builder.config.ccache.as_ref().unwrap()) + .env("SCCACHE_TARGET", target.triple) + .env("SCCACHE_CC", &cc) + .env("SCCACHE_CXX", &cxx); + + // Building LLVM on MSVC can be a little ludicrous at times. We're so far + // off the beaten path here that I'm not really sure this is even half + // supported any more. Here we're trying to: + // + // * Build LLVM on MSVC + // * Build LLVM with `clang-cl` instead of `cl.exe` + // * Build a project with `sccache` + // * Build for 32-bit as well + // * Build with Ninja + // + // For `cl.exe` there are different binaries to compile 32/64 bit which + // we use but for `clang-cl` there's only one which internally + // multiplexes via flags. As a result it appears that CMake's detection + // of a compiler's architecture and such on MSVC **doesn't** pass any + // custom flags we pass in CMAKE_CXX_FLAGS below. This means that if we + // use `clang-cl.exe` it's always diagnosed as a 64-bit compiler which + // definitely causes problems since all the env vars are pointing to + // 32-bit libraries. + // + // To hack around this... again... we pass an argument that's + // unconditionally passed in the sccache shim. This'll get CMake to + // correctly diagnose it's doing a 32-bit compilation and LLVM will + // internally configure itself appropriately. + if builder.config.llvm_clang_cl.is_some() && target.contains("i686") { + cfg.env("SCCACHE_EXTRA_ARGS", "-m32"); + } + } else { + // If ccache is configured we inform the build a little differently how + // to invoke ccache while also invoking our compilers. + if use_compiler_launcher { + if let Some(ref ccache) = builder.config.ccache { + cfg.define("CMAKE_C_COMPILER_LAUNCHER", ccache) + .define("CMAKE_CXX_COMPILER_LAUNCHER", ccache); + } + } + cfg.define("CMAKE_C_COMPILER", sanitize_cc(&cc)) + .define("CMAKE_CXX_COMPILER", sanitize_cc(&cxx)) + .define("CMAKE_ASM_COMPILER", sanitize_cc(&cc)); + } + + cfg.build_arg("-j").build_arg(builder.jobs().to_string()); + let mut cflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::C).join(" ").into(); + if let Some(ref s) = builder.config.llvm_cflags { + cflags.push(" "); + cflags.push(s); + } + + if builder.config.llvm_clang_cl.is_some() { + cflags.push(&format!(" --target={target}")); + } + for flag in extra_compiler_flags { + cflags.push(&format!(" {flag}")); + } + cfg.define("CMAKE_C_FLAGS", cflags); + let mut cxxflags: OsString = builder.cflags(target, GitRepo::Llvm, CLang::Cxx).join(" ").into(); + if let Some(ref s) = builder.config.llvm_cxxflags { + cxxflags.push(" "); + cxxflags.push(s); + } + if builder.config.llvm_clang_cl.is_some() { + cxxflags.push(&format!(" --target={target}")); + } + for flag in extra_compiler_flags { + cxxflags.push(&format!(" {flag}")); + } + cfg.define("CMAKE_CXX_FLAGS", cxxflags); + if let Some(ar) = builder.ar(target) { + if ar.is_absolute() { + // LLVM build breaks if `CMAKE_AR` is a relative path, for some reason it + // tries to resolve this path in the LLVM build directory. + cfg.define("CMAKE_AR", sanitize_cc(&ar)); + } + } + + if let Some(ranlib) = builder.ranlib(target) { + if ranlib.is_absolute() { + // LLVM build breaks if `CMAKE_RANLIB` is a relative path, for some reason it + // tries to resolve this path in the LLVM build directory. + cfg.define("CMAKE_RANLIB", sanitize_cc(&ranlib)); + } + } + + if let Some(ref flags) = builder.config.llvm_ldflags { + ldflags.push_all(flags); + } + + if let Some(flags) = get_var("LDFLAGS", &builder.config.build.triple, &target.triple) { + ldflags.push_all(&flags); + } + + // For distribution we want the LLVM tools to be *statically* linked to libstdc++. + // We also do this if the user explicitly requested static libstdc++. + if builder.config.llvm_static_stdcpp + && !target.contains("msvc") + && !target.contains("netbsd") + && !target.contains("solaris") + { + if target.contains("apple") || target.contains("windows") { + ldflags.push_all("-static-libstdc++"); + } else { + ldflags.push_all("-Wl,-Bsymbolic -static-libstdc++"); + } + } + + cfg.define("CMAKE_SHARED_LINKER_FLAGS", &ldflags.shared); + cfg.define("CMAKE_MODULE_LINKER_FLAGS", &ldflags.module); + cfg.define("CMAKE_EXE_LINKER_FLAGS", &ldflags.exe); + + if env::var_os("SCCACHE_ERROR_LOG").is_some() { + cfg.env("RUSTC_LOG", "sccache=warn"); + } +} + +fn configure_llvm(builder: &Builder<'_>, target: TargetSelection, cfg: &mut cmake::Config) { + // ThinLTO is only available when building with LLVM, enabling LLD is required. + // Apple's linker ld64 supports ThinLTO out of the box though, so don't use LLD on Darwin. + if builder.config.llvm_thin_lto { + cfg.define("LLVM_ENABLE_LTO", "Thin"); + if !target.contains("apple") { + cfg.define("LLVM_ENABLE_LLD", "ON"); + } + } + + if let Some(ref linker) = builder.config.llvm_use_linker { + cfg.define("LLVM_USE_LINKER", linker); + } + + if builder.config.llvm_allow_old_toolchain { + cfg.define("LLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN", "YES"); + } +} + +// Adapted from https://github.com/alexcrichton/cc-rs/blob/fba7feded71ee4f63cfe885673ead6d7b4f2f454/src/lib.rs#L2347-L2365 +fn get_var(var_base: &str, host: &str, target: &str) -> Option { + let kind = if host == target { "HOST" } else { "TARGET" }; + let target_u = target.replace("-", "_"); + env::var_os(&format!("{var_base}_{target}")) + .or_else(|| env::var_os(&format!("{}_{}", var_base, target_u))) + .or_else(|| env::var_os(&format!("{}_{}", kind, var_base))) + .or_else(|| env::var_os(var_base)) +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Lld { + pub target: TargetSelection, +} + +impl Step for Lld { + type Output = PathBuf; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/lld") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Lld { target: run.target }); + } + + /// Compile LLD for `target`. + fn run(self, builder: &Builder<'_>) -> PathBuf { + if builder.config.dry_run() { + return PathBuf::from("lld-out-dir-test-gen"); + } + let target = self.target; + + let LlvmResult { llvm_config, llvm_cmake_dir } = builder.ensure(Llvm { target }); + + // The `dist` step packages LLD next to LLVM's binaries for download-ci-llvm. The root path + // we usually expect here is `./build/$triple/ci-llvm/`, with the binaries in its `bin` + // subfolder. We check if that's the case, and if LLD's binary already exists there next to + // `llvm-config`: if so, we can use it instead of building LLVM/LLD from source. + let ci_llvm_bin = llvm_config.parent().unwrap(); + if ci_llvm_bin.is_dir() && ci_llvm_bin.file_name().unwrap() == "bin" { + let lld_path = ci_llvm_bin.join(exe("lld", target)); + if lld_path.exists() { + // The following steps copying `lld` as `rust-lld` to the sysroot, expect it in the + // `bin` subfolder of this step's out dir. + return ci_llvm_bin.parent().unwrap().to_path_buf(); + } + } + + let out_dir = builder.lld_out(target); + let done_stamp = out_dir.join("lld-finished-building"); + if done_stamp.exists() { + return out_dir; + } + + let _guard = builder.msg_unstaged(Kind::Build, "LLD", target); + let _time = helpers::timeit(&builder); + t!(fs::create_dir_all(&out_dir)); + + let mut cfg = cmake::Config::new(builder.src.join("src/llvm-project/lld")); + let mut ldflags = LdFlags::default(); + + // When building LLD as part of a build with instrumentation on windows, for example + // when doing PGO on CI, cmake or clang-cl don't automatically link clang's + // profiler runtime in. In that case, we need to manually ask cmake to do it, to avoid + // linking errors, much like LLVM's cmake setup does in that situation. + if builder.config.llvm_profile_generate && target.contains("msvc") { + if let Some(clang_cl_path) = builder.config.llvm_clang_cl.as_ref() { + // Find clang's runtime library directory and push that as a search path to the + // cmake linker flags. + let clang_rt_dir = get_clang_cl_resource_dir(clang_cl_path); + ldflags.push_all(&format!("/libpath:{}", clang_rt_dir.display())); + } + } + + // LLD is built as an LLVM tool, but is distributed outside of the `llvm-tools` component, + // which impacts where it expects to find LLVM's shared library. This causes #80703. + // + // LLD is distributed at "$root/lib/rustlib/$host/bin/rust-lld", but the `libLLVM-*.so` it + // needs is distributed at "$root/lib". The default rpath of "$ORIGIN/../lib" points at the + // lib path for LLVM tools, not the one for rust binaries. + // + // (The `llvm-tools` component copies the .so there for the other tools, and with that + // component installed, one can successfully invoke `rust-lld` directly without rustup's + // `LD_LIBRARY_PATH` overrides) + // + if builder.config.rpath_enabled(target) + && helpers::use_host_linker(target) + && builder.config.llvm_link_shared() + && target.contains("linux") + { + // So we inform LLD where it can find LLVM's libraries by adding an rpath entry to the + // expected parent `lib` directory. + // + // Be careful when changing this path, we need to ensure it's quoted or escaped: + // `$ORIGIN` would otherwise be expanded when the `LdFlags` are passed verbatim to + // cmake. + ldflags.push_all("-Wl,-rpath,'$ORIGIN/../../../'"); + } + + configure_cmake(builder, target, &mut cfg, true, ldflags, &[]); + configure_llvm(builder, target, &mut cfg); + + // Re-use the same flags as llvm to control the level of debug information + // generated for lld. + let profile = match (builder.config.llvm_optimize, builder.config.llvm_release_debuginfo) { + (false, _) => "Debug", + (true, false) => "Release", + (true, true) => "RelWithDebInfo", + }; + + cfg.out_dir(&out_dir) + .profile(profile) + .define("LLVM_CMAKE_DIR", llvm_cmake_dir) + .define("LLVM_INCLUDE_TESTS", "OFF"); + + if target != builder.config.build { + // Use the host llvm-tblgen binary. + cfg.define( + "LLVM_TABLEGEN_EXE", + llvm_config.with_file_name("llvm-tblgen").with_extension(EXE_EXTENSION), + ); + } + + cfg.build(); + + t!(File::create(&done_stamp)); + out_dir + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Sanitizers { + pub target: TargetSelection, +} + +impl Step for Sanitizers { + type Output = Vec; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("sanitizers") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Sanitizers { target: run.target }); + } + + /// Builds sanitizer runtime libraries. + fn run(self, builder: &Builder<'_>) -> Self::Output { + let compiler_rt_dir = builder.src.join("src/llvm-project/compiler-rt"); + if !compiler_rt_dir.exists() { + return Vec::new(); + } + + let out_dir = builder.native_dir(self.target).join("sanitizers"); + let runtimes = supported_sanitizers(&out_dir, self.target, &builder.config.channel); + if runtimes.is_empty() { + return runtimes; + } + + let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: builder.config.build }); + if builder.config.dry_run() { + return runtimes; + } + + let stamp = out_dir.join("sanitizers-finished-building"); + let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); + + if stamp.is_done() { + if stamp.hash.is_none() { + builder.info(&format!( + "Rebuild sanitizers by removing the file `{}`", + stamp.path.display() + )); + } + return runtimes; + } + + let _guard = builder.msg_unstaged(Kind::Build, "sanitizers", self.target); + t!(stamp.remove()); + let _time = helpers::timeit(&builder); + + let mut cfg = cmake::Config::new(&compiler_rt_dir); + cfg.profile("Release"); + cfg.define("CMAKE_C_COMPILER_TARGET", self.target.triple); + cfg.define("COMPILER_RT_BUILD_BUILTINS", "OFF"); + cfg.define("COMPILER_RT_BUILD_CRT", "OFF"); + cfg.define("COMPILER_RT_BUILD_LIBFUZZER", "OFF"); + cfg.define("COMPILER_RT_BUILD_PROFILE", "OFF"); + cfg.define("COMPILER_RT_BUILD_SANITIZERS", "ON"); + cfg.define("COMPILER_RT_BUILD_XRAY", "OFF"); + cfg.define("COMPILER_RT_DEFAULT_TARGET_ONLY", "ON"); + cfg.define("COMPILER_RT_USE_LIBCXX", "OFF"); + cfg.define("LLVM_CONFIG_PATH", &llvm_config); + + // On Darwin targets the sanitizer runtimes are build as universal binaries. + // Unfortunately sccache currently lacks support to build them successfully. + // Disable compiler launcher on Darwin targets to avoid potential issues. + let use_compiler_launcher = !self.target.contains("apple-darwin"); + let extra_compiler_flags: &[&str] = + if self.target.contains("apple") { &["-fembed-bitcode=off"] } else { &[] }; + configure_cmake( + builder, + self.target, + &mut cfg, + use_compiler_launcher, + LdFlags::default(), + extra_compiler_flags, + ); + + t!(fs::create_dir_all(&out_dir)); + cfg.out_dir(out_dir); + + for runtime in &runtimes { + cfg.build_target(&runtime.cmake_target); + cfg.build(); + } + t!(stamp.write()); + + runtimes + } +} + +#[derive(Clone, Debug)] +pub struct SanitizerRuntime { + /// CMake target used to build the runtime. + pub cmake_target: String, + /// Path to the built runtime library. + pub path: PathBuf, + /// Library filename that will be used rustc. + pub name: String, +} + +/// Returns sanitizers available on a given target. +fn supported_sanitizers( + out_dir: &Path, + target: TargetSelection, + channel: &str, +) -> Vec { + let darwin_libs = |os: &str, components: &[&str]| -> Vec { + components + .iter() + .map(move |c| SanitizerRuntime { + cmake_target: format!("clang_rt.{}_{}_dynamic", c, os), + path: out_dir + .join(&format!("build/lib/darwin/libclang_rt.{}_{}_dynamic.dylib", c, os)), + name: format!("librustc-{}_rt.{}.dylib", channel, c), + }) + .collect() + }; + + let common_libs = |os: &str, arch: &str, components: &[&str]| -> Vec { + components + .iter() + .map(move |c| SanitizerRuntime { + cmake_target: format!("clang_rt.{}-{}", c, arch), + path: out_dir.join(&format!("build/lib/{}/libclang_rt.{}-{}.a", os, c, arch)), + name: format!("librustc-{}_rt.{}.a", channel, c), + }) + .collect() + }; + + match &*target.triple { + "aarch64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "aarch64-apple-ios" => darwin_libs("ios", &["asan", "tsan"]), + "aarch64-apple-ios-sim" => darwin_libs("iossim", &["asan", "tsan"]), + "aarch64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "aarch64-unknown-fuchsia" => common_libs("fuchsia", "aarch64", &["asan"]), + "aarch64-unknown-linux-gnu" => { + common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) + } + "aarch64-unknown-linux-ohos" => { + common_libs("linux", "aarch64", &["asan", "lsan", "msan", "tsan", "hwasan"]) + } + "x86_64-apple-darwin" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "x86_64-unknown-fuchsia" => common_libs("fuchsia", "x86_64", &["asan"]), + "x86_64-apple-ios" => darwin_libs("iossim", &["asan", "tsan"]), + "x86_64-apple-ios-macabi" => darwin_libs("osx", &["asan", "lsan", "tsan"]), + "x86_64-unknown-freebsd" => common_libs("freebsd", "x86_64", &["asan", "msan", "tsan"]), + "x86_64-unknown-netbsd" => { + common_libs("netbsd", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } + "x86_64-unknown-illumos" => common_libs("illumos", "x86_64", &["asan"]), + "x86_64-pc-solaris" => common_libs("solaris", "x86_64", &["asan"]), + "x86_64-unknown-linux-gnu" => { + common_libs("linux", "x86_64", &["asan", "lsan", "msan", "safestack", "tsan"]) + } + "x86_64-unknown-linux-musl" => { + common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } + "s390x-unknown-linux-gnu" => { + common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) + } + "s390x-unknown-linux-musl" => { + common_libs("linux", "s390x", &["asan", "lsan", "msan", "tsan"]) + } + "x86_64-unknown-linux-ohos" => { + common_libs("linux", "x86_64", &["asan", "lsan", "msan", "tsan"]) + } + _ => Vec::new(), + } +} + +struct HashStamp { + path: PathBuf, + hash: Option>, +} + +impl HashStamp { + fn new(path: PathBuf, hash: Option<&str>) -> Self { + HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) } + } + + fn is_done(&self) -> bool { + match fs::read(&self.path) { + Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(), + Err(e) if e.kind() == io::ErrorKind::NotFound => false, + Err(e) => { + panic!("failed to read stamp file `{}`: {}", self.path.display(), e); + } + } + } + + fn remove(&self) -> io::Result<()> { + match fs::remove_file(&self.path) { + Ok(()) => Ok(()), + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + Ok(()) + } else { + Err(e) + } + } + } + } + + fn write(&self) -> io::Result<()> { + fs::write(&self.path, self.hash.as_deref().unwrap_or(b"")) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrtBeginEnd { + pub target: TargetSelection, +} + +impl Step for CrtBeginEnd { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/compiler-rt/lib/crt") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CrtBeginEnd { target: run.target }); + } + + /// Build crtbegin.o/crtend.o for musl target. + fn run(self, builder: &Builder<'_>) -> Self::Output { + builder.update_submodule(&Path::new("src/llvm-project")); + + let out_dir = builder.native_dir(self.target).join("crt"); + + if builder.config.dry_run() { + return out_dir; + } + + let crtbegin_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtbegin.c"); + let crtend_src = builder.src.join("src/llvm-project/compiler-rt/lib/builtins/crtend.c"); + if up_to_date(&crtbegin_src, &out_dir.join("crtbegin.o")) + && up_to_date(&crtend_src, &out_dir.join("crtendS.o")) + { + return out_dir; + } + + let _guard = builder.msg_unstaged(Kind::Build, "crtbegin.o and crtend.o", self.target); + t!(fs::create_dir_all(&out_dir)); + + let mut cfg = cc::Build::new(); + + if let Some(ar) = builder.ar(self.target) { + cfg.archiver(ar); + } + cfg.compiler(builder.cc(self.target)); + cfg.cargo_metadata(false) + .out_dir(&out_dir) + .target(&self.target.triple) + .host(&builder.config.build.triple) + .warnings(false) + .debug(false) + .opt_level(3) + .file(crtbegin_src) + .file(crtend_src); + + // Those flags are defined in src/llvm-project/compiler-rt/lib/crt/CMakeLists.txt + // Currently only consumer of those objects is musl, which use .init_array/.fini_array + // instead of .ctors/.dtors + cfg.flag("-std=c11") + .define("CRT_HAS_INITFINI_ARRAY", None) + .define("EH_USE_FRAME_REGISTRY", None); + + cfg.compile("crt"); + + t!(fs::copy(out_dir.join("crtbegin.o"), out_dir.join("crtbeginS.o"))); + t!(fs::copy(out_dir.join("crtend.o"), out_dir.join("crtendS.o"))); + out_dir + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Libunwind { + pub target: TargetSelection, +} + +impl Step for Libunwind { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/llvm-project/libunwind") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Libunwind { target: run.target }); + } + + /// Build libunwind.a + fn run(self, builder: &Builder<'_>) -> Self::Output { + builder.update_submodule(&Path::new("src/llvm-project")); + + if builder.config.dry_run() { + return PathBuf::new(); + } + + let out_dir = builder.native_dir(self.target).join("libunwind"); + let root = builder.src.join("src/llvm-project/libunwind"); + + if up_to_date(&root, &out_dir.join("libunwind.a")) { + return out_dir; + } + + let _guard = builder.msg_unstaged(Kind::Build, "libunwind.a", self.target); + t!(fs::create_dir_all(&out_dir)); + + let mut cc_cfg = cc::Build::new(); + let mut cpp_cfg = cc::Build::new(); + + cpp_cfg.cpp(true); + cpp_cfg.cpp_set_stdlib(None); + cpp_cfg.flag("-nostdinc++"); + cpp_cfg.flag("-fno-exceptions"); + cpp_cfg.flag("-fno-rtti"); + cpp_cfg.flag_if_supported("-fvisibility-global-new-delete-hidden"); + + for cfg in [&mut cc_cfg, &mut cpp_cfg].iter_mut() { + if let Some(ar) = builder.ar(self.target) { + cfg.archiver(ar); + } + cfg.target(&self.target.triple); + cfg.host(&builder.config.build.triple); + cfg.warnings(false); + cfg.debug(false); + // get_compiler() need set opt_level first. + cfg.opt_level(3); + cfg.flag("-fstrict-aliasing"); + cfg.flag("-funwind-tables"); + cfg.flag("-fvisibility=hidden"); + cfg.define("_LIBUNWIND_DISABLE_VISIBILITY_ANNOTATIONS", None); + cfg.include(root.join("include")); + cfg.cargo_metadata(false); + cfg.out_dir(&out_dir); + + if self.target.contains("x86_64-fortanix-unknown-sgx") { + cfg.static_flag(true); + cfg.flag("-fno-stack-protector"); + cfg.flag("-ffreestanding"); + cfg.flag("-fexceptions"); + + // easiest way to undefine since no API available in cc::Build to undefine + cfg.flag("-U_FORTIFY_SOURCE"); + cfg.define("_FORTIFY_SOURCE", "0"); + cfg.define("RUST_SGX", "1"); + cfg.define("__NO_STRING_INLINES", None); + cfg.define("__NO_MATH_INLINES", None); + cfg.define("_LIBUNWIND_IS_BAREMETAL", None); + cfg.define("__LIBUNWIND_IS_NATIVE_ONLY", None); + cfg.define("NDEBUG", None); + } + if self.target.contains("windows") { + cfg.define("_LIBUNWIND_HIDE_SYMBOLS", "1"); + cfg.define("_LIBUNWIND_IS_NATIVE_ONLY", "1"); + } + } + + cc_cfg.compiler(builder.cc(self.target)); + if let Ok(cxx) = builder.cxx(self.target) { + cpp_cfg.compiler(cxx); + } else { + cc_cfg.compiler(builder.cc(self.target)); + } + + // Don't set this for clang + // By default, Clang builds C code in GNU C17 mode. + // By default, Clang builds C++ code according to the C++98 standard, + // with many C++11 features accepted as extensions. + if cc_cfg.get_compiler().is_like_gnu() { + cc_cfg.flag("-std=c99"); + } + if cpp_cfg.get_compiler().is_like_gnu() { + cpp_cfg.flag("-std=c++11"); + } + + if self.target.contains("x86_64-fortanix-unknown-sgx") || self.target.contains("musl") { + // use the same GCC C compiler command to compile C++ code so we do not need to setup the + // C++ compiler env variables on the builders. + // Don't set this for clang++, as clang++ is able to compile this without libc++. + if cpp_cfg.get_compiler().is_like_gnu() { + cpp_cfg.cpp(false); + cpp_cfg.compiler(builder.cc(self.target)); + } + } + + let mut c_sources = vec![ + "Unwind-sjlj.c", + "UnwindLevel1-gcc-ext.c", + "UnwindLevel1.c", + "UnwindRegistersRestore.S", + "UnwindRegistersSave.S", + ]; + + let cpp_sources = vec!["Unwind-EHABI.cpp", "Unwind-seh.cpp", "libunwind.cpp"]; + let cpp_len = cpp_sources.len(); + + if self.target.contains("x86_64-fortanix-unknown-sgx") { + c_sources.push("UnwindRustSgx.c"); + } + + for src in c_sources { + cc_cfg.file(root.join("src").join(src).canonicalize().unwrap()); + } + + for src in &cpp_sources { + cpp_cfg.file(root.join("src").join(src).canonicalize().unwrap()); + } + + cpp_cfg.compile("unwind-cpp"); + + // FIXME: https://github.com/alexcrichton/cc-rs/issues/545#issuecomment-679242845 + let mut count = 0; + for entry in fs::read_dir(&out_dir).unwrap() { + let file = entry.unwrap().path().canonicalize().unwrap(); + if file.is_file() && file.extension() == Some(OsStr::new("o")) { + // file name starts with "Unwind-EHABI", "Unwind-seh" or "libunwind" + let file_name = file.file_name().unwrap().to_str().expect("UTF-8 file name"); + if cpp_sources.iter().any(|f| file_name.starts_with(&f[..f.len() - 4])) { + cc_cfg.object(&file); + count += 1; + } + } + } + assert_eq!(cpp_len, count, "Can't get object files from {:?}", &out_dir); + + cc_cfg.compile("unwind"); + out_dir + } +} diff --git a/src/bootstrap/src/core/build_steps/mod.rs b/src/bootstrap/src/core/build_steps/mod.rs new file mode 100644 index 000000000..50d83789b --- /dev/null +++ b/src/bootstrap/src/core/build_steps/mod.rs @@ -0,0 +1,15 @@ +pub(crate) mod check; +pub(crate) mod clean; +pub(crate) mod compile; +pub(crate) mod dist; +pub(crate) mod doc; +pub(crate) mod format; +pub(crate) mod install; +pub(crate) mod llvm; +pub(crate) mod run; +pub(crate) mod setup; +pub(crate) mod suggest; +pub(crate) mod synthetic_targets; +pub(crate) mod test; +pub(crate) mod tool; +pub(crate) mod toolstate; diff --git a/src/bootstrap/src/core/build_steps/run.rs b/src/bootstrap/src/core/build_steps/run.rs new file mode 100644 index 000000000..d1d6b7e86 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/run.rs @@ -0,0 +1,301 @@ +use std::path::PathBuf; +use std::process::Command; + +use crate::core::build_steps::dist::distdir; +use crate::core::build_steps::test; +use crate::core::build_steps::tool::{self, SourceType, Tool}; +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::core::config::flags::get_completion; +use crate::core::config::TargetSelection; +use crate::utils::helpers::output; +use crate::Mode; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + + /// Runs the `expand-yaml_anchors` tool. + /// + /// This tool in `src/tools` reads the CI configuration files written in YAML and expands the + /// anchors in them, since GitHub Actions doesn't support them. + fn run(self, builder: &Builder<'_>) { + builder.info("Expanding YAML anchors in the GitHub Actions configuration"); + builder.run_delaying_failure( + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("generate").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct BuildManifest; + +impl Step for BuildManifest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/build-manifest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BuildManifest); + } + + fn run(self, builder: &Builder<'_>) { + // This gets called by `promote-release` + // (https://github.com/rust-lang/promote-release). + let mut cmd = builder.tool_cmd(Tool::BuildManifest); + let sign = builder.config.dist_sign_folder.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") + }); + let addr = builder.config.dist_upload_addr.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") + }); + + let today = output(Command::new("date").arg("+%Y-%m-%d")); + + cmd.arg(sign); + cmd.arg(distdir(builder)); + cmd.arg(today.trim()); + cmd.arg(addr); + cmd.arg(&builder.config.channel); + + builder.create_dir(&distdir(builder)); + builder.run(&mut cmd); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct BumpStage0; + +impl Step for BumpStage0 { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/bump-stage0") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BumpStage0); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let mut cmd = builder.tool_cmd(Tool::BumpStage0); + cmd.args(builder.config.args()); + builder.run(&mut cmd); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct ReplaceVersionPlaceholder; + +impl Step for ReplaceVersionPlaceholder { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/replace-version-placeholder") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ReplaceVersionPlaceholder); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let mut cmd = builder.tool_cmd(Tool::ReplaceVersionPlaceholder); + cmd.arg(&builder.src); + builder.run(&mut cmd); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Miri { + stage: u32, + host: TargetSelection, + target: TargetSelection, +} + +impl Step for Miri { + type Output = (); + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/miri") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Miri { + stage: run.builder.top_stage, + host: run.build_triple(), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let target = self.target; + let compiler = builder.compiler(stage, host); + + let miri = builder + .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let miri_sysroot = test::Miri::build_miri_sysroot(builder, compiler, &miri, target); + + // # Run miri. + // Running it via `cargo run` as that figures out the right dylib path. + // add_rustc_lib_path does not add the path that contains librustc_driver-<...>.so. + let mut miri = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "run", + "src/tools/miri", + SourceType::InTree, + &[], + ); + miri.add_rustc_lib_path(builder, compiler); + // Forward arguments. + miri.arg("--").arg("--target").arg(target.rustc_target_arg()); + miri.args(builder.config.args()); + + // miri tests need to know about the stage sysroot + miri.env("MIRI_SYSROOT", &miri_sysroot); + + let mut miri = Command::from(miri); + builder.run(&mut miri); + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct CollectLicenseMetadata; + +impl Step for CollectLicenseMetadata { + type Output = PathBuf; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/collect-license-metadata") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CollectLicenseMetadata); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let Some(reuse) = &builder.config.reuse else { + panic!("REUSE is required to collect the license metadata"); + }; + + // Temporary location, it will be moved to src/etc once it's accurate. + let dest = builder.out.join("license-metadata.json"); + + let mut cmd = builder.tool_cmd(Tool::CollectLicenseMetadata); + cmd.env("REUSE_EXE", reuse); + cmd.env("DEST", &dest); + builder.run(&mut cmd); + + dest + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct GenerateCopyright; + +impl Step for GenerateCopyright { + type Output = PathBuf; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/generate-copyright") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(GenerateCopyright); + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + let license_metadata = builder.ensure(CollectLicenseMetadata); + + // Temporary location, it will be moved to the proper one once it's accurate. + let dest = builder.out.join("COPYRIGHT.md"); + + let mut cmd = builder.tool_cmd(Tool::GenerateCopyright); + cmd.env("LICENSE_METADATA", &license_metadata); + cmd.env("DEST", &dest); + builder.run(&mut cmd); + + dest + } +} + +#[derive(Debug, PartialOrd, Ord, Copy, Clone, Hash, PartialEq, Eq)] +pub struct GenerateWindowsSys; + +impl Step for GenerateWindowsSys { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/generate-windows-sys") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(GenerateWindowsSys); + } + + fn run(self, builder: &Builder<'_>) { + let mut cmd = builder.tool_cmd(Tool::GenerateWindowsSys); + cmd.arg(&builder.src); + builder.run(&mut cmd); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct GenerateCompletions; + +macro_rules! generate_completions { + ( $( ( $shell:ident, $filename:expr ) ),* ) => { + $( + if let Some(comp) = get_completion($shell, &$filename) { + std::fs::write(&$filename, comp).expect(&format!("writing {} completion", stringify!($shell))); + } + )* + }; +} + +impl Step for GenerateCompletions { + type Output = (); + + /// Uses `clap_complete` to generate shell completions. + fn run(self, builder: &Builder<'_>) { + use clap_complete::shells::{Bash, Fish, PowerShell, Zsh}; + + generate_completions!( + (Bash, builder.src.join("src/etc/completions/x.py.sh")), + (Zsh, builder.src.join("src/etc/completions/x.py.zsh")), + (Fish, builder.src.join("src/etc/completions/x.py.fish")), + (PowerShell, builder.src.join("src/etc/completions/x.py.ps1")) + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("generate-completions") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(GenerateCompletions); + } +} diff --git a/src/bootstrap/src/core/build_steps/setup.rs b/src/bootstrap/src/core/build_steps/setup.rs new file mode 100644 index 000000000..486a1e20f --- /dev/null +++ b/src/bootstrap/src/core/build_steps/setup.rs @@ -0,0 +1,623 @@ +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::Config; +use crate::{t, CONFIG_CHANGE_HISTORY}; +use sha2::Digest; +use std::env::consts::EXE_SUFFIX; +use std::fmt::Write as _; +use std::fs::File; +use std::io::Write; +use std::path::{Path, PathBuf, MAIN_SEPARATOR}; +use std::process::Command; +use std::str::FromStr; +use std::{fmt, fs, io}; + +#[cfg(test)] +#[path = "../../tests/setup.rs"] +mod tests; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub enum Profile { + Compiler, + Codegen, + Library, + Tools, + Dist, + None, +} + +/// A list of historical hashes of `src/etc/rust_analyzer_settings.json`. +/// New entries should be appended whenever this is updated so we can detect +/// outdated vs. user-modified settings files. +static SETTINGS_HASHES: &[&str] = &[ + "ea67e259dedf60d4429b6c349a564ffcd1563cf41c920a856d1f5b16b4701ac8", + "56e7bf011c71c5d81e0bf42e84938111847a810eee69d906bba494ea90b51922", + "af1b5efe196aed007577899db9dae15d6dbc923d6fa42fa0934e68617ba9bbe0", + "3468fea433c25fff60be6b71e8a215a732a7b1268b6a83bf10d024344e140541", + "47d227f424bf889b0d899b9cc992d5695e1b78c406e183cd78eafefbe5488923", + "b526bd58d0262dd4dda2bff5bc5515b705fb668a46235ace3e057f807963a11a", +]; +static RUST_ANALYZER_SETTINGS: &str = include_str!("../../../../etc/rust_analyzer_settings.json"); + +impl Profile { + fn include_path(&self, src_path: &Path) -> PathBuf { + PathBuf::from(format!("{}/src/bootstrap/defaults/config.{}.toml", src_path.display(), self)) + } + + pub fn all() -> impl Iterator { + use Profile::*; + // N.B. these are ordered by how they are displayed, not alphabetically + [Library, Compiler, Codegen, Tools, Dist, None].iter().copied() + } + + pub fn purpose(&self) -> String { + use Profile::*; + match self { + Library => "Contribute to the standard library", + Compiler => "Contribute to the compiler itself", + Codegen => "Contribute to the compiler, and also modify LLVM or codegen", + Tools => "Contribute to tools which depend on the compiler, but do not modify it directly (e.g. rustdoc, clippy, miri)", + Dist => "Install Rust from source", + None => "Do not modify `config.toml`" + } + .to_string() + } + + pub fn all_for_help(indent: &str) -> String { + let mut out = String::new(); + for choice in Profile::all() { + writeln!(&mut out, "{}{}: {}", indent, choice, choice.purpose()).unwrap(); + } + out + } + + pub fn as_str(&self) -> &'static str { + match self { + Profile::Compiler => "compiler", + Profile::Codegen => "codegen", + Profile::Library => "library", + Profile::Tools => "tools", + Profile::Dist => "dist", + Profile::None => "none", + } + } +} + +impl FromStr for Profile { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "lib" | "library" => Ok(Profile::Library), + "compiler" => Ok(Profile::Compiler), + "llvm" | "codegen" => Ok(Profile::Codegen), + "maintainer" | "dist" | "user" => Ok(Profile::Dist), + "tools" | "tool" | "rustdoc" | "clippy" | "miri" | "rustfmt" | "rls" => { + Ok(Profile::Tools) + } + "none" => Ok(Profile::None), + _ => Err(format!("unknown profile: '{s}'")), + } + } +} + +impl fmt::Display for Profile { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(self.as_str()) + } +} + +impl Step for Profile { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(mut run: ShouldRun<'_>) -> ShouldRun<'_> { + for choice in Profile::all() { + run = run.alias(choice.as_str()); + } + run + } + + fn make_run(run: RunConfig<'_>) { + if run.builder.config.dry_run() { + return; + } + + let path = &run.builder.config.config.clone().unwrap_or(PathBuf::from("config.toml")); + if path.exists() { + eprintln!(); + eprintln!( + "ERROR: you asked for a new config file, but one already exists at `{}`", + t!(path.canonicalize()).display() + ); + + match prompt_user( + "Do you wish to override the existing configuration (which will allow the setup process to continue)?: [y/N]", + ) { + Ok(Some(PromptResult::Yes)) => { + t!(fs::remove_file(path)); + } + _ => { + println!("Exiting."); + crate::exit!(1); + } + } + } + + // for Profile, `run.paths` will have 1 and only 1 element + // this is because we only accept at most 1 path from user input. + // If user calls `x.py setup` without arguments, the interactive TUI + // will guide user to provide one. + let profile = if run.paths.len() > 1 { + // HACK: `builder` runs this step with all paths if no path was passed. + t!(interactive_path()) + } else { + run.paths + .first() + .unwrap() + .assert_single_path() + .path + .as_path() + .as_os_str() + .to_str() + .unwrap() + .parse() + .unwrap() + }; + + run.builder.ensure(profile); + } + + fn run(self, builder: &Builder<'_>) { + // During ./x.py setup once you select the codegen profile. + // The submodule will be downloaded. It does not work in the + // tarball case since they don't include Git and submodules + // are already included. + if !builder.rust_info().is_from_tarball() { + if self == Profile::Codegen { + builder.update_submodule(&Path::new("src/llvm-project")); + } + } + setup(&builder.build.config, self) + } +} + +pub fn setup(config: &Config, profile: Profile) { + let suggestions: &[&str] = match profile { + Profile::Codegen | Profile::Compiler | Profile::None => &["check", "build", "test"], + Profile::Tools => &[ + "check", + "build", + "test tests/rustdoc*", + "test src/tools/clippy", + "test src/tools/miri", + "test src/tools/rustfmt", + ], + Profile::Library => &["check", "build", "test library/std", "doc"], + Profile::Dist => &["dist", "build"], + }; + + println!(); + + println!("To get started, try one of the following commands:"); + for cmd in suggestions { + println!("- `x.py {cmd}`"); + } + + if profile != Profile::Dist { + println!( + "For more suggestions, see https://rustc-dev-guide.rust-lang.org/building/suggested.html" + ); + } + + if profile == Profile::Tools { + eprintln!(); + eprintln!( + "NOTE: the `tools` profile sets up the `stage2` toolchain (use \ + `rustup toolchain link 'name' build/host/stage2` to use rustc)" + ) + } + + let path = &config.config.clone().unwrap_or(PathBuf::from("config.toml")); + setup_config_toml(path, profile, config); +} + +fn setup_config_toml(path: &PathBuf, profile: Profile, config: &Config) { + if profile == Profile::None { + return; + } + + let latest_change_id = CONFIG_CHANGE_HISTORY.last().unwrap(); + let settings = format!( + "# Includes one of the default files in src/bootstrap/defaults\n\ + profile = \"{profile}\"\n\ + change-id = {latest_change_id}\n" + ); + + t!(fs::write(path, settings)); + + let include_path = profile.include_path(&config.src); + println!("`x.py` will now use the configuration at {}", include_path.display()); +} + +/// Creates a toolchain link for stage1 using `rustup` +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct Link; +impl Step for Link { + type Output = (); + const DEFAULT: bool = true; + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("link") + } + fn make_run(run: RunConfig<'_>) { + if run.builder.config.dry_run() { + return; + } + if let [cmd] = &run.paths[..] { + if cmd.assert_single_path().path.as_path().as_os_str() == "link" { + run.builder.ensure(Link); + } + } + } + fn run(self, builder: &Builder<'_>) -> Self::Output { + let config = &builder.config; + if config.dry_run() { + return; + } + let stage_path = + ["build", config.build.rustc_target_arg(), "stage1"].join(&MAIN_SEPARATOR.to_string()); + + if !rustup_installed() { + eprintln!("`rustup` is not installed; cannot link `stage1` toolchain"); + } else if stage_dir_exists(&stage_path[..]) && !config.dry_run() { + attempt_toolchain_link(&stage_path[..]); + } + } +} + +fn rustup_installed() -> bool { + Command::new("rustup") + .arg("--version") + .stdout(std::process::Stdio::null()) + .output() + .map_or(false, |output| output.status.success()) +} + +fn stage_dir_exists(stage_path: &str) -> bool { + match fs::create_dir(&stage_path) { + Ok(_) => true, + Err(_) => Path::new(&stage_path).exists(), + } +} + +fn attempt_toolchain_link(stage_path: &str) { + if toolchain_is_linked() { + return; + } + + if !ensure_stage1_toolchain_placeholder_exists(stage_path) { + eprintln!( + "Failed to create a template for stage 1 toolchain or confirm that it already exists" + ); + return; + } + + if try_link_toolchain(&stage_path) { + println!( + "Added `stage1` rustup toolchain; try `cargo +stage1 build` on a separate rust project to run a newly-built toolchain" + ); + } else { + eprintln!("`rustup` failed to link stage 1 build to `stage1` toolchain"); + eprintln!( + "To manually link stage 1 build to `stage1` toolchain, run:\n + `rustup toolchain link stage1 {}`", + &stage_path + ); + } +} + +fn toolchain_is_linked() -> bool { + match Command::new("rustup") + .args(&["toolchain", "list"]) + .stdout(std::process::Stdio::piped()) + .output() + { + Ok(toolchain_list) => { + if !String::from_utf8_lossy(&toolchain_list.stdout).contains("stage1") { + return false; + } + // The toolchain has already been linked. + println!( + "`stage1` toolchain already linked; not attempting to link `stage1` toolchain" + ); + } + Err(_) => { + // In this case, we don't know if the `stage1` toolchain has been linked; + // but `rustup` failed, so let's not go any further. + println!( + "`rustup` failed to list current toolchains; not attempting to link `stage1` toolchain" + ); + } + } + true +} + +fn try_link_toolchain(stage_path: &str) -> bool { + Command::new("rustup") + .stdout(std::process::Stdio::null()) + .args(&["toolchain", "link", "stage1", &stage_path]) + .output() + .map_or(false, |output| output.status.success()) +} + +fn ensure_stage1_toolchain_placeholder_exists(stage_path: &str) -> bool { + let pathbuf = PathBuf::from(stage_path); + + if fs::create_dir_all(pathbuf.join("lib")).is_err() { + return false; + }; + + let pathbuf = pathbuf.join("bin"); + if fs::create_dir_all(&pathbuf).is_err() { + return false; + }; + + let pathbuf = pathbuf.join(format!("rustc{EXE_SUFFIX}")); + + if pathbuf.exists() { + return true; + } + + // Take care not to overwrite the file + let result = File::options().append(true).create(true).open(&pathbuf); + if result.is_err() { + return false; + } + + return true; +} + +// Used to get the path for `Subcommand::Setup` +pub fn interactive_path() -> io::Result { + fn abbrev_all() -> impl Iterator { + ('a'..) + .zip(1..) + .map(|(letter, number)| (letter.to_string(), number.to_string())) + .zip(Profile::all()) + } + + fn parse_with_abbrev(input: &str) -> Result { + let input = input.trim().to_lowercase(); + for ((letter, number), profile) in abbrev_all() { + if input == letter || input == number { + return Ok(profile); + } + } + input.parse() + } + + println!("Welcome to the Rust project! What do you want to do with x.py?"); + for ((letter, _), profile) in abbrev_all() { + println!("{}) {}: {}", letter, profile, profile.purpose()); + } + let template = loop { + print!( + "Please choose one ({}): ", + abbrev_all().map(|((l, _), _)| l).collect::>().join("/") + ); + io::stdout().flush()?; + let mut input = String::new(); + io::stdin().read_line(&mut input)?; + if input.is_empty() { + eprintln!("EOF on stdin, when expecting answer to question. Giving up."); + crate::exit!(1); + } + break match parse_with_abbrev(&input) { + Ok(profile) => profile, + Err(err) => { + eprintln!("ERROR: {err}"); + eprintln!("NOTE: press Ctrl+C to exit"); + continue; + } + }; + }; + Ok(template) +} + +#[derive(PartialEq)] +enum PromptResult { + Yes, // y/Y/yes + No, // n/N/no + Print, // p/P/print +} + +/// Prompt a user for a answer, looping until they enter an accepted input or nothing +fn prompt_user(prompt: &str) -> io::Result> { + let mut input = String::new(); + loop { + print!("{prompt} "); + io::stdout().flush()?; + input.clear(); + io::stdin().read_line(&mut input)?; + match input.trim().to_lowercase().as_str() { + "y" | "yes" => return Ok(Some(PromptResult::Yes)), + "n" | "no" => return Ok(Some(PromptResult::No)), + "p" | "print" => return Ok(Some(PromptResult::Print)), + "" => return Ok(None), + _ => { + eprintln!("ERROR: unrecognized option '{}'", input.trim()); + eprintln!("NOTE: press Ctrl+C to exit"); + } + }; + } +} + +/// Installs `src/etc/pre-push.sh` as a Git hook +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct Hook; + +impl Step for Hook { + type Output = (); + const DEFAULT: bool = true; + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("hook") + } + fn make_run(run: RunConfig<'_>) { + if run.builder.config.dry_run() { + return; + } + if let [cmd] = &run.paths[..] { + if cmd.assert_single_path().path.as_path().as_os_str() == "hook" { + run.builder.ensure(Hook); + } + } + } + fn run(self, builder: &Builder<'_>) -> Self::Output { + let config = &builder.config; + if config.dry_run() { + return; + } + t!(install_git_hook_maybe(&config)); + } +} + +// install a git hook to automatically run tidy, if they want +fn install_git_hook_maybe(config: &Config) -> io::Result<()> { + let git = t!(config.git().args(&["rev-parse", "--git-common-dir"]).output().map(|output| { + assert!(output.status.success(), "failed to run `git`"); + PathBuf::from(t!(String::from_utf8(output.stdout)).trim()) + })); + let hooks_dir = git.join("hooks"); + let dst = hooks_dir.join("pre-push"); + if dst.exists() { + // The git hook has already been set up, or the user already has a custom hook. + return Ok(()); + } + + println!( + "\nRust's CI will automatically fail if it doesn't pass `tidy`, the internal tool for ensuring code quality. +If you'd like, x.py can install a git hook for you that will automatically run `test tidy` before +pushing your code to ensure your code is up to par. If you decide later that this behavior is +undesirable, simply delete the `pre-push` file from .git/hooks." + ); + + if prompt_user("Would you like to install the git hook?: [y/N]")? != Some(PromptResult::Yes) { + println!("Ok, skipping installation!"); + return Ok(()); + } + if !hooks_dir.exists() { + // We need to (try to) create the hooks directory first. + let _ = fs::create_dir(hooks_dir); + } + let src = config.src.join("src").join("etc").join("pre-push.sh"); + match fs::hard_link(src, &dst) { + Err(e) => { + eprintln!( + "ERROR: could not create hook {}: do you already have the git hook installed?\n{}", + dst.display(), + e + ); + return Err(e); + } + Ok(_) => println!("Linked `src/etc/pre-push.sh` to `.git/hooks/pre-push`"), + }; + Ok(()) +} + +/// Sets up or displays `src/etc/rust_analyzer_settings.json` +#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] +pub struct Vscode; + +impl Step for Vscode { + type Output = (); + const DEFAULT: bool = true; + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("vscode") + } + fn make_run(run: RunConfig<'_>) { + if run.builder.config.dry_run() { + return; + } + if let [cmd] = &run.paths[..] { + if cmd.assert_single_path().path.as_path().as_os_str() == "vscode" { + run.builder.ensure(Vscode); + } + } + } + fn run(self, builder: &Builder<'_>) -> Self::Output { + let config = &builder.config; + if config.dry_run() { + return; + } + t!(create_vscode_settings_maybe(&config)); + } +} + +/// Create a `.vscode/settings.json` file for rustc development, or just print it +fn create_vscode_settings_maybe(config: &Config) -> io::Result<()> { + let (current_hash, historical_hashes) = SETTINGS_HASHES.split_last().unwrap(); + let vscode_settings = config.src.join(".vscode").join("settings.json"); + // If None, no settings.json exists + // If Some(true), is a previous version of settings.json + // If Some(false), is not a previous version (i.e. user modified) + // If it's up to date we can just skip this + let mut mismatched_settings = None; + if let Ok(current) = fs::read_to_string(&vscode_settings) { + let mut hasher = sha2::Sha256::new(); + hasher.update(¤t); + let hash = hex::encode(hasher.finalize().as_slice()); + if hash == *current_hash { + return Ok(()); + } else if historical_hashes.contains(&hash.as_str()) { + mismatched_settings = Some(true); + } else { + mismatched_settings = Some(false); + } + } + println!( + "\nx.py can automatically install the recommended `.vscode/settings.json` file for rustc development" + ); + match mismatched_settings { + Some(true) => eprintln!( + "WARNING: existing `.vscode/settings.json` is out of date, x.py will update it" + ), + Some(false) => eprintln!( + "WARNING: existing `.vscode/settings.json` has been modified by user, x.py will back it up and replace it" + ), + _ => (), + } + let should_create = match prompt_user( + "Would you like to create/update `settings.json`, or only print suggested settings?: [y/p/N]", + )? { + Some(PromptResult::Yes) => true, + Some(PromptResult::Print) => false, + _ => { + println!("Ok, skipping settings!"); + return Ok(()); + } + }; + if should_create { + let path = config.src.join(".vscode"); + if !path.exists() { + fs::create_dir(&path)?; + } + let verb = match mismatched_settings { + // exists but outdated, we can replace this + Some(true) => "Updated", + // exists but user modified, back it up + Some(false) => { + // exists and is not current version or outdated, so back it up + let mut backup = vscode_settings.clone(); + backup.set_extension("json.bak"); + eprintln!("WARNING: copying `settings.json` to `settings.json.bak`"); + fs::copy(&vscode_settings, &backup)?; + "Updated" + } + _ => "Created", + }; + fs::write(&vscode_settings, &RUST_ANALYZER_SETTINGS)?; + println!("{verb} `.vscode/settings.json`"); + } else { + println!("\n{RUST_ANALYZER_SETTINGS}"); + } + Ok(()) +} diff --git a/src/bootstrap/src/core/build_steps/suggest.rs b/src/bootstrap/src/core/build_steps/suggest.rs new file mode 100644 index 000000000..93da27560 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/suggest.rs @@ -0,0 +1,78 @@ +#![cfg_attr(feature = "build-metrics", allow(unused))] + +use clap::Parser; +use std::path::PathBuf; +use std::str::FromStr; + +use crate::core::build_steps::tool::Tool; +use crate::core::builder::Builder; + +/// Suggests a list of possible `x.py` commands to run based on modified files in branch. +pub fn suggest(builder: &Builder<'_>, run: bool) { + let git_config = builder.config.git_config(); + let suggestions = builder + .tool_cmd(Tool::SuggestTests) + .env("SUGGEST_TESTS_GIT_REPOSITORY", git_config.git_repository) + .env("SUGGEST_TESTS_NIGHTLY_BRANCH", git_config.nightly_branch) + .output() + .expect("failed to run `suggest-tests` tool"); + + if !suggestions.status.success() { + println!("failed to run `suggest-tests` tool ({})", suggestions.status); + println!( + "`suggest_tests` stdout:\n{}`suggest_tests` stderr:\n{}", + String::from_utf8(suggestions.stdout).unwrap(), + String::from_utf8(suggestions.stderr).unwrap() + ); + panic!("failed to run `suggest-tests`"); + } + + let suggestions = String::from_utf8(suggestions.stdout).unwrap(); + let suggestions = suggestions + .lines() + .map(|line| { + let mut sections = line.split_ascii_whitespace(); + + // this code expects one suggestion per line in the following format: + // {some number of flags} [optional stage number] + let cmd = sections.next().unwrap(); + let stage = sections.next_back().map(|s| str::parse(s).ok()).flatten(); + let paths: Vec = sections.map(|p| PathBuf::from_str(p).unwrap()).collect(); + + (cmd, stage, paths) + }) + .collect::>(); + + if !suggestions.is_empty() { + println!("==== SUGGESTIONS ===="); + for sug in &suggestions { + print!("x {} ", sug.0); + if let Some(stage) = sug.1 { + print!("--stage {stage} "); + } + + for path in &sug.2 { + print!("{} ", path.display()); + } + println!(); + } + println!("====================="); + } else { + println!("No suggestions found!"); + return; + } + + if run { + for sug in suggestions { + let mut build: crate::Build = builder.build.clone(); + build.config.paths = sug.2; + build.config.cmd = crate::core::config::flags::Flags::parse_from(["x.py", sug.0]).cmd; + if let Some(stage) = sug.1 { + build.config.stage = stage; + } + build.build(); + } + } else { + println!("HELP: consider using the `--run` flag to automatically run suggested tests"); + } +} diff --git a/src/bootstrap/src/core/build_steps/synthetic_targets.rs b/src/bootstrap/src/core/build_steps/synthetic_targets.rs new file mode 100644 index 000000000..d2c65b740 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/synthetic_targets.rs @@ -0,0 +1,82 @@ +//! In some cases, parts of bootstrap need to change part of a target spec just for one or a few +//! steps. Adding these targets to rustc proper would "leak" this implementation detail of +//! bootstrap, and would make it more complex to apply additional changes if the need arises. +//! +//! To address that problem, this module implements support for "synthetic targets". Synthetic +//! targets are custom target specs generated using builtin target specs as their base. You can use +//! one of the target specs already defined in this module, or create new ones by adding a new step +//! that calls create_synthetic_target. + +use crate::core::builder::{Builder, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::Compiler; +use std::process::{Command, Stdio}; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub(crate) struct MirOptPanicAbortSyntheticTarget { + pub(crate) compiler: Compiler, + pub(crate) base: TargetSelection, +} + +impl Step for MirOptPanicAbortSyntheticTarget { + type Output = TargetSelection; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> Self::Output { + create_synthetic_target(builder, self.compiler, "miropt-abort", self.base, |spec| { + spec.insert("panic-strategy".into(), "abort".into()); + }) + } +} + +fn create_synthetic_target( + builder: &Builder<'_>, + compiler: Compiler, + suffix: &str, + base: TargetSelection, + customize: impl FnOnce(&mut serde_json::Map), +) -> TargetSelection { + if base.contains("synthetic") { + // This check is not strictly needed, but nothing currently needs recursive synthetic + // targets. If the need arises, removing this in the future *SHOULD* be safe. + panic!("cannot create synthetic targets with other synthetic targets as their base"); + } + + let name = format!("{base}-synthetic-{suffix}"); + let path = builder.out.join("synthetic-target-specs").join(format!("{name}.json")); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + + if builder.config.dry_run() { + std::fs::write(&path, b"dry run\n").unwrap(); + return TargetSelection::create_synthetic(&name, path.to_str().unwrap()); + } + + let mut cmd = Command::new(builder.rustc(compiler)); + cmd.arg("--target").arg(base.rustc_target_arg()); + cmd.args(["-Zunstable-options", "--print", "target-spec-json"]); + cmd.stdout(Stdio::piped()); + + let output = cmd.spawn().unwrap().wait_with_output().unwrap(); + if !output.status.success() { + panic!("failed to gather the target spec for {base}"); + } + + let mut spec: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); + let spec_map = spec.as_object_mut().unwrap(); + + // The `is-builtin` attribute of a spec needs to be removed, otherwise rustc will complain. + spec_map.remove("is-builtin"); + + customize(spec_map); + + std::fs::write(&path, &serde_json::to_vec_pretty(&spec).unwrap()).unwrap(); + let target = TargetSelection::create_synthetic(&name, path.to_str().unwrap()); + crate::utils::cc_detect::find_target(builder, target); + + target +} diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs new file mode 100644 index 000000000..d2aa89dee --- /dev/null +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -0,0 +1,3315 @@ +//! Implementation of the test-related targets of the build system. +//! +//! This file implements the various regression test suites that we execute on +//! our CI. + +use std::env; +use std::ffi::OsStr; +use std::ffi::OsString; +use std::fs; +use std::iter; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use clap_complete::shells; + +use crate::core::build_steps::compile; +use crate::core::build_steps::dist; +use crate::core::build_steps::doc::DocumentationFormat; +use crate::core::build_steps::llvm; +use crate::core::build_steps::synthetic_targets::MirOptPanicAbortSyntheticTarget; +use crate::core::build_steps::tool::{self, SourceType, Tool}; +use crate::core::build_steps::toolstate::ToolState; +use crate::core::builder::crate_description; +use crate::core::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; +use crate::core::config::flags::get_completion; +use crate::core::config::flags::Subcommand; +use crate::core::config::TargetSelection; +use crate::utils; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::exec::BootstrapCommand; +use crate::utils::helpers::{ + self, add_link_lib_path, dylib_path, dylib_path_var, output, t, + target_supports_cranelift_backend, up_to_date, +}; +use crate::utils::render_tests::{add_flags_and_try_run_tests, try_run_tests}; +use crate::{envify, CLang, DocTests, GitRepo, Mode}; + +const ADB_TEST_DIR: &str = "/data/local/tmp/work"; + +// mir-opt tests have different variants depending on whether a target is 32bit or 64bit, and +// blessing them requires blessing with each target. To aid developers, when blessing the mir-opt +// test suite the corresponding target of the opposite pointer size is also blessed. +// +// This array serves as the known mappings between 32bit and 64bit targets. If you're developing on +// a target where a target with the opposite pointer size exists, feel free to add it here. +const MIR_OPT_BLESS_TARGET_MAPPING: &[(&str, &str)] = &[ + // (32bit, 64bit) + ("i686-unknown-linux-gnu", "x86_64-unknown-linux-gnu"), + ("i686-unknown-linux-musl", "x86_64-unknown-linux-musl"), + ("i686-pc-windows-msvc", "x86_64-pc-windows-msvc"), + ("i686-pc-windows-gnu", "x86_64-pc-windows-gnu"), + ("i686-apple-darwin", "x86_64-apple-darwin"), + // ARM Macs don't have a corresponding 32-bit target that they can (easily) + // build for, so there is no entry for "aarch64-apple-darwin" here. +]; + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrateBootstrap { + path: Interned, + host: TargetSelection, +} + +impl Step for CrateBootstrap { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/jsondoclint") + .path("src/tools/suggest-tests") + .path("src/tools/replace-version-placeholder") + .alias("tidyselftest") + } + + fn make_run(run: RunConfig<'_>) { + for path in run.paths { + let path = INTERNER.intern_path(path.assert_single_path().path.clone()); + run.builder.ensure(CrateBootstrap { host: run.target, path }); + } + } + + fn run(self, builder: &Builder<'_>) { + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let mut path = self.path.to_str().unwrap(); + if path == "tidyselftest" { + path = "src/tools/tidy"; + } + + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + path, + SourceType::InTree, + &[], + ); + let crate_name = path.rsplit_once('/').unwrap().1; + run_cargo_test(cargo, &[], &[], crate_name, crate_name, compiler, bootstrap_host, builder); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Linkcheck { + host: TargetSelection, +} + +impl Step for Linkcheck { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will verify the validity of all our links in the + /// documentation to ensure we don't have a bunch of dead ones. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let hosts = &builder.hosts; + let targets = &builder.targets; + + // if we have different hosts and targets, some things may be built for + // the host (e.g. rustc) and others for the target (e.g. std). The + // documentation built for each will contain broken links to + // docs built for the other platform (e.g. rustc linking to cargo) + if (hosts != targets) && !hosts.is_empty() && !targets.is_empty() { + panic!( + "Linkcheck currently does not support builds with different hosts and targets. +You can skip linkcheck with --skip src/tools/linkchecker" + ); + } + + builder.info(&format!("Linkcheck ({host})")); + + // Test the linkchecker itself. + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + "src/tools/linkchecker", + SourceType::InTree, + &[], + ); + run_cargo_test( + cargo, + &[], + &[], + "linkchecker", + "linkchecker self tests", + compiler, + bootstrap_host, + builder, + ); + + if builder.doc_tests == DocTests::No { + return; + } + + // Build all the default documentation. + builder.default_doc(&[]); + + // Build the linkchecker before calling `msg`, since GHA doesn't support nested groups. + let mut linkchecker = builder.tool_cmd(Tool::Linkchecker); + + // Run the linkchecker. + let _guard = + builder.msg(Kind::Test, compiler.stage, "Linkcheck", bootstrap_host, bootstrap_host); + let _time = helpers::timeit(&builder); + builder.run_delaying_failure(linkchecker.arg(builder.out.join(host.triple).join("doc"))); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + let run = run.path("src/tools/linkchecker"); + run.default_condition(builder.config.docs) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Linkcheck { host: run.target }); + } +} + +fn check_if_tidy_is_installed() -> bool { + Command::new("tidy") + .arg("--version") + .stdout(Stdio::null()) + .status() + .map_or(false, |status| status.success()) +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct HtmlCheck { + target: TargetSelection, +} + +impl Step for HtmlCheck { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let run = run.path("src/tools/html-checker"); + run.lazy_default_condition(Box::new(check_if_tidy_is_installed)) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(HtmlCheck { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + if !check_if_tidy_is_installed() { + eprintln!("not running HTML-check tool because `tidy` is missing"); + eprintln!( + "Note that `tidy` is not the in-tree `src/tools/tidy` but needs to be installed" + ); + panic!("Cannot run html-check tests"); + } + // Ensure that a few different kinds of documentation are available. + builder.default_doc(&[]); + builder.ensure(crate::core::build_steps::doc::Rustc::new( + builder.top_stage, + self.target, + builder, + )); + + builder.run_delaying_failure( + builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target)), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Cargotest { + stage: u32, + host: TargetSelection, +} + +impl Step for Cargotest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/cargotest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargotest { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler. + /// + /// This tool in `src/tools` will check out a few Rust projects and run `cargo + /// test` to ensure that we don't regress the test suites there. + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(self.stage, self.host); + builder.ensure(compile::Rustc::new(compiler, compiler.host)); + let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host }); + + // Note that this is a short, cryptic, and not scoped directory name. This + // is currently to minimize the length of path on Windows where we otherwise + // quickly run into path name limit constraints. + let out_dir = builder.out.join("ct"); + t!(fs::create_dir_all(&out_dir)); + + let _time = helpers::timeit(&builder); + let mut cmd = builder.tool_cmd(Tool::CargoTest); + builder.run_delaying_failure( + cmd.arg(&cargo) + .arg(&out_dir) + .args(builder.config.test_args()) + .env("RUSTC", builder.rustc(compiler)) + .env("RUSTDOC", builder.rustdoc(compiler)), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Cargo { + stage: u32, + host: TargetSelection, +} + +impl Step for Cargo { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/cargo") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for `cargo` packaged with Rust. + fn run(self, builder: &Builder<'_>) { + let compiler = builder.compiler(self.stage, self.host); + + builder.ensure(tool::Cargo { compiler, target: self.host }); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + self.host, + "test", + "src/tools/cargo", + SourceType::Submodule, + &[], + ); + + // NOTE: can't use `run_cargo_test` because we need to overwrite `PATH` + let mut cargo = prepare_cargo_test(cargo, &[], &[], "cargo", compiler, self.host, builder); + + // Don't run cross-compile tests, we may not have cross-compiled libstd libs + // available. + cargo.env("CFG_DISABLE_CROSS_TESTS", "1"); + // Forcibly disable tests using nightly features since any changes to + // those features won't be able to land. + cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1"); + cargo.env("PATH", &path_for_cargo(builder, compiler)); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::CargoPackage { + crates: vec!["cargo".into()], + target: self.host.triple.to_string(), + host: self.host.triple.to_string(), + stage: self.stage, + }, + builder, + ); + + let _time = helpers::timeit(&builder); + add_flags_and_try_run_tests(builder, &mut cargo); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustAnalyzer { + stage: u32, + host: TargetSelection, +} + +impl Step for RustAnalyzer { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-analyzer") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rust-analyzer + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, + // but we do need the standard library to be present. + builder.ensure(compile::Std::new(compiler, host)); + + let workspace_path = "src/tools/rust-analyzer"; + // until the whole RA test suite runs on `i686`, we only run + // `proc-macro-srv` tests + let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv"; + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + host, + "test", + crate_path, + SourceType::InTree, + &["sysroot-abi".to_owned()], + ); + cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES); + + let dir = builder.src.join(workspace_path); + // needed by rust-analyzer to find its own text fixtures, cf. + // https://github.com/rust-analyzer/expect-test/issues/33 + cargo.env("CARGO_WORKSPACE_DIR", &dir); + + // RA's test suite tries to write to the source directory, that can't + // work in Rust CI + cargo.env("SKIP_SLOW_TESTS", "1"); + + cargo.add_rustc_lib_path(builder, compiler); + run_cargo_test(cargo, &[], &[], "rust-analyzer", "rust-analyzer", compiler, host, builder); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Rustfmt { + stage: u32, + host: TargetSelection, +} + +impl Step for Rustfmt { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustfmt") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustfmt { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rustfmt. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + builder + .ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/rustfmt", + SourceType::InTree, + &[], + ); + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + cargo.env("RUSTFMT_TEST_DIR", dir); + + cargo.add_rustc_lib_path(builder, compiler); + + run_cargo_test(cargo, &[], &[], "rustfmt", "rustfmt", compiler, host, builder); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustDemangler { + stage: u32, + host: TargetSelection, +} + +impl Step for RustDemangler { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-demangler") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for rust-demangler. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + let rust_demangler = builder + .ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/rust-demangler", + SourceType::InTree, + &[], + ); + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + + cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler); + cargo.add_rustc_lib_path(builder, compiler); + + run_cargo_test( + cargo, + &[], + &[], + "rust-demangler", + "rust-demangler", + compiler, + host, + builder, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Miri { + stage: u32, + host: TargetSelection, + target: TargetSelection, +} + +impl Miri { + /// Run `cargo miri setup` for the given target, return where the Miri sysroot was put. + pub fn build_miri_sysroot( + builder: &Builder<'_>, + compiler: Compiler, + miri: &Path, + target: TargetSelection, + ) -> String { + let miri_sysroot = builder.out.join(compiler.host.triple).join("miri-sysroot"); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + compiler.host, + "run", + "src/tools/miri/cargo-miri", + SourceType::InTree, + &[], + ); + cargo.add_rustc_lib_path(builder, compiler); + cargo.arg("--").arg("miri").arg("setup"); + cargo.arg("--target").arg(target.rustc_target_arg()); + + // Tell `cargo miri setup` where to find the sources. + cargo.env("MIRI_LIB_SRC", builder.src.join("library")); + // Tell it where to find Miri. + cargo.env("MIRI", &miri); + // Tell it where to put the sysroot. + cargo.env("MIRI_SYSROOT", &miri_sysroot); + // Debug things. + cargo.env("RUST_BACKTRACE", "1"); + + let mut cargo = Command::from(cargo); + let _guard = builder.msg( + Kind::Build, + compiler.stage + 1, + "miri sysroot", + compiler.host, + compiler.host, + ); + builder.run(&mut cargo); + + // # Determine where Miri put its sysroot. + // To this end, we run `cargo miri setup --print-sysroot` and capture the output. + // (We do this separately from the above so that when the setup actually + // happens we get some output.) + // We re-use the `cargo` from above. + cargo.arg("--print-sysroot"); + + // FIXME: Is there a way in which we can re-use the usual `run` helpers? + if builder.config.dry_run() { + String::new() + } else { + builder.verbose(&format!("running: {cargo:?}")); + let out = + cargo.output().expect("We already ran `cargo miri setup` before and that worked"); + assert!(out.status.success(), "`cargo miri setup` returned with non-0 exit code"); + // Output is "\n". + let stdout = String::from_utf8(out.stdout) + .expect("`cargo miri setup` stdout is not valid UTF-8"); + let sysroot = stdout.trim_end(); + builder.verbose(&format!("`cargo miri setup --print-sysroot` said: {sysroot:?}")); + sysroot.to_owned() + } + } +} + +impl Step for Miri { + type Output = (); + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/miri") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Miri { + stage: run.builder.top_stage, + host: run.build_triple(), + target: run.target, + }); + } + + /// Runs `cargo test` for miri. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let target = self.target; + let compiler = builder.compiler(stage, host); + // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri. + // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage. + let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host); + + let miri = builder + .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let _cargo_miri = builder + .ensure(tool::CargoMiri { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + // The stdlib we need might be at a different stage. And just asking for the + // sysroot does not seem to populate it, so we do that first. + builder.ensure(compile::Std::new(compiler_std, host)); + let sysroot = builder.sysroot(compiler_std); + // We also need a Miri sysroot. + let miri_sysroot = Miri::build_miri_sysroot(builder, compiler, &miri, target); + + // # Run `cargo test`. + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/miri", + SourceType::InTree, + &[], + ); + let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "miri", host, target); + + cargo.add_rustc_lib_path(builder, compiler); + + // miri tests need to know about the stage sysroot + cargo.env("MIRI_SYSROOT", &miri_sysroot); + cargo.env("MIRI_HOST_SYSROOT", sysroot); + cargo.env("MIRI", &miri); + if builder.config.locked_deps { + // enforce lockfiles + cargo.env("CARGO_EXTRA_FLAGS", "--locked"); + } + + // Set the target. + cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); + + // This can NOT be `run_cargo_test` since the Miri test runner + // does not understand the flags added by `add_flags_and_try_run_test`. + let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); + { + let _time = helpers::timeit(&builder); + builder.run(&mut cargo); + } + + // Run it again for mir-opt-level 4 to catch some miscompilations. + if builder.config.test_args().is_empty() { + cargo.env("MIRIFLAGS", "-O -Zmir-opt-level=4 -Cdebug-assertions=yes"); + // Optimizations can change backtraces + cargo.env("MIRI_SKIP_UI_CHECKS", "1"); + // `MIRI_SKIP_UI_CHECKS` and `RUSTC_BLESS` are incompatible + cargo.env_remove("RUSTC_BLESS"); + // Optimizations can change error locations and remove UB so don't run `fail` tests. + cargo.args(&["tests/pass", "tests/panic"]); + + let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); + { + let _time = helpers::timeit(&builder); + builder.run(&mut cargo); + } + } + + // # Run `cargo miri test`. + // This is just a smoke test (Miri's own CI invokes this in a bunch of different ways and ensures + // that we get the desired output), but that is sufficient to make sure that the libtest harness + // itself executes properly under Miri. + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "run", + "src/tools/miri/cargo-miri", + SourceType::Submodule, + &[], + ); + cargo.add_rustc_lib_path(builder, compiler); + cargo.arg("--").arg("miri").arg("test"); + if builder.config.locked_deps { + cargo.arg("--locked"); + } + cargo + .arg("--manifest-path") + .arg(builder.src.join("src/tools/miri/test-cargo-miri/Cargo.toml")); + cargo.arg("--target").arg(target.rustc_target_arg()); + cargo.arg("--tests"); // don't run doctests, they are too confused by the staging + cargo.arg("--").args(builder.config.test_args()); + + // Tell `cargo miri` where to find things. + cargo.env("MIRI_SYSROOT", &miri_sysroot); + cargo.env("MIRI_HOST_SYSROOT", sysroot); + cargo.env("MIRI", &miri); + // Debug things. + cargo.env("RUST_BACKTRACE", "1"); + + let mut cargo = Command::from(cargo); + { + let _time = helpers::timeit(&builder); + builder.run(&mut cargo); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CompiletestTest { + host: TargetSelection, +} + +impl Step for CompiletestTest { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/compiletest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(CompiletestTest { host: run.target }); + } + + /// Runs `cargo test` for compiletest. + fn run(self, builder: &Builder<'_>) { + let host = self.host; + let compiler = builder.compiler(builder.top_stage, host); + + // We need `ToolStd` for the locally-built sysroot because + // compiletest uses unstable features of the `test` crate. + builder.ensure(compile::Std::new(compiler, host)); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolStd, + host, + "test", + "src/tools/compiletest", + SourceType::InTree, + &[], + ); + cargo.allow_features("test"); + run_cargo_test( + cargo, + &[], + &[], + "compiletest", + "compiletest self test", + compiler, + host, + builder, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Clippy { + stage: u32, + host: TargetSelection, +} + +impl Step for Clippy { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/clippy") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Clippy { stage: run.builder.top_stage, host: run.target }); + } + + /// Runs `cargo test` for clippy. + fn run(self, builder: &Builder<'_>) { + let stage = self.stage; + let host = self.host; + let compiler = builder.compiler(stage, host); + + builder + .ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() }) + .expect("in-tree tool"); + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + host, + "test", + "src/tools/clippy", + SourceType::InTree, + &[], + ); + + cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); + cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); + let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir()); + cargo.env("HOST_LIBS", host_libs); + + cargo.add_rustc_lib_path(builder, compiler); + let mut cargo = prepare_cargo_test(cargo, &[], &[], "clippy", compiler, host, builder); + + let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host); + + // Clippy reports errors if it blessed the outputs + if builder.run_cmd(BootstrapCommand::from(&mut cargo).allow_failure()) { + // The tests succeeded; nothing to do. + return; + } + + if !builder.config.cmd.bless() { + crate::exit!(1); + } + } +} + +fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString { + // Configure PATH to find the right rustc. NB. we have to use PATH + // and not RUSTC because the Cargo test suite has tests that will + // fail if rustc is not spelled `rustc`. + let path = builder.sysroot(compiler).join("bin"); + let old_path = env::var_os("PATH").unwrap_or_default(); + env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("") +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocTheme { + pub compiler: Compiler, +} + +impl Step for RustdocTheme { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustdoc-themes") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.target); + + run.builder.ensure(RustdocTheme { compiler }); + } + + fn run(self, builder: &Builder<'_>) { + let rustdoc = builder.bootstrap_out.join("rustdoc"); + let mut cmd = builder.tool_cmd(Tool::RustdocTheme); + cmd.arg(rustdoc.to_str().unwrap()) + .arg(builder.src.join("src/librustdoc/html/static/css/rustdoc.css").to_str().unwrap()) + .env("RUSTC_STAGE", self.compiler.stage.to_string()) + .env("RUSTC_SYSROOT", builder.sysroot(self.compiler)) + .env("RUSTDOC_LIBDIR", builder.sysroot_libdir(self.compiler, self.compiler.host)) + .env("CFG_RELEASE_CHANNEL", &builder.config.channel) + .env("RUSTDOC_REAL", builder.rustdoc(self.compiler)) + .env("RUSTC_BOOTSTRAP", "1"); + if let Some(linker) = builder.linker(self.compiler.host) { + cmd.env("RUSTDOC_LINKER", linker); + } + if builder.is_fuse_ld_lld(self.compiler.host) { + cmd.env( + "RUSTDOC_LLD_NO_THREADS", + helpers::lld_flag_no_threads(self.compiler.host.contains("windows")), + ); + } + builder.run_delaying_failure(&mut cmd); + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocJSStd { + pub target: TargetSelection, +} + +impl Step for RustdocJSStd { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.nodejs.is_some(); + run.suite_path("tests/rustdoc-js-std").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustdocJSStd { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let nodejs = + builder.config.nodejs.as_ref().expect("need nodejs to run rustdoc-js-std tests"); + let mut command = Command::new(nodejs); + command + .arg(builder.src.join("src/tools/rustdoc-js/tester.js")) + .arg("--crate-name") + .arg("std") + .arg("--resource-suffix") + .arg(&builder.version) + .arg("--doc-folder") + .arg(builder.doc_out(self.target)) + .arg("--test-folder") + .arg(builder.src.join("tests/rustdoc-js-std")); + for path in &builder.paths { + if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) + { + if !p.ends_with(".js") { + eprintln!("A non-js file was given: `{}`", path.display()); + panic!("Cannot run rustdoc-js-std tests"); + } + command.arg("--test-file").arg(path); + } + } + builder.ensure(crate::core::build_steps::doc::Std::new( + builder.top_stage, + self.target, + builder, + DocumentationFormat::HTML, + )); + let _guard = builder.msg( + Kind::Test, + builder.top_stage, + "rustdoc-js-std", + builder.config.build, + self.target, + ); + builder.run(&mut command); + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocJSNotStd { + pub target: TargetSelection, + pub compiler: Compiler, +} + +impl Step for RustdocJSNotStd { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let default = run.builder.config.nodejs.is_some(); + run.suite_path("tests/rustdoc-js").default_condition(default) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocJSNotStd { target: run.target, compiler }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: "js-doc-test", + suite: "rustdoc-js", + path: "tests/rustdoc-js", + compare_mode: None, + }); + } +} + +fn get_browser_ui_test_version_inner(npm: &Path, global: bool) -> Option { + let mut command = Command::new(&npm); + command.arg("list").arg("--parseable").arg("--long").arg("--depth=0"); + if global { + command.arg("--global"); + } + let lines = command + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).into_owned()) + .unwrap_or(String::new()); + lines + .lines() + .find_map(|l| l.split(':').nth(1)?.strip_prefix("browser-ui-test@")) + .map(|v| v.to_owned()) +} + +fn get_browser_ui_test_version(npm: &Path) -> Option { + get_browser_ui_test_version_inner(npm, false) + .or_else(|| get_browser_ui_test_version_inner(npm, true)) +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustdocGUI { + pub target: TargetSelection, + pub compiler: Compiler, +} + +impl Step for RustdocGUI { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + let run = run.suite_path("tests/rustdoc-gui"); + run.lazy_default_condition(Box::new(move || { + builder.config.nodejs.is_some() + && builder + .config + .npm + .as_ref() + .map(|p| get_browser_ui_test_version(p).is_some()) + .unwrap_or(false) + })) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(RustdocGUI { target: run.target, compiler }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(compile::Std::new(self.compiler, self.target)); + + let mut cmd = builder.tool_cmd(Tool::RustdocGUITest); + + let out_dir = builder.test_out(self.target).join("rustdoc-gui"); + builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler)); + + if let Some(src) = builder.config.src.to_str() { + cmd.arg("--rust-src").arg(src); + } + + if let Some(out_dir) = out_dir.to_str() { + cmd.arg("--out-dir").arg(out_dir); + } + + if let Some(initial_cargo) = builder.config.initial_cargo.to_str() { + cmd.arg("--initial-cargo").arg(initial_cargo); + } + + cmd.arg("--jobs").arg(builder.jobs().to_string()); + + cmd.env("RUSTDOC", builder.rustdoc(self.compiler)) + .env("RUSTC", builder.rustc(self.compiler)); + + for path in &builder.paths { + if let Some(p) = helpers::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { + if !p.ends_with(".goml") { + eprintln!("A non-goml file was given: `{}`", path.display()); + panic!("Cannot run rustdoc-gui tests"); + } + if let Some(name) = path.file_name().and_then(|f| f.to_str()) { + cmd.arg("--goml-file").arg(name); + } + } + } + + for test_arg in builder.config.test_args() { + cmd.arg("--test-arg").arg(test_arg); + } + + if let Some(ref nodejs) = builder.config.nodejs { + cmd.arg("--nodejs").arg(nodejs); + } + + if let Some(ref npm) = builder.config.npm { + cmd.arg("--npm").arg(npm); + } + + let _time = helpers::timeit(&builder); + let _guard = builder.msg_sysroot_tool( + Kind::Test, + self.compiler.stage, + "rustdoc-gui", + self.compiler.host, + self.target, + ); + try_run_tests(builder, &mut cmd, true); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Tidy; + +impl Step for Tidy { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Runs the `tidy` tool. + /// + /// This tool in `src/tools` checks up on various bits and pieces of style and + /// otherwise just implements a few lint-like checks that are specific to the + /// compiler itself. + /// + /// Once tidy passes, this step also runs `fmt --check` if tests are being run + /// for the `dev` or `nightly` channels. + fn run(self, builder: &Builder<'_>) { + let mut cmd = builder.tool_cmd(Tool::Tidy); + cmd.arg(&builder.src); + cmd.arg(&builder.initial_cargo); + cmd.arg(&builder.out); + // Tidy is heavily IO constrained. Still respect `-j`, but use a higher limit if `jobs` hasn't been configured. + let jobs = builder.config.jobs.unwrap_or_else(|| { + 8 * std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 + }); + cmd.arg(jobs.to_string()); + if builder.is_verbose() { + cmd.arg("--verbose"); + } + if builder.config.cmd.bless() { + cmd.arg("--bless"); + } + if let Some(s) = builder.config.cmd.extra_checks() { + cmd.arg(format!("--extra-checks={s}")); + } + let mut args = std::env::args_os(); + if let Some(_) = args.find(|arg| arg == OsStr::new("--")) { + cmd.arg("--"); + cmd.args(args); + } + + if builder.config.channel == "dev" || builder.config.channel == "nightly" { + builder.info("fmt check"); + if builder.initial_rustfmt().is_none() { + let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); + eprintln!( + "\ +ERROR: no `rustfmt` binary found in {PATH} +INFO: `rust.channel` is currently set to \"{CHAN}\" +HELP: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file +HELP: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to `x.py test`", + PATH = inferred_rustfmt_dir.display(), + CHAN = builder.config.channel, + ); + crate::exit!(1); + } + crate::core::build_steps::format::format(&builder, !builder.config.cmd.bless(), &[]); + } + + builder.info("tidy check"); + builder.run_delaying_failure(&mut cmd); + + builder.ensure(ExpandYamlAnchors); + + builder.info("x.py completions check"); + let [bash, zsh, fish, powershell] = ["x.py.sh", "x.py.zsh", "x.py.fish", "x.py.ps1"] + .map(|filename| builder.src.join("src/etc/completions").join(filename)); + if builder.config.cmd.bless() { + builder.ensure(crate::core::build_steps::run::GenerateCompletions); + } else if get_completion(shells::Bash, &bash).is_some() + || get_completion(shells::Fish, &fish).is_some() + || get_completion(shells::PowerShell, &powershell).is_some() + || crate::flags::get_completion(shells::Zsh, &zsh).is_some() + { + eprintln!( + "x.py completions were changed; run `x.py run generate-completions` to update them" + ); + crate::exit!(1); + } + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/tidy") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Tidy); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ExpandYamlAnchors; + +impl Step for ExpandYamlAnchors { + type Output = (); + const ONLY_HOSTS: bool = true; + + /// Ensure the `generate-ci-config` tool was run locally. + /// + /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the + /// appropriate configuration for all our CI providers. This step ensures the tool was called + /// by the user before committing CI changes. + fn run(self, builder: &Builder<'_>) { + // NOTE: `.github/` is not included in dist-src tarballs + if !builder.src.join(".github/workflows/ci.yml").exists() { + builder.info("Skipping YAML anchors check: GitHub Actions config not found"); + return; + } + builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded"); + builder.run_delaying_failure( + &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src), + ); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/expand-yaml-anchors") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ExpandYamlAnchors); + } +} + +fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf { + builder.out.join(host.triple).join("test") +} + +macro_rules! default_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false }); + }; +} + +macro_rules! default_test_with_compare_mode { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, + compare_mode: $compare_mode:expr }) => { + test_with_compare_mode!($name { + path: $path, + mode: $mode, + suite: $suite, + default: true, + host: false, + compare_mode: $compare_mode + }); + }; +} + +macro_rules! host_test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { + test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true }); + }; +} + +macro_rules! test { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, + host: $host:expr }) => { + test_definitions!($name { + path: $path, + mode: $mode, + suite: $suite, + default: $default, + host: $host, + compare_mode: None + }); + }; +} + +macro_rules! test_with_compare_mode { + ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, + host: $host:expr, compare_mode: $compare_mode:expr }) => { + test_definitions!($name { + path: $path, + mode: $mode, + suite: $suite, + default: $default, + host: $host, + compare_mode: Some($compare_mode) + }); + }; +} + +macro_rules! test_definitions { + ($name:ident { + path: $path:expr, + mode: $mode:expr, + suite: $suite:expr, + default: $default:expr, + host: $host:expr, + compare_mode: $compare_mode:expr + }) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = $host; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path($path) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure($name { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode: $mode, + suite: $suite, + path: $path, + compare_mode: $compare_mode, + }) + } + } + }; +} + +/// Declares an alias for running the [`Coverage`] tests in only one mode. +/// Adapted from [`test_definitions`]. +macro_rules! coverage_test_alias { + ($name:ident { + alias_and_mode: $alias_and_mode:expr, + default: $default:expr, + only_hosts: $only_hosts:expr $(,)? + }) => { + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl $name { + const MODE: &'static str = $alias_and_mode; + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = $only_hosts; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias($alias_and_mode) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure($name { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + Coverage { compiler: self.compiler, target: self.target } + .run_unified_suite(builder, Self::MODE) + } + } + }; +} + +default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" }); + +default_test!(RunPassValgrind { + path: "tests/run-pass-valgrind", + mode: "run-pass-valgrind", + suite: "run-pass-valgrind" +}); + +default_test!(Codegen { path: "tests/codegen", mode: "codegen", suite: "codegen" }); + +default_test!(CodegenUnits { + path: "tests/codegen-units", + mode: "codegen-units", + suite: "codegen-units" +}); + +default_test!(Incremental { path: "tests/incremental", mode: "incremental", suite: "incremental" }); + +default_test_with_compare_mode!(Debuginfo { + path: "tests/debuginfo", + mode: "debuginfo", + suite: "debuginfo", + compare_mode: "split-dwarf" +}); + +host_test!(UiFullDeps { path: "tests/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); + +host_test!(Rustdoc { path: "tests/rustdoc", mode: "rustdoc", suite: "rustdoc" }); +host_test!(RustdocUi { path: "tests/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" }); + +host_test!(RustdocJson { path: "tests/rustdoc-json", mode: "rustdoc-json", suite: "rustdoc-json" }); + +host_test!(Pretty { path: "tests/pretty", mode: "pretty", suite: "pretty" }); + +default_test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make" }); + +host_test!(RunMakeFullDeps { + path: "tests/run-make-fulldeps", + mode: "run-make", + suite: "run-make-fulldeps" +}); + +default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); + +/// Custom test step that is responsible for running the coverage tests +/// in multiple different modes. +/// +/// Each individual mode also has its own alias that will run the tests in +/// just that mode. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Coverage { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Coverage { + const PATH: &'static str = "tests/coverage"; + const SUITE: &'static str = "coverage"; + + fn run_unified_suite(&self, builder: &Builder<'_>, mode: &'static str) { + builder.ensure(Compiletest { + compiler: self.compiler, + target: self.target, + mode, + suite: Self::SUITE, + path: Self::PATH, + compare_mode: None, + }) + } +} + +impl Step for Coverage { + type Output = (); + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path(Self::PATH) + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + + run.builder.ensure(Coverage { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + self.run_unified_suite(builder, CoverageMap::MODE); + self.run_unified_suite(builder, CoverageRun::MODE); + } +} + +// Aliases for running the coverage tests in only one mode. +coverage_test_alias!(CoverageMap { + alias_and_mode: "coverage-map", + default: true, + only_hosts: false, +}); +coverage_test_alias!(CoverageRun { + alias_and_mode: "coverage-run", + default: true, + only_hosts: true, +}); + +host_test!(CoverageRunRustdoc { + path: "tests/coverage-run-rustdoc", + mode: "coverage-run", + suite: "coverage-run-rustdoc" +}); + +// For the mir-opt suite we do not use macros, as we need custom behavior when blessing. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct MirOpt { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for MirOpt { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = false; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.suite_path("tests/mir-opt") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); + run.builder.ensure(MirOpt { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let run = |target| { + builder.ensure(Compiletest { + compiler: self.compiler, + target, + mode: "mir-opt", + suite: "mir-opt", + path: "tests/mir-opt", + compare_mode: None, + }) + }; + + // We use custom logic to bless the mir-opt suite: mir-opt tests have multiple variants + // (32bit vs 64bit, and panic=abort vs panic=unwind), and all of them needs to be blessed. + // When blessing, we try best-effort to also bless the other variants, to aid developers. + if builder.config.cmd.bless() { + let targets = MIR_OPT_BLESS_TARGET_MAPPING + .iter() + .filter(|(target_32bit, target_64bit)| { + *target_32bit == &*self.target.triple || *target_64bit == &*self.target.triple + }) + .next() + .map(|(target_32bit, target_64bit)| { + let target_32bit = TargetSelection::from_user(target_32bit); + let target_64bit = TargetSelection::from_user(target_64bit); + + // Running compiletest requires a C compiler to be available, but it might not + // have been detected by bootstrap if the target we're testing wasn't in the + // --target flags. + if !builder.cc.borrow().contains_key(&target_32bit) { + utils::cc_detect::find_target(builder, target_32bit); + } + if !builder.cc.borrow().contains_key(&target_64bit) { + utils::cc_detect::find_target(builder, target_64bit); + } + + vec![target_32bit, target_64bit] + }) + .unwrap_or_else(|| { + eprintln!( + "\ +Note that not all variants of mir-opt tests are going to be blessed, as no mapping between +a 32bit and a 64bit target was found for {target}. +You can add that mapping by changing MIR_OPT_BLESS_TARGET_MAPPING in src/bootstrap/test.rs", + target = self.target, + ); + vec![self.target] + }); + + for target in targets { + run(target); + + let panic_abort_target = builder.ensure(MirOptPanicAbortSyntheticTarget { + compiler: self.compiler, + base: target, + }); + run(panic_abort_target); + } + } else { + run(self.target); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +struct Compiletest { + compiler: Compiler, + target: TargetSelection, + mode: &'static str, + suite: &'static str, + path: &'static str, + compare_mode: Option<&'static str>, +} + +impl Step for Compiletest { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Executes the `compiletest` tool to run a suite of tests. + /// + /// Compiles all tests with `compiler` for `target` with the specified + /// compiletest `mode` and `suite` arguments. For example `mode` can be + /// "run-pass" or `suite` can be something like `debuginfo`. + fn run(self, builder: &Builder<'_>) { + if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() { + eprintln!("\ +ERROR: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail +HELP: to test the compiler, use `--stage 1` instead +HELP: to test the standard library, use `--stage 0 library/std` instead +NOTE: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." + ); + crate::exit!(1); + } + + let mut compiler = self.compiler; + let target = self.target; + let mode = self.mode; + let suite = self.suite; + + // Path for test suite + let suite_path = self.path; + + // Skip codegen tests if they aren't enabled in configuration. + if !builder.config.codegen_tests && suite == "codegen" { + return; + } + + // Support stage 1 ui-fulldeps. This is somewhat complicated: ui-fulldeps tests for the most + // part test the *API* of the compiler, not how it compiles a given file. As a result, we + // can run them against the stage 1 sources as long as we build them with the stage 0 + // bootstrap compiler. + // NOTE: Only stage 1 is special cased because we need the rustc_private artifacts to match the + // running compiler in stage 2 when plugins run. + let stage_id = if suite == "ui-fulldeps" && compiler.stage == 1 { + compiler = builder.compiler(compiler.stage - 1, target); + format!("stage{}-{}", compiler.stage + 1, target) + } else { + format!("stage{}-{}", compiler.stage, target) + }; + + if suite.ends_with("fulldeps") { + builder.ensure(compile::Rustc::new(compiler, target)); + } + + if suite == "debuginfo" { + builder + .ensure(dist::DebuggerScripts { sysroot: builder.sysroot(compiler), host: target }); + } + + builder.ensure(compile::Std::new(compiler, target)); + // ensure that `libproc_macro` is available on the host. + builder.ensure(compile::Std::new(compiler, compiler.host)); + + // Also provide `rust_test_helpers` for the host. + builder.ensure(TestHelpers { target: compiler.host }); + + // As well as the target, except for plain wasm32, which can't build it + if !target.contains("wasm") || target.contains("emscripten") { + builder.ensure(TestHelpers { target }); + } + + builder.ensure(RemoteCopyLibs { compiler, target }); + + let mut cmd = builder.tool_cmd(Tool::Compiletest); + + // compiletest currently has... a lot of arguments, so let's just pass all + // of them! + + cmd.arg("--compile-lib-path").arg(builder.rustc_libdir(compiler)); + cmd.arg("--run-lib-path").arg(builder.sysroot_libdir(compiler, target)); + cmd.arg("--rustc-path").arg(builder.rustc(compiler)); + + let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js"); + + // Avoid depending on rustdoc when we don't need it. + if mode == "rustdoc" + || mode == "run-make" + || (mode == "ui" && is_rustdoc) + || mode == "js-doc-test" + || mode == "rustdoc-json" + || suite == "coverage-run-rustdoc" + { + cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler)); + } + + if mode == "rustdoc-json" { + // Use the beta compiler for jsondocck + let json_compiler = compiler.with_stage(0); + cmd.arg("--jsondocck-path") + .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); + cmd.arg("--jsondoclint-path") + .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); + } + + if mode == "coverage-map" { + let coverage_dump = builder.ensure(tool::CoverageDump { + compiler: compiler.with_stage(0), + target: compiler.host, + }); + cmd.arg("--coverage-dump-path").arg(coverage_dump); + } + + if mode == "coverage-run" { + // The demangler doesn't need the current compiler, so we can avoid + // unnecessary rebuilds by using the bootstrap compiler instead. + let rust_demangler = builder + .ensure(tool::RustDemangler { + compiler: compiler.with_stage(0), + target: compiler.host, + extra_features: Vec::new(), + }) + .expect("in-tree tool"); + cmd.arg("--rust-demangler-path").arg(rust_demangler); + } + + cmd.arg("--src-base").arg(builder.src.join("tests").join(suite)); + cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); + + // When top stage is 0, that means that we're testing an externally provided compiler. + // In that case we need to use its specific sysroot for tests to pass. + let sysroot = if builder.top_stage == 0 { + builder.initial_sysroot.clone() + } else { + builder.sysroot(compiler).to_path_buf() + }; + cmd.arg("--sysroot-base").arg(sysroot); + cmd.arg("--stage-id").arg(stage_id); + cmd.arg("--suite").arg(suite); + cmd.arg("--mode").arg(mode); + cmd.arg("--target").arg(target.rustc_target_arg()); + cmd.arg("--host").arg(&*compiler.host.triple); + cmd.arg("--llvm-filecheck").arg(builder.llvm_filecheck(builder.config.build)); + + if builder.config.cmd.bless() { + cmd.arg("--bless"); + } + + if builder.config.cmd.force_rerun() { + cmd.arg("--force-rerun"); + } + + let compare_mode = + builder.config.cmd.compare_mode().or_else(|| { + if builder.config.test_compare_mode { self.compare_mode } else { None } + }); + + if let Some(ref pass) = builder.config.cmd.pass() { + cmd.arg("--pass"); + cmd.arg(pass); + } + + if let Some(ref run) = builder.config.cmd.run() { + cmd.arg("--run"); + cmd.arg(run); + } + + if let Some(ref nodejs) = builder.config.nodejs { + cmd.arg("--nodejs").arg(nodejs); + } else if mode == "js-doc-test" { + panic!("need nodejs to run js-doc-test suite"); + } + if let Some(ref npm) = builder.config.npm { + cmd.arg("--npm").arg(npm); + } + if builder.config.rust_optimize_tests { + cmd.arg("--optimize-tests"); + } + if builder.config.cmd.only_modified() { + cmd.arg("--only-modified"); + } + + let mut flags = if is_rustdoc { Vec::new() } else { vec!["-Crpath".to_string()] }; + flags.push(format!("-Cdebuginfo={}", builder.config.rust_debuginfo_level_tests)); + flags.extend(builder.config.cmd.rustc_args().iter().map(|s| s.to_string())); + + if let Some(linker) = builder.linker(target) { + cmd.arg("--target-linker").arg(linker); + } + if let Some(linker) = builder.linker(compiler.host) { + cmd.arg("--host-linker").arg(linker); + } + + let mut hostflags = flags.clone(); + hostflags.push(format!("-Lnative={}", builder.test_helpers_out(compiler.host).display())); + hostflags.extend(builder.lld_flags(compiler.host)); + for flag in hostflags { + cmd.arg("--host-rustcflags").arg(flag); + } + + let mut targetflags = flags; + targetflags.push(format!("-Lnative={}", builder.test_helpers_out(target).display())); + targetflags.extend(builder.lld_flags(target)); + for flag in targetflags { + cmd.arg("--target-rustcflags").arg(flag); + } + + cmd.arg("--python").arg(builder.python()); + + if let Some(ref gdb) = builder.config.gdb { + cmd.arg("--gdb").arg(gdb); + } + + let run = |cmd: &mut Command| { + cmd.output().map(|output| { + String::from_utf8_lossy(&output.stdout) + .lines() + .next() + .unwrap_or_else(|| panic!("{:?} failed {:?}", cmd, output)) + .to_string() + }) + }; + let lldb_exe = "lldb"; + let lldb_version = Command::new(lldb_exe) + .arg("--version") + .output() + .map(|output| String::from_utf8_lossy(&output.stdout).to_string()) + .ok(); + if let Some(ref vers) = lldb_version { + cmd.arg("--lldb-version").arg(vers); + let lldb_python_dir = run(Command::new(lldb_exe).arg("-P")).ok(); + if let Some(ref dir) = lldb_python_dir { + cmd.arg("--lldb-python-dir").arg(dir); + } + } + + if helpers::forcing_clang_based_tests() { + let clang_exe = builder.llvm_out(target).join("bin").join("clang"); + cmd.arg("--run-clang-based-tests-with").arg(clang_exe); + } + + for exclude in &builder.config.skip { + cmd.arg("--skip"); + cmd.arg(&exclude); + } + + // Get paths from cmd args + let paths = match &builder.config.cmd { + Subcommand::Test { .. } => &builder.config.paths[..], + _ => &[], + }; + + // Get test-args by striping suite path + let mut test_args: Vec<&str> = paths + .iter() + .filter_map(|p| helpers::is_valid_test_suite_arg(p, suite_path, builder)) + .collect(); + + test_args.append(&mut builder.config.test_args()); + + // On Windows, replace forward slashes in test-args by backslashes + // so the correct filters are passed to libtest + if cfg!(windows) { + let test_args_win: Vec = + test_args.iter().map(|s| s.replace("/", "\\")).collect(); + cmd.args(&test_args_win); + } else { + cmd.args(&test_args); + } + + if builder.is_verbose() { + cmd.arg("--verbose"); + } + + cmd.arg("--json"); + + let mut llvm_components_passed = false; + let mut copts_passed = false; + if builder.config.llvm_enabled() { + let llvm::LlvmResult { llvm_config, .. } = + builder.ensure(llvm::Llvm { target: builder.config.build }); + if !builder.config.dry_run() { + let llvm_version = output(Command::new(&llvm_config).arg("--version")); + let llvm_components = output(Command::new(&llvm_config).arg("--components")); + // Remove trailing newline from llvm-config output. + cmd.arg("--llvm-version") + .arg(llvm_version.trim()) + .arg("--llvm-components") + .arg(llvm_components.trim()); + llvm_components_passed = true; + } + if !builder.is_rust_llvm(target) { + cmd.arg("--system-llvm"); + } + + // Tests that use compiler libraries may inherit the `-lLLVM` link + // requirement, but the `-L` library path is not propagated across + // separate compilations. We can add LLVM's library path to the + // platform-specific environment variable as a workaround. + if !builder.config.dry_run() && suite.ends_with("fulldeps") { + let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); + add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd); + } + + if !builder.config.dry_run() + && (matches!(suite, "run-make" | "run-make-fulldeps") || mode == "coverage-run") + { + // The llvm/bin directory contains many useful cross-platform + // tools. Pass the path to run-make tests so they can use them. + // (The coverage-run tests also need these tools to process + // coverage reports.) + let llvm_bin_path = llvm_config + .parent() + .expect("Expected llvm-config to be contained in directory"); + assert!(llvm_bin_path.is_dir()); + cmd.arg("--llvm-bin-dir").arg(llvm_bin_path); + } + + if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { + // If LLD is available, add it to the PATH + if builder.config.lld_enabled { + let lld_install_root = + builder.ensure(llvm::Lld { target: builder.config.build }); + + let lld_bin_path = lld_install_root.join("bin"); + + let old_path = env::var_os("PATH").unwrap_or_default(); + let new_path = env::join_paths( + std::iter::once(lld_bin_path).chain(env::split_paths(&old_path)), + ) + .expect("Could not add LLD bin path to PATH"); + cmd.env("PATH", new_path); + } + } + } + + // Only pass correct values for these flags for the `run-make` suite as it + // requires that a C++ compiler was configured which isn't always the case. + if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { + cmd.arg("--cc") + .arg(builder.cc(target)) + .arg("--cxx") + .arg(builder.cxx(target).unwrap()) + .arg("--cflags") + .arg(builder.cflags(target, GitRepo::Rustc, CLang::C).join(" ")) + .arg("--cxxflags") + .arg(builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" ")); + copts_passed = true; + if let Some(ar) = builder.ar(target) { + cmd.arg("--ar").arg(ar); + } + } + + if !llvm_components_passed { + cmd.arg("--llvm-components").arg(""); + } + if !copts_passed { + cmd.arg("--cc") + .arg("") + .arg("--cxx") + .arg("") + .arg("--cflags") + .arg("") + .arg("--cxxflags") + .arg(""); + } + + if builder.remote_tested(target) { + cmd.arg("--remote-test-client").arg(builder.tool_exe(Tool::RemoteTestClient)); + } + + // Running a C compiler on MSVC requires a few env vars to be set, to be + // sure to set them here. + // + // Note that if we encounter `PATH` we make sure to append to our own `PATH` + // rather than stomp over it. + if !builder.config.dry_run() && target.contains("msvc") { + for &(ref k, ref v) in builder.cc.borrow()[&target].env() { + if k != "PATH" { + cmd.env(k, v); + } + } + } + cmd.env("RUSTC_BOOTSTRAP", "1"); + // Override the rustc version used in symbol hashes to reduce the amount of normalization + // needed when diffing test output. + cmd.env("RUSTC_FORCE_RUSTC_VERSION", "compiletest"); + cmd.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); + builder.add_rust_test_threads(&mut cmd); + + if builder.config.sanitizers_enabled(target) { + cmd.env("RUSTC_SANITIZER_SUPPORT", "1"); + } + + if builder.config.profiler_enabled(target) { + cmd.env("RUSTC_PROFILER_SUPPORT", "1"); + } + + cmd.env("RUST_TEST_TMPDIR", builder.tempdir()); + + cmd.arg("--adb-path").arg("adb"); + cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); + if target.contains("android") && !builder.config.dry_run() { + // Assume that cc for this target comes from the android sysroot + cmd.arg("--android-cross-path") + .arg(builder.cc(target).parent().unwrap().parent().unwrap()); + } else { + cmd.arg("--android-cross-path").arg(""); + } + + if builder.config.cmd.rustfix_coverage() { + cmd.arg("--rustfix-coverage"); + } + + cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo); + + cmd.arg("--channel").arg(&builder.config.channel); + + if !builder.config.omit_git_hash { + cmd.arg("--git-hash"); + } + + let git_config = builder.config.git_config(); + cmd.arg("--git-repository").arg(git_config.git_repository); + cmd.arg("--nightly-branch").arg(git_config.nightly_branch); + + builder.ci_env.force_coloring_in_ci(&mut cmd); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::Compiletest { + suite: suite.into(), + mode: mode.into(), + compare_mode: None, + target: self.target.triple.to_string(), + host: self.compiler.host.triple.to_string(), + stage: self.compiler.stage, + }, + builder, + ); + + let _group = builder.msg( + Kind::Test, + compiler.stage, + &format!("compiletest suite={suite} mode={mode}"), + compiler.host, + target, + ); + try_run_tests(builder, &mut cmd, false); + + if let Some(compare_mode) = compare_mode { + cmd.arg("--compare-mode").arg(compare_mode); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::Compiletest { + suite: suite.into(), + mode: mode.into(), + compare_mode: Some(compare_mode.into()), + target: self.target.triple.to_string(), + host: self.compiler.host.triple.to_string(), + stage: self.compiler.stage, + }, + builder, + ); + + builder.info(&format!( + "Check compiletest suite={} mode={} compare_mode={} ({} -> {})", + suite, mode, compare_mode, &compiler.host, target + )); + let _time = helpers::timeit(&builder); + try_run_tests(builder, &mut cmd, false); + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +struct BookTest { + compiler: Compiler, + path: PathBuf, + name: &'static str, + is_ext_doc: bool, +} + +impl Step for BookTest { + type Output = (); + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Runs the documentation tests for a book in `src/doc`. + /// + /// This uses the `rustdoc` that sits next to `compiler`. + fn run(self, builder: &Builder<'_>) { + // External docs are different from local because: + // - Some books need pre-processing by mdbook before being tested. + // - They need to save their state to toolstate. + // - They are only tested on the "checktools" builders. + // + // The local docs are tested by default, and we don't want to pay the + // cost of building mdbook, so they use `rustdoc --test` directly. + // Also, the unstable book is special because SUMMARY.md is generated, + // so it is easier to just run `rustdoc` on its files. + if self.is_ext_doc { + self.run_ext_doc(builder); + } else { + self.run_local_doc(builder); + } + } +} + +impl BookTest { + /// This runs the equivalent of `mdbook test` (via the rustbook wrapper) + /// which in turn runs `rustdoc --test` on each file in the book. + fn run_ext_doc(self, builder: &Builder<'_>) { + let compiler = self.compiler; + + builder.ensure(compile::Std::new(compiler, compiler.host)); + + // mdbook just executes a binary named "rustdoc", so we need to update + // PATH so that it points to our rustdoc. + let mut rustdoc_path = builder.rustdoc(compiler); + rustdoc_path.pop(); + let old_path = env::var_os("PATH").unwrap_or_default(); + let new_path = env::join_paths(iter::once(rustdoc_path).chain(env::split_paths(&old_path))) + .expect("could not add rustdoc to PATH"); + + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + let path = builder.src.join(&self.path); + // Books often have feature-gated example text. + rustbook_cmd.env("RUSTC_BOOTSTRAP", "1"); + rustbook_cmd.env("PATH", new_path).arg("test").arg(path); + builder.add_rust_test_threads(&mut rustbook_cmd); + let _guard = builder.msg( + Kind::Test, + compiler.stage, + format_args!("mdbook {}", self.path.display()), + compiler.host, + compiler.host, + ); + let _time = helpers::timeit(&builder); + let toolstate = if builder.run_delaying_failure(&mut rustbook_cmd) { + ToolState::TestPass + } else { + ToolState::TestFail + }; + builder.save_toolstate(self.name, toolstate); + } + + /// This runs `rustdoc --test` on all `.md` files in the path. + fn run_local_doc(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let host = self.compiler.host; + + builder.ensure(compile::Std::new(compiler, host)); + + let _guard = + builder.msg(Kind::Test, compiler.stage, &format!("book {}", self.name), host, host); + + // Do a breadth-first traversal of the `src/doc` directory and just run + // tests for all files that end in `*.md` + let mut stack = vec![builder.src.join(self.path)]; + let _time = helpers::timeit(&builder); + let mut files = Vec::new(); + while let Some(p) = stack.pop() { + if p.is_dir() { + stack.extend(t!(p.read_dir()).map(|p| t!(p).path())); + continue; + } + + if p.extension().and_then(|s| s.to_str()) != Some("md") { + continue; + } + + files.push(p); + } + + files.sort(); + + for file in files { + markdown_test(builder, compiler, &file); + } + } +} + +macro_rules! test_book { + ($($name:ident, $path:expr, $book_name:expr, default=$default:expr;)+) => { + $( + #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] + pub struct $name { + compiler: Compiler, + } + + impl Step for $name { + type Output = (); + const DEFAULT: bool = $default; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path($path) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.target), + }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(BookTest { + compiler: self.compiler, + path: PathBuf::from($path), + name: $book_name, + is_ext_doc: !$default, + }); + } + } + )+ + } +} + +test_book!( + Nomicon, "src/doc/nomicon", "nomicon", default=false; + Reference, "src/doc/reference", "reference", default=false; + RustdocBook, "src/doc/rustdoc", "rustdoc", default=true; + RustcBook, "src/doc/rustc", "rustc", default=true; + RustByExample, "src/doc/rust-by-example", "rust-by-example", default=false; + EmbeddedBook, "src/doc/embedded-book", "embedded-book", default=false; + TheBook, "src/doc/book", "book", default=false; + UnstableBook, "src/doc/unstable-book", "unstable-book", default=true; + EditionGuide, "src/doc/edition-guide", "edition-guide", default=false; +); + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ErrorIndex { + compiler: Compiler, +} + +impl Step for ErrorIndex { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/error_index_generator") + } + + fn make_run(run: RunConfig<'_>) { + // error_index_generator depends on librustdoc. Use the compiler that + // is normally used to build rustdoc for other tests (like compiletest + // tests in tests/rustdoc) so that it shares the same artifacts. + let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); + run.builder.ensure(ErrorIndex { compiler }); + } + + /// Runs the error index generator tool to execute the tests located in the error + /// index. + /// + /// The `error_index_generator` tool lives in `src/tools` and is used to + /// generate a markdown file from the error indexes of the code base which is + /// then passed to `rustdoc --test`. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + + let dir = testdir(builder, compiler.host); + t!(fs::create_dir_all(&dir)); + let output = dir.join("error-index.md"); + + let mut tool = tool::ErrorIndex::command(builder); + tool.arg("markdown").arg(&output); + + let guard = + builder.msg(Kind::Test, compiler.stage, "error-index", compiler.host, compiler.host); + let _time = helpers::timeit(&builder); + builder.run_quiet(&mut tool); + drop(guard); + // The tests themselves need to link to std, so make sure it is + // available. + builder.ensure(compile::Std::new(compiler, compiler.host)); + markdown_test(builder, compiler, &output); + } +} + +fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> bool { + if let Ok(contents) = fs::read_to_string(markdown) { + if !contents.contains("```") { + return true; + } + } + + builder.verbose(&format!("doc tests for: {}", markdown.display())); + let mut cmd = builder.rustdoc_cmd(compiler); + builder.add_rust_test_threads(&mut cmd); + // allow for unstable options such as new editions + cmd.arg("-Z"); + cmd.arg("unstable-options"); + cmd.arg("--test"); + cmd.arg(markdown); + cmd.env("RUSTC_BOOTSTRAP", "1"); + + let test_args = builder.config.test_args().join(" "); + cmd.arg("--test-args").arg(test_args); + + if builder.config.verbose_tests { + builder.run_delaying_failure(&mut cmd) + } else { + builder.run_quiet_delaying_failure(&mut cmd) + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustcGuide; + +impl Step for RustcGuide { + type Output = (); + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/doc/rustc-dev-guide") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustcGuide); + } + + fn run(self, builder: &Builder<'_>) { + let relative_path = Path::new("src").join("doc").join("rustc-dev-guide"); + builder.update_submodule(&relative_path); + + let src = builder.src.join(relative_path); + let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); + let toolstate = if builder.run_delaying_failure(rustbook_cmd.arg("linkcheck").arg(&src)) { + ToolState::TestPass + } else { + ToolState::TestFail + }; + builder.save_toolstate("rustc-dev-guide", toolstate); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CrateLibrustc { + compiler: Compiler, + target: TargetSelection, + crates: Vec>, +} + +impl Step for CrateLibrustc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("rustc-main") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = builder.compiler_for(builder.top_stage, host, host); + let crates = run + .paths + .iter() + .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) + .collect(); + + builder.ensure(CrateLibrustc { compiler, target: run.target, crates }); + } + + fn run(self, builder: &Builder<'_>) { + builder.ensure(Crate { + compiler: self.compiler, + target: self.target, + mode: Mode::Rustc, + crates: self.crates, + }); + } +} + +/// Given a `cargo test` subcommand, add the appropriate flags and run it. +/// +/// Returns whether the test succeeded. +fn run_cargo_test<'a>( + cargo: impl Into, + libtest_args: &[&str], + crates: &[Interned], + primary_crate: &str, + description: impl Into>, + compiler: Compiler, + target: TargetSelection, + builder: &Builder<'_>, +) -> bool { + let mut cargo = + prepare_cargo_test(cargo, libtest_args, crates, primary_crate, compiler, target, builder); + let _time = helpers::timeit(&builder); + let _group = description.into().and_then(|what| { + builder.msg_sysroot_tool(Kind::Test, compiler.stage, what, compiler.host, target) + }); + + #[cfg(feature = "build-metrics")] + builder.metrics.begin_test_suite( + build_helper::metrics::TestSuiteMetadata::CargoPackage { + crates: crates.iter().map(|c| c.to_string()).collect(), + target: target.triple.to_string(), + host: compiler.host.triple.to_string(), + stage: compiler.stage, + }, + builder, + ); + add_flags_and_try_run_tests(builder, &mut cargo) +} + +/// Given a `cargo test` subcommand, pass it the appropriate test flags given a `builder`. +fn prepare_cargo_test( + cargo: impl Into, + libtest_args: &[&str], + crates: &[Interned], + primary_crate: &str, + compiler: Compiler, + target: TargetSelection, + builder: &Builder<'_>, +) -> Command { + let mut cargo = cargo.into(); + + // Propegate `--bless` if it has not already been set/unset + // Any tools that want to use this should bless if `RUSTC_BLESS` is set to + // anything other than `0`. + if builder.config.cmd.bless() && !cargo.get_envs().any(|v| v.0 == "RUSTC_BLESS") { + cargo.env("RUSTC_BLESS", "Gesundheit"); + } + + // Pass in some standard flags then iterate over the graph we've discovered + // in `cargo metadata` with the maps above and figure out what `-p` + // arguments need to get passed. + if builder.kind == Kind::Test && !builder.fail_fast { + cargo.arg("--no-fail-fast"); + } + match builder.doc_tests { + DocTests::Only => { + cargo.arg("--doc"); + } + DocTests::No => { + let krate = &builder + .crates + .get(&INTERNER.intern_str(primary_crate)) + .unwrap_or_else(|| panic!("missing crate {primary_crate}")); + if krate.has_lib { + cargo.arg("--lib"); + } + cargo.args(&["--bins", "--examples", "--tests", "--benches"]); + } + DocTests::Yes => {} + } + + for &krate in crates { + cargo.arg("-p").arg(krate); + } + + cargo.arg("--").args(&builder.config.test_args()).args(libtest_args); + if !builder.config.verbose_tests { + cargo.arg("--quiet"); + } + + // The tests are going to run with the *target* libraries, so we need to + // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent. + // + // Note that to run the compiler we need to run with the *host* libraries, + // but our wrapper scripts arrange for that to be the case anyway. + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target))); + cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + if target.contains("emscripten") { + cargo.env( + format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), + builder.config.nodejs.as_ref().expect("nodejs not configured"), + ); + } else if target.starts_with("wasm32") { + let node = builder.config.nodejs.as_ref().expect("nodejs not configured"); + let runner = format!("{} {}/src/etc/wasm32-shim.js", node.display(), builder.src.display()); + cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), &runner); + } else if builder.remote_tested(target) { + cargo.env( + format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), + format!("{} run 0", builder.tool_exe(Tool::RemoteTestClient).display()), + ); + } + + cargo +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct Crate { + pub compiler: Compiler, + pub target: TargetSelection, + pub mode: Mode, + pub crates: Vec>, +} + +impl Step for Crate { + type Output = (); + const DEFAULT: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.crate_or_deps("sysroot") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = builder.compiler_for(builder.top_stage, host, host); + let crates = run + .paths + .iter() + .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) + .collect(); + + builder.ensure(Crate { compiler, target: run.target, mode: Mode::Std, crates }); + } + + /// Runs all unit tests plus documentation tests for a given crate defined + /// by a `Cargo.toml` (single manifest) + /// + /// This is what runs tests for crates like the standard library, compiler, etc. + /// It essentially is the driver for running `cargo test`. + /// + /// Currently this runs all tests for a DAG by passing a bunch of `-p foo` + /// arguments, and those arguments are discovered from `cargo metadata`. + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + let mode = self.mode; + + // See [field@compile::Std::force_recompile]. + builder.ensure(compile::Std::force_recompile(compiler, target)); + builder.ensure(RemoteCopyLibs { compiler, target }); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let mut cargo = + builder.cargo(compiler, mode, SourceType::InTree, target, builder.kind.as_str()); + match mode { + Mode::Std => { + compile::std_cargo(builder, target, compiler.stage, &mut cargo); + // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, + // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. + // Override it. + if builder.download_rustc() && compiler.stage > 0 { + let sysroot = builder + .out + .join(compiler.host.triple) + .join(format!("stage{}-test-sysroot", compiler.stage)); + cargo.env("RUSTC_SYSROOT", sysroot); + } + } + Mode::Rustc => { + compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); + } + _ => panic!("can only test libraries"), + }; + + run_cargo_test( + cargo, + &[], + &self.crates, + &self.crates[0], + &*crate_description(&self.crates), + compiler, + target, + builder, + ); + } +} + +/// Rustdoc is special in various ways, which is why this step is different from `Crate`. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrateRustdoc { + host: TargetSelection, +} + +impl Step for CrateRustdoc { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["src/librustdoc", "src/tools/rustdoc"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + + builder.ensure(CrateRustdoc { host: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let target = self.host; + + let compiler = if builder.download_rustc() { + builder.compiler(builder.top_stage, target) + } else { + // Use the previous stage compiler to reuse the artifacts that are + // created when running compiletest for tests/rustdoc. If this used + // `compiler`, then it would cause rustdoc to be built *again*, which + // isn't really necessary. + builder.compiler_for(builder.top_stage, target, target) + }; + // NOTE: normally `ensure(Rustc)` automatically runs `ensure(Std)` for us. However, when + // using `download-rustc`, the rustc_private artifacts may be in a *different sysroot* from + // the target rustdoc (`ci-rustc-sysroot` vs `stage2`). In that case, we need to ensure this + // explicitly to make sure it ends up in the stage2 sysroot. + builder.ensure(compile::Std::new(compiler, target)); + builder.ensure(compile::Rustc::new(compiler, target)); + + let mut cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + builder.kind.as_str(), + "src/tools/rustdoc", + SourceType::InTree, + &[], + ); + if self.host.contains("musl") { + cargo.arg("'-Ctarget-feature=-crt-static'"); + } + + // This is needed for running doctests on librustdoc. This is a bit of + // an unfortunate interaction with how bootstrap works and how cargo + // sets up the dylib path, and the fact that the doctest (in + // html/markdown.rs) links to rustc-private libs. For stage1, the + // compiler host dylibs (in stage1/lib) are not the same as the target + // dylibs (in stage1/lib/rustlib/...). This is different from a normal + // rust distribution where they are the same. + // + // On the cargo side, normal tests use `target_process` which handles + // setting up the dylib for a *target* (stage1/lib/rustlib/... in this + // case). However, for doctests it uses `rustdoc_process` which only + // sets up the dylib path for the *host* (stage1/lib), which is the + // wrong directory. + // + // Recall that we special-cased `compiler_for(top_stage)` above, so we always use stage1. + // + // It should be considered to just stop running doctests on + // librustdoc. There is only one test, and it doesn't look too + // important. There might be other ways to avoid this, but it seems + // pretty convoluted. + // + // See also https://github.com/rust-lang/rust/issues/13983 where the + // host vs target dylibs for rustdoc are consistently tricky to deal + // with. + // + // Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution. + let libdir = if builder.download_rustc() { + builder.rustc_libdir(compiler) + } else { + builder.sysroot_libdir(compiler, target).to_path_buf() + }; + let mut dylib_path = dylib_path(); + dylib_path.insert(0, PathBuf::from(&*libdir)); + cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); + + run_cargo_test( + cargo, + &[], + &[INTERNER.intern_str("rustdoc:0.0.0")], + "rustdoc", + "rustdoc", + compiler, + target, + builder, + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct CrateRustdocJsonTypes { + host: TargetSelection, +} + +impl Step for CrateRustdocJsonTypes { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/rustdoc-json-types") + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + + builder.ensure(CrateRustdocJsonTypes { host: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let target = self.host; + + // Use the previous stage compiler to reuse the artifacts that are + // created when running compiletest for tests/rustdoc. If this used + // `compiler`, then it would cause rustdoc to be built *again*, which + // isn't really necessary. + let compiler = builder.compiler_for(builder.top_stage, target, target); + builder.ensure(compile::Rustc::new(compiler, target)); + + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolRustc, + target, + builder.kind.as_str(), + "src/rustdoc-json-types", + SourceType::InTree, + &[], + ); + + // FIXME: this looks very wrong, libtest doesn't accept `-C` arguments and the quotes are fishy. + let libtest_args = if self.host.contains("musl") { + ["'-Ctarget-feature=-crt-static'"].as_slice() + } else { + &[] + }; + + run_cargo_test( + cargo, + libtest_args, + &[INTERNER.intern_str("rustdoc-json-types")], + "rustdoc-json-types", + "rustdoc-json-types", + compiler, + target, + builder, + ); + } +} + +/// Some test suites are run inside emulators or on remote devices, and most +/// of our test binaries are linked dynamically which means we need to ship +/// the standard library and such to the emulator ahead of time. This step +/// represents this and is a dependency of all test suites. +/// +/// Most of the time this is a no-op. For some steps such as shipping data to +/// QEMU we have to build our own tools so we've got conditional dependencies +/// on those programs as well. Note that the remote test client is built for +/// the build target (us) and the server is built for the target. +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RemoteCopyLibs { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for RemoteCopyLibs { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + if !builder.remote_tested(target) { + return; + } + + builder.ensure(compile::Std::new(compiler, target)); + + builder.info(&format!("REMOTE copy libs to emulator ({target})")); + + let server = builder.ensure(tool::RemoteTestServer { compiler, target }); + + // Spawn the emulator and wait for it to come online + let tool = builder.tool_exe(Tool::RemoteTestClient); + let mut cmd = Command::new(&tool); + cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); + if let Some(rootfs) = builder.qemu_rootfs(target) { + cmd.arg(rootfs); + } + builder.run(&mut cmd); + + // Push all our dylibs to the emulator + for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) { + let f = t!(f); + let name = f.file_name().into_string().unwrap(); + if helpers::is_dylib(&name) { + builder.run(Command::new(&tool).arg("push").arg(f.path())); + } + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Distcheck; + +impl Step for Distcheck { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("distcheck") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Distcheck); + } + + /// Runs "distcheck", a 'make check' from a tarball + fn run(self, builder: &Builder<'_>) { + builder.info("Distcheck"); + let dir = builder.tempdir().join("distcheck"); + let _ = fs::remove_dir_all(&dir); + t!(fs::create_dir_all(&dir)); + + // Guarantee that these are built before we begin running. + builder.ensure(dist::PlainSourceTarball); + builder.ensure(dist::Src); + + let mut cmd = Command::new("tar"); + cmd.arg("-xf") + .arg(builder.ensure(dist::PlainSourceTarball).tarball()) + .arg("--strip-components=1") + .current_dir(&dir); + builder.run(&mut cmd); + builder.run( + Command::new("./configure") + .args(&builder.config.configure_args) + .arg("--enable-vendor") + .current_dir(&dir), + ); + builder.run( + Command::new(helpers::make(&builder.config.build.triple)) + .arg("check") + .current_dir(&dir), + ); + + // Now make sure that rust-src has all of libstd's dependencies + builder.info("Distcheck rust-src"); + let dir = builder.tempdir().join("distcheck-src"); + let _ = fs::remove_dir_all(&dir); + t!(fs::create_dir_all(&dir)); + + let mut cmd = Command::new("tar"); + cmd.arg("-xf") + .arg(builder.ensure(dist::Src).tarball()) + .arg("--strip-components=1") + .current_dir(&dir); + builder.run(&mut cmd); + + let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml"); + builder.run( + Command::new(&builder.initial_cargo) + // Will read the libstd Cargo.toml + // which uses the unstable `public-dependency` feature. + .env("RUSTC_BOOTSTRAP", "1") + .arg("generate-lockfile") + .arg("--manifest-path") + .arg(&toml) + .current_dir(&dir), + ); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct Bootstrap; + +impl Step for Bootstrap { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + /// Tests the build system itself. + fn run(self, builder: &Builder<'_>) { + let host = builder.config.build; + let compiler = builder.compiler(0, host); + let _guard = builder.msg(Kind::Test, 0, "bootstrap", host, host); + + let mut check_bootstrap = Command::new(&builder.python()); + check_bootstrap + .args(["-m", "unittest", "bootstrap_test.py"]) + .env("BUILD_DIR", &builder.out) + .env("BUILD_PLATFORM", &builder.build.build.triple) + .current_dir(builder.src.join("src/bootstrap/")); + // NOTE: we intentionally don't pass test_args here because the args for unittest and cargo test are mutually incompatible. + // Use `python -m unittest` manually if you want to pass arguments. + builder.run_delaying_failure(&mut check_bootstrap); + + let mut cmd = Command::new(&builder.initial_cargo); + cmd.arg("test") + .current_dir(builder.src.join("src/bootstrap")) + .env("RUSTFLAGS", "-Cdebuginfo=2") + .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) + .env("RUSTC_BOOTSTRAP", "1") + .env("RUSTDOC", builder.rustdoc(compiler)) + .env("RUSTC", &builder.initial_rustc); + if let Some(flags) = option_env!("RUSTFLAGS") { + // Use the same rustc flags for testing as for "normal" compilation, + // so that Cargo doesn’t recompile the entire dependency graph every time: + // https://github.com/rust-lang/rust/issues/49215 + cmd.env("RUSTFLAGS", flags); + } + // rustbuild tests are racy on directory creation so just run them one at a time. + // Since there's not many this shouldn't be a problem. + run_cargo_test(cmd, &["--test-threads=1"], &[], "bootstrap", None, compiler, host, builder); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/bootstrap") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Bootstrap); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct TierCheck { + pub compiler: Compiler, +} + +impl Step for TierCheck { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/tier-check") + } + + fn make_run(run: RunConfig<'_>) { + let compiler = + run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); + run.builder.ensure(TierCheck { compiler }); + } + + /// Tests the Platform Support page in the rustc book. + fn run(self, builder: &Builder<'_>) { + builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); + let mut cargo = tool::prepare_tool_cargo( + builder, + self.compiler, + Mode::ToolStd, + self.compiler.host, + "run", + "src/tools/tier-check", + SourceType::InTree, + &[], + ); + cargo.arg(builder.src.join("src/doc/rustc/src/platform-support.md")); + cargo.arg(&builder.rustc(self.compiler)); + if builder.is_verbose() { + cargo.arg("--verbose"); + } + + let _guard = builder.msg( + Kind::Test, + self.compiler.stage, + "platform support check", + self.compiler.host, + self.compiler.host, + ); + builder.run_delaying_failure(&mut cargo.into()); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct LintDocs { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for LintDocs { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/lint-docs") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(LintDocs { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + /// Tests that the lint examples in the rustc book generate the correct + /// lints and have the expected format. + fn run(self, builder: &Builder<'_>) { + builder.ensure(crate::core::build_steps::doc::RustcBook { + compiler: self.compiler, + target: self.target, + validate: true, + }); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct RustInstaller; + +impl Step for RustInstaller { + type Output = (); + const ONLY_HOSTS: bool = true; + const DEFAULT: bool = true; + + /// Ensure the version placeholder replacement tool builds + fn run(self, builder: &Builder<'_>) { + let bootstrap_host = builder.config.build; + let compiler = builder.compiler(0, bootstrap_host); + let cargo = tool::prepare_tool_cargo( + builder, + compiler, + Mode::ToolBootstrap, + bootstrap_host, + "test", + "src/tools/rust-installer", + SourceType::InTree, + &[], + ); + + let _guard = builder.msg( + Kind::Test, + compiler.stage, + "rust-installer", + bootstrap_host, + bootstrap_host, + ); + run_cargo_test(cargo, &[], &[], "installer", None, compiler, bootstrap_host, builder); + + // We currently don't support running the test.sh script outside linux(?) environments. + // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a + // set of scripts, which will likely allow dropping this if. + if bootstrap_host != "x86_64-unknown-linux-gnu" { + return; + } + + let mut cmd = + std::process::Command::new(builder.src.join("src/tools/rust-installer/test.sh")); + let tmpdir = testdir(builder, compiler.host).join("rust-installer"); + let _ = std::fs::remove_dir_all(&tmpdir); + let _ = std::fs::create_dir_all(&tmpdir); + cmd.current_dir(&tmpdir); + cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target")); + cmd.env("CARGO", &builder.initial_cargo); + cmd.env("RUSTC", &builder.initial_rustc); + cmd.env("TMP_DIR", &tmpdir); + builder.run_delaying_failure(&mut cmd); + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rust-installer") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Self); + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct TestHelpers { + pub target: TargetSelection, +} + +impl Step for TestHelpers { + type Output = (); + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("tests/auxiliary/rust_test_helpers.c") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(TestHelpers { target: run.target }) + } + + /// Compiles the `rust_test_helpers.c` library which we used in various + /// `run-pass` tests for ABI testing. + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run() { + return; + } + // The x86_64-fortanix-unknown-sgx target doesn't have a working C + // toolchain. However, some x86_64 ELF objects can be linked + // without issues. Use this hack to compile the test helpers. + let target = if self.target == "x86_64-fortanix-unknown-sgx" { + TargetSelection::from_user("x86_64-unknown-linux-gnu") + } else { + self.target + }; + let dst = builder.test_helpers_out(target); + let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); + if up_to_date(&src, &dst.join("librust_test_helpers.a")) { + return; + } + + let _guard = builder.msg_unstaged(Kind::Build, "test helpers", target); + t!(fs::create_dir_all(&dst)); + let mut cfg = cc::Build::new(); + + // We may have found various cross-compilers a little differently due to our + // extra configuration, so inform cc of these compilers. Note, though, that + // on MSVC we still need cc's detection of env vars (ugh). + if !target.contains("msvc") { + if let Some(ar) = builder.ar(target) { + cfg.archiver(ar); + } + cfg.compiler(builder.cc(target)); + } + cfg.cargo_metadata(false) + .out_dir(&dst) + .target(&target.triple) + .host(&builder.config.build.triple) + .opt_level(0) + .warnings(false) + .debug(false) + .file(builder.src.join("tests/auxiliary/rust_test_helpers.c")) + .compile("rust_test_helpers"); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CodegenCranelift { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for CodegenCranelift { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_cranelift"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); + + if builder.doc_tests == DocTests::Only { + return; + } + + if !target_supports_cranelift_backend(run.target) { + builder.info("target not supported by rustc_codegen_cranelift. skipping"); + return; + } + + if builder.remote_tested(run.target) { + builder.info("remote testing is not supported by rustc_codegen_cranelift. skipping"); + return; + } + + if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) { + builder.info("cranelift not in rust.codegen-backends. skipping"); + return; + } + + builder.ensure(CodegenCranelift { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + + builder.ensure(compile::Std::new(compiler, target)); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let build_cargo = || { + let mut cargo = builder.cargo( + compiler, + Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works + SourceType::InTree, + target, + "run", + ); + cargo.current_dir(&builder.src.join("compiler/rustc_codegen_cranelift")); + cargo + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc_codegen_cranelift/build_system/Cargo.toml")); + compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + // Avoid incremental cache issues when changing rustc + cargo.env("CARGO_BUILD_INCREMENTAL", "false"); + + cargo + }; + + builder.info(&format!( + "{} cranelift stage{} ({} -> {})", + Kind::Test.description(), + compiler.stage, + &compiler.host, + target + )); + let _time = helpers::timeit(&builder); + + // FIXME handle vendoring for source tarballs before removing the --skip-test below + let download_dir = builder.out.join("cg_clif_download"); + + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. + /* + let mut prepare_cargo = build_cargo(); + prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir); + #[allow(deprecated)] + builder.config.try_run(&mut prepare_cargo.into()).unwrap(); + */ + + let mut cargo = build_cargo(); + cargo + .arg("--") + .arg("test") + .arg("--download-dir") + .arg(&download_dir) + .arg("--out-dir") + .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_clif")) + .arg("--no-unstable-features") + .arg("--use-backend") + .arg("cranelift") + // Avoid having to vendor the standard library dependencies + .arg("--sysroot") + .arg("llvm") + // These tests depend on crates that are not yet vendored + // FIXME remove once vendoring is handled + .arg("--skip-test") + .arg("testsuite.extended_sysroot"); + cargo.args(builder.config.test_args()); + + let mut cmd: Command = cargo.into(); + builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast()); + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub struct CodegenGCC { + compiler: Compiler, + target: TargetSelection, +} + +impl Step for CodegenGCC { + type Output = (); + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.paths(&["compiler/rustc_codegen_gcc"]) + } + + fn make_run(run: RunConfig<'_>) { + let builder = run.builder; + let host = run.build_triple(); + let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); + + if builder.doc_tests == DocTests::Only { + return; + } + + let triple = run.target.triple; + let target_supported = + if triple.contains("linux") { triple.contains("x86_64") } else { false }; + if !target_supported { + builder.info("target not supported by rustc_codegen_gcc. skipping"); + return; + } + + if builder.remote_tested(run.target) { + builder.info("remote testing is not supported by rustc_codegen_gcc. skipping"); + return; + } + + if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("gcc")) { + builder.info("gcc not in rust.codegen-backends. skipping"); + return; + } + + builder.ensure(CodegenGCC { compiler, target: run.target }); + } + + fn run(self, builder: &Builder<'_>) { + let compiler = self.compiler; + let target = self.target; + + builder.ensure(compile::Std::new_with_extra_rust_args( + compiler, + target, + &["-Csymbol-mangling-version=v0", "-Cpanic=abort"], + )); + + // If we're not doing a full bootstrap but we're testing a stage2 + // version of libstd, then what we're actually testing is the libstd + // produced in stage1. Reflect that here by updating the compiler that + // we're working with automatically. + let compiler = builder.compiler_for(compiler.stage, compiler.host, target); + + let build_cargo = || { + let mut cargo = builder.cargo( + compiler, + Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works + SourceType::InTree, + target, + "run", + ); + cargo.current_dir(&builder.src.join("compiler/rustc_codegen_gcc")); + cargo + .arg("--manifest-path") + .arg(builder.src.join("compiler/rustc_codegen_gcc/build_system/Cargo.toml")); + compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); + + // Avoid incremental cache issues when changing rustc + cargo.env("CARGO_BUILD_INCREMENTAL", "false"); + cargo.rustflag("-Cpanic=abort"); + + cargo + }; + + builder.info(&format!( + "{} GCC stage{} ({} -> {})", + Kind::Test.description(), + compiler.stage, + &compiler.host, + target + )); + let _time = helpers::timeit(&builder); + + // FIXME: Uncomment the `prepare` command below once vendoring is implemented. + /* + let mut prepare_cargo = build_cargo(); + prepare_cargo.arg("--").arg("prepare"); + #[allow(deprecated)] + builder.config.try_run(&mut prepare_cargo.into()).unwrap(); + */ + + let mut cargo = build_cargo(); + + cargo + .arg("--") + .arg("test") + .arg("--use-system-gcc") + .arg("--use-backend") + .arg("gcc") + .arg("--out-dir") + .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_gcc")) + .arg("--release") + .arg("--no-default-features") + .arg("--mini-tests") + .arg("--std-tests"); + cargo.args(builder.config.test_args()); + + let mut cmd: Command = cargo.into(); + builder.run_cmd(BootstrapCommand::from(&mut cmd).fail_fast()); + } +} diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs new file mode 100644 index 000000000..d1bc05e51 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -0,0 +1,849 @@ +use std::env; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::core::build_steps::compile; +use crate::core::build_steps::toolstate::ToolState; +use crate::core::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; +use crate::core::config::TargetSelection; +use crate::utils::channel::GitInfo; +use crate::utils::exec::BootstrapCommand; +use crate::utils::helpers::{add_dylib_path, exe, t}; +use crate::Compiler; +use crate::Mode; +use crate::{gha, Kind}; + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub enum SourceType { + InTree, + Submodule, +} + +#[derive(Debug, Clone, Hash, PartialEq, Eq)] +struct ToolBuild { + compiler: Compiler, + target: TargetSelection, + tool: &'static str, + path: &'static str, + mode: Mode, + is_optional_tool: bool, + source_type: SourceType, + extra_features: Vec, + /// Nightly-only features that are allowed (comma-separated list). + allow_features: &'static str, +} + +impl Builder<'_> { + #[track_caller] + fn msg_tool( + &self, + mode: Mode, + tool: &str, + build_stage: u32, + host: &TargetSelection, + target: &TargetSelection, + ) -> Option { + match mode { + // depends on compiler stage, different to host compiler + Mode::ToolRustc => self.msg_sysroot_tool( + Kind::Build, + build_stage, + format_args!("tool {tool}"), + *host, + *target, + ), + // doesn't depend on compiler, same as host compiler + _ => self.msg(Kind::Build, build_stage, format_args!("tool {tool}"), *host, *target), + } + } +} + +impl Step for ToolBuild { + type Output = Option; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + /// Builds a tool in `src/tools` + /// + /// This will build the specified tool with the specified `host` compiler in + /// `stage` into the normal cargo output directory. + fn run(self, builder: &Builder<'_>) -> Option { + let compiler = self.compiler; + let target = self.target; + let mut tool = self.tool; + let path = self.path; + let is_optional_tool = self.is_optional_tool; + + match self.mode { + Mode::ToolRustc => { + builder.ensure(compile::Std::new(compiler, compiler.host)); + builder.ensure(compile::Rustc::new(compiler, target)); + } + Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)), + Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs + _ => panic!("unexpected Mode for tool build"), + } + + let mut cargo = prepare_tool_cargo( + builder, + compiler, + self.mode, + target, + "build", + path, + self.source_type, + &self.extra_features, + ); + if !self.allow_features.is_empty() { + cargo.allow_features(self.allow_features); + } + let _guard = builder.msg_tool( + self.mode, + self.tool, + self.compiler.stage, + &self.compiler.host, + &self.target, + ); + + let mut cargo = Command::from(cargo); + // we check this in `is_optional_tool` in a second + let is_expected = builder.run_cmd(BootstrapCommand::from(&mut cargo).allow_failure()); + + builder.save_toolstate( + tool, + if is_expected { ToolState::TestFail } else { ToolState::BuildFail }, + ); + + if !is_expected { + if !is_optional_tool { + crate::exit!(1); + } else { + None + } + } else { + // HACK(#82501): on Windows, the tools directory gets added to PATH when running tests, and + // compiletest confuses HTML tidy with the in-tree tidy. Name the in-tree tidy something + // different so the problem doesn't come up. + if tool == "tidy" { + tool = "rust-tidy"; + } + let cargo_out = builder.cargo_out(compiler, self.mode, target).join(exe(tool, target)); + let bin = builder.tools_dir(compiler).join(exe(tool, target)); + builder.copy(&cargo_out, &bin); + Some(bin) + } + } +} + +pub fn prepare_tool_cargo( + builder: &Builder<'_>, + compiler: Compiler, + mode: Mode, + target: TargetSelection, + command: &'static str, + path: &str, + source_type: SourceType, + extra_features: &[String], +) -> CargoCommand { + let mut cargo = builder.cargo(compiler, mode, source_type, target, command); + let dir = builder.src.join(path); + cargo.arg("--manifest-path").arg(dir.join("Cargo.toml")); + + let mut features = extra_features.to_vec(); + if builder.build.config.cargo_native_static { + if path.ends_with("cargo") + || path.ends_with("rls") + || path.ends_with("clippy") + || path.ends_with("miri") + || path.ends_with("rustfmt") + { + cargo.env("LIBZ_SYS_STATIC", "1"); + } + if path.ends_with("cargo") { + features.push("all-static".to_string()); + } + } + + // clippy tests need to know about the stage sysroot. Set them consistently while building to + // avoid rebuilding when running tests. + cargo.env("SYSROOT", builder.sysroot(compiler)); + + // if tools are using lzma we want to force the build script to build its + // own copy + cargo.env("LZMA_API_STATIC", "1"); + + // CFG_RELEASE is needed by rustfmt (and possibly other tools) which + // import rustc-ap-rustc_attr which requires this to be set for the + // `#[cfg(version(...))]` attribute. + cargo.env("CFG_RELEASE", builder.rust_release()); + cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel); + cargo.env("CFG_VERSION", builder.rust_version()); + cargo.env("CFG_RELEASE_NUM", &builder.version); + cargo.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); + if let Some(ref ver_date) = builder.rust_info().commit_date() { + cargo.env("CFG_VER_DATE", ver_date); + } + if let Some(ref ver_hash) = builder.rust_info().sha() { + cargo.env("CFG_VER_HASH", ver_hash); + } + + let info = GitInfo::new(builder.config.omit_git_hash, &dir); + if let Some(sha) = info.sha() { + cargo.env("CFG_COMMIT_HASH", sha); + } + if let Some(sha_short) = info.sha_short() { + cargo.env("CFG_SHORT_COMMIT_HASH", sha_short); + } + if let Some(date) = info.commit_date() { + cargo.env("CFG_COMMIT_DATE", date); + } + if !features.is_empty() { + cargo.arg("--features").arg(&features.join(", ")); + } + cargo +} + +macro_rules! bootstrap_tool { + ($( + $name:ident, $path:expr, $tool_name:expr + $(,is_external_tool = $external:expr)* + $(,is_unstable_tool = $unstable:expr)* + $(,allow_features = $allow_features:expr)? + ; + )+) => { + #[derive(Copy, PartialEq, Eq, Clone)] + pub enum Tool { + $( + $name, + )+ + } + + impl<'a> Builder<'a> { + pub fn tool_exe(&self, tool: Tool) -> PathBuf { + match tool { + $(Tool::$name => + self.ensure($name { + compiler: self.compiler(0, self.config.build), + target: self.config.build, + }), + )+ + } + } + } + + $( + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + } + + impl Step for $name { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path($path) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + // snapshot compiler + compiler: run.builder.compiler(0, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: $tool_name, + mode: if false $(|| $unstable)* { + // use in-tree libraries for unstable features + Mode::ToolStd + } else { + Mode::ToolBootstrap + }, + path: $path, + is_optional_tool: false, + source_type: if false $(|| $external)* { + SourceType::Submodule + } else { + SourceType::InTree + }, + extra_features: vec![], + allow_features: concat!($($allow_features)*), + }).expect("expected to build -- essential tool") + } + } + )+ + } +} + +bootstrap_tool!( + Rustbook, "src/tools/rustbook", "rustbook"; + UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen"; + Tidy, "src/tools/tidy", "tidy"; + Linkchecker, "src/tools/linkchecker", "linkchecker"; + CargoTest, "src/tools/cargotest", "cargotest"; + Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true, allow_features = "test"; + BuildManifest, "src/tools/build-manifest", "build-manifest"; + RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; + RustInstaller, "src/tools/rust-installer", "rust-installer"; + RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; + ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; + LintDocs, "src/tools/lint-docs", "lint-docs"; + JsonDocCk, "src/tools/jsondocck", "jsondocck"; + JsonDocLint, "src/tools/jsondoclint", "jsondoclint"; + HtmlChecker, "src/tools/html-checker", "html-checker"; + BumpStage0, "src/tools/bump-stage0", "bump-stage0"; + ReplaceVersionPlaceholder, "src/tools/replace-version-placeholder", "replace-version-placeholder"; + CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata"; + GenerateCopyright, "src/tools/generate-copyright", "generate-copyright"; + SuggestTests, "src/tools/suggest-tests", "suggest-tests"; + GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys"; + RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; + OptimizedDist, "src/tools/opt-dist", "opt-dist"; + CoverageDump, "src/tools/coverage-dump", "coverage-dump"; +); + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] +pub struct ErrorIndex { + pub compiler: Compiler, +} + +impl ErrorIndex { + pub fn command(builder: &Builder<'_>) -> Command { + // Error-index-generator links with the rustdoc library, so we need to add `rustc_lib_paths` + // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. + let host = builder.config.build; + let compiler = builder.compiler_for(builder.top_stage, host, host); + let mut cmd = Command::new(builder.ensure(ErrorIndex { compiler })); + let mut dylib_paths = builder.rustc_lib_paths(compiler); + dylib_paths.push(PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))); + add_dylib_path(dylib_paths, &mut cmd); + cmd + } +} + +impl Step for ErrorIndex { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/error_index_generator") + } + + fn make_run(run: RunConfig<'_>) { + // Compile the error-index in the same stage as rustdoc to avoid + // recompiling rustdoc twice if we can. + // + // NOTE: This `make_run` isn't used in normal situations, only if you + // manually build the tool with `x.py build + // src/tools/error-index-generator` which almost nobody does. + // Normally, `x.py test` or `x.py doc` will use the + // `ErrorIndex::command` function instead. + let compiler = + run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build); + run.builder.ensure(ErrorIndex { compiler }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.compiler.host, + tool: "error_index_generator", + mode: Mode::ToolRustc, + path: "src/tools/error_index_generator", + is_optional_tool: false, + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) + .expect("expected to build -- essential tool") + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RemoteTestServer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RemoteTestServer { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/remote-test-server") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RemoteTestServer { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "remote-test-server", + mode: Mode::ToolStd, + path: "src/tools/remote-test-server", + is_optional_tool: false, + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) + .expect("expected to build -- essential tool") + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] +pub struct Rustdoc { + /// This should only ever be 0 or 2. + /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here. + pub compiler: Compiler, +} + +impl Step for Rustdoc { + type Output = PathBuf; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/rustdoc").path("src/librustdoc") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Rustdoc { + // NOTE: this is somewhat unique in that we actually want a *target* + // compiler here, because rustdoc *is* a compiler. We won't be using + // this as the compiler to build with, but rather this is "what + // compiler are we producing"? + compiler: run.builder.compiler(run.builder.top_stage, run.target), + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let target_compiler = self.compiler; + if target_compiler.stage == 0 { + if !target_compiler.is_snapshot(builder) { + panic!("rustdoc in stage 0 must be snapshot rustdoc"); + } + return builder.initial_rustc.with_file_name(exe("rustdoc", target_compiler.host)); + } + let target = target_compiler.host; + // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise + // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage + // compilers, which isn't what we want. Rustdoc should be linked in the same way as the + // rustc compiler it's paired with, so it must be built with the previous stage compiler. + let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); + + // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually + // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build + // it. + builder.ensure(compile::Std::new(build_compiler, target_compiler.host)); + builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host)); + // NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0 + // compiler, since you do just as much work. + if !builder.config.dry_run() && builder.download_rustc() && build_compiler.stage == 0 { + println!( + "WARNING: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" + ); + } + + // The presence of `target_compiler` ensures that the necessary libraries (codegen backends, + // compiler libraries, ...) are built. Rustdoc does not require the presence of any + // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since + // they'll be linked to those libraries). As such, don't explicitly `ensure` any additional + // libraries here. The intuition here is that If we've built a compiler, we should be able + // to build rustdoc. + // + let mut features = Vec::new(); + if builder.config.jemalloc { + features.push("jemalloc".to_string()); + } + + let mut cargo = prepare_tool_cargo( + builder, + build_compiler, + Mode::ToolRustc, + target, + "build", + "src/tools/rustdoc", + SourceType::InTree, + features.as_slice(), + ); + + if builder.config.rustc_parallel { + cargo.rustflag("--cfg=parallel_compiler"); + } + + let _guard = builder.msg_tool( + Mode::ToolRustc, + "rustdoc", + build_compiler.stage, + &self.compiler.host, + &target, + ); + builder.run(&mut cargo.into()); + + // Cargo adds a number of paths to the dylib search path on windows, which results in + // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" + // rustdoc a different name. + let tool_rustdoc = builder + .cargo_out(build_compiler, Mode::ToolRustc, target) + .join(exe("rustdoc_tool_binary", target_compiler.host)); + + // don't create a stage0-sysroot/bin directory. + if target_compiler.stage > 0 { + let sysroot = builder.sysroot(target_compiler); + let bindir = sysroot.join("bin"); + t!(fs::create_dir_all(&bindir)); + let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host)); + let _ = fs::remove_file(&bin_rustdoc); + builder.copy(&tool_rustdoc, &bin_rustdoc); + bin_rustdoc + } else { + tool_rustdoc + } + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct Cargo { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for Cargo { + type Output = PathBuf; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/cargo").default_condition( + builder.config.extended + && builder.config.tools.as_ref().map_or( + true, + // If `tools` is set, search list for this tool. + |tools| tools.iter().any(|tool| tool == "cargo"), + ), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(Cargo { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let cargo_bin_path = builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "cargo", + mode: Mode::ToolRustc, + path: "src/tools/cargo", + is_optional_tool: false, + source_type: SourceType::Submodule, + extra_features: Vec::new(), + allow_features: "", + }) + .expect("expected to build -- essential tool"); + cargo_bin_path + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct LldWrapper { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for LldWrapper { + type Output = PathBuf; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let src_exe = builder + .ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "lld-wrapper", + mode: Mode::ToolStd, + path: "src/tools/lld-wrapper", + is_optional_tool: false, + source_type: SourceType::InTree, + extra_features: Vec::new(), + allow_features: "", + }) + .expect("expected to build -- essential tool"); + + src_exe + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzer { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl RustAnalyzer { + pub const ALLOW_FEATURES: &'static str = + "proc_macro_internals,proc_macro_diagnostic,proc_macro_span,proc_macro_span_shrink"; +} + +impl Step for RustAnalyzer { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path("src/tools/rust-analyzer").default_condition( + builder.config.extended + && builder + .config + .tools + .as_ref() + .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzer { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer", + mode: Mode::ToolStd, + path: "src/tools/rust-analyzer", + extra_features: vec!["rust-analyzer/in-rust-tree".to_owned()], + is_optional_tool: false, + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + }) + } +} + +#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] +pub struct RustAnalyzerProcMacroSrv { + pub compiler: Compiler, + pub target: TargetSelection, +} + +impl Step for RustAnalyzerProcMacroSrv { + type Output = Option; + const DEFAULT: bool = true; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + // Allow building `rust-analyzer-proc-macro-srv` both as part of the `rust-analyzer` and as a stand-alone tool. + run.path("src/tools/rust-analyzer") + .path("src/tools/rust-analyzer/crates/proc-macro-srv-cli") + .default_condition(builder.config.tools.as_ref().map_or(true, |tools| { + tools + .iter() + .any(|tool| tool == "rust-analyzer" || tool == "rust-analyzer-proc-macro-srv") + })) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(RustAnalyzerProcMacroSrv { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + }); + } + + fn run(self, builder: &Builder<'_>) -> Option { + let path = builder.ensure(ToolBuild { + compiler: self.compiler, + target: self.target, + tool: "rust-analyzer-proc-macro-srv", + mode: Mode::ToolStd, + path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", + extra_features: vec!["sysroot-abi".to_owned()], + is_optional_tool: false, + source_type: SourceType::InTree, + allow_features: RustAnalyzer::ALLOW_FEATURES, + })?; + + // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` + // so that r-a can use it. + let libexec_path = builder.sysroot(self.compiler).join("libexec"); + t!(fs::create_dir_all(&libexec_path)); + builder.copy(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); + + Some(path) + } +} + +macro_rules! tool_extended { + (($sel:ident, $builder:ident), + $($name:ident, + $path:expr, + $tool_name:expr, + stable = $stable:expr + $(,tool_std = $tool_std:literal)? + $(,allow_features = $allow_features:expr)? + $(,add_bins_to_sysroot = $add_bins_to_sysroot:expr)? + ;)+) => { + $( + #[derive(Debug, Clone, Hash, PartialEq, Eq)] + pub struct $name { + pub compiler: Compiler, + pub target: TargetSelection, + pub extra_features: Vec, + } + + impl Step for $name { + type Output = Option; + const DEFAULT: bool = true; // Overwritten below + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + let builder = run.builder; + run.path($path).default_condition( + builder.config.extended + && builder.config.tools.as_ref().map_or( + // By default, on nightly/dev enable all tools, else only + // build stable tools. + $stable || builder.build.unstable_features(), + // If `tools` is set, search list for this tool. + |tools| { + tools.iter().any(|tool| match tool.as_ref() { + "clippy" => $tool_name == "clippy-driver", + x => $tool_name == x, + }) + }), + ) + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure($name { + compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), + target: run.target, + extra_features: Vec::new(), + }); + } + + #[allow(unused_mut)] + fn run(mut $sel, $builder: &Builder<'_>) -> Option { + let tool = $builder.ensure(ToolBuild { + compiler: $sel.compiler, + target: $sel.target, + tool: $tool_name, + mode: if false $(|| $tool_std)? { Mode::ToolStd } else { Mode::ToolRustc }, + path: $path, + extra_features: $sel.extra_features, + is_optional_tool: true, + source_type: SourceType::InTree, + allow_features: concat!($($allow_features)*), + })?; + + if (false $(|| !$add_bins_to_sysroot.is_empty())?) && $sel.compiler.stage > 0 { + let bindir = $builder.sysroot($sel.compiler).join("bin"); + t!(fs::create_dir_all(&bindir)); + + #[allow(unused_variables)] + let tools_out = $builder + .cargo_out($sel.compiler, Mode::ToolRustc, $sel.target); + + $(for add_bin in $add_bins_to_sysroot { + let bin_source = tools_out.join(exe(add_bin, $sel.target)); + let bin_destination = bindir.join(exe(add_bin, $sel.compiler.host)); + $builder.copy(&bin_source, &bin_destination); + })? + + let tool = bindir.join(exe($tool_name, $sel.compiler.host)); + Some(tool) + } else { + Some(tool) + } + } + } + )+ + } +} + +// NOTE: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs` +// to make `./x.py build ` work. +// NOTE: Most submodule updates for tools are handled by bootstrap.py, since they're needed just to +// invoke Cargo to build bootstrap. See the comment there for more details. +tool_extended!((self, builder), + Cargofmt, "src/tools/rustfmt", "cargo-fmt", stable=true; + CargoClippy, "src/tools/clippy", "cargo-clippy", stable=true; + Clippy, "src/tools/clippy", "clippy-driver", stable=true, add_bins_to_sysroot = ["clippy-driver", "cargo-clippy"]; + Miri, "src/tools/miri", "miri", stable=false, add_bins_to_sysroot = ["miri"]; + CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri", stable=true, add_bins_to_sysroot = ["cargo-miri"]; + // FIXME: tool_std is not quite right, we shouldn't allow nightly features. + // But `builder.cargo` doesn't know how to handle ToolBootstrap in stages other than 0, + // and this is close enough for now. + Rls, "src/tools/rls", "rls", stable=true, tool_std=true; + RustDemangler, "src/tools/rust-demangler", "rust-demangler", stable=false, tool_std=true; + Rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, add_bins_to_sysroot = ["rustfmt", "cargo-fmt"]; +); + +impl<'a> Builder<'a> { + /// Gets a `Command` which is ready to run `tool` in `stage` built for + /// `host`. + pub fn tool_cmd(&self, tool: Tool) -> Command { + let mut cmd = Command::new(self.tool_exe(tool)); + let compiler = self.compiler(0, self.config.build); + let host = &compiler.host; + // Prepares the `cmd` provided to be able to run the `compiler` provided. + // + // Notably this munges the dynamic library lookup path to point to the + // right location to run `compiler`. + let mut lib_paths: Vec = vec![ + self.build.rustc_snapshot_libdir(), + self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps"), + ]; + + // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make + // mode) and that C compiler may need some extra PATH modification. Do + // so here. + if compiler.host.contains("msvc") { + let curpaths = env::var_os("PATH").unwrap_or_default(); + let curpaths = env::split_paths(&curpaths).collect::>(); + for &(ref k, ref v) in self.cc.borrow()[&compiler.host].env() { + if k != "PATH" { + continue; + } + for path in env::split_paths(v) { + if !curpaths.contains(&path) { + lib_paths.push(path); + } + } + } + } + + add_dylib_path(lib_paths, &mut cmd); + + // Provide a RUSTC for this command to use. + cmd.env("RUSTC", &self.initial_rustc); + + cmd + } +} diff --git a/src/bootstrap/src/core/build_steps/toolstate.rs b/src/bootstrap/src/core/build_steps/toolstate.rs new file mode 100644 index 000000000..a451f92b6 --- /dev/null +++ b/src/bootstrap/src/core/build_steps/toolstate.rs @@ -0,0 +1,478 @@ +use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; +use crate::utils::helpers::t; +use serde_derive::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::env; +use std::fmt; +use std::fs; +use std::io::{Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time; + +// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then. +const BETA_WEEK_START: u64 = 35; + +#[cfg(target_os = "linux")] +const OS: Option<&str> = Some("linux"); + +#[cfg(windows)] +const OS: Option<&str> = Some("windows"); + +#[cfg(all(not(target_os = "linux"), not(windows)))] +const OS: Option<&str> = None; + +type ToolstateData = HashMap, ToolState>; + +#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, PartialOrd)] +#[serde(rename_all = "kebab-case")] +/// Whether a tool can be compiled, tested or neither +pub enum ToolState { + /// The tool compiles successfully, but the test suite fails + TestFail = 1, + /// The tool compiles successfully and its test suite passes + TestPass = 2, + /// The tool can't even be compiled + BuildFail = 0, +} + +impl fmt::Display for ToolState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "{}", + match self { + ToolState::TestFail => "test-fail", + ToolState::TestPass => "test-pass", + ToolState::BuildFail => "build-fail", + } + ) + } +} + +/// Number of days after the last promotion of beta. +/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. +/// The Wednesday after this has value 0. +/// We track this value to prevent regressing tools in the last week of the 6-week cycle. +fn days_since_beta_promotion() -> u64 { + let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed()); + (since_epoch.as_secs() / 86400 - 20) % 42 +} + +// These tools must test-pass on the beta/stable channels. +// +// On the nightly channel, their build step must be attempted, but they may not +// be able to build successfully. +static STABLE_TOOLS: &[(&str, &str)] = &[ + ("book", "src/doc/book"), + ("nomicon", "src/doc/nomicon"), + ("reference", "src/doc/reference"), + ("rust-by-example", "src/doc/rust-by-example"), + ("edition-guide", "src/doc/edition-guide"), +]; + +// These tools are permitted to not build on the beta/stable channels. +// +// We do require that we checked whether they build or not on the tools builder, +// though, as otherwise we will be unable to file an issue if they start +// failing. +static NIGHTLY_TOOLS: &[(&str, &str)] = &[ + ("embedded-book", "src/doc/embedded-book"), + // ("rustc-dev-guide", "src/doc/rustc-dev-guide"), +]; + +fn print_error(tool: &str, submodule: &str) { + eprintln!(); + eprintln!("We detected that this PR updated '{tool}', but its tests failed."); + eprintln!(); + eprintln!("If you do intend to update '{tool}', please check the error messages above and"); + eprintln!("commit another update."); + eprintln!(); + eprintln!("If you do NOT intend to update '{tool}', please ensure you did not accidentally"); + eprintln!("change the submodule at '{submodule}'. You may ask your reviewer for the"); + eprintln!("proper steps."); + crate::exit!(3); +} + +fn check_changed_files(toolstates: &HashMap, ToolState>) { + // Changed files + let output = std::process::Command::new("git") + .arg("diff") + .arg("--name-status") + .arg("HEAD") + .arg("HEAD^") + .output(); + let output = match output { + Ok(o) => o, + Err(e) => { + eprintln!("Failed to get changed files: {e:?}"); + crate::exit!(1); + } + }; + + let output = t!(String::from_utf8(output.stdout)); + + for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + let changed = output.lines().any(|l| l.starts_with('M') && l.ends_with(submodule)); + eprintln!("Verifying status of {tool}..."); + if !changed { + continue; + } + + eprintln!("This PR updated '{submodule}', verifying if status is 'test-pass'..."); + if toolstates[*tool] != ToolState::TestPass { + print_error(tool, submodule); + } + } +} + +#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] +pub struct ToolStateCheck; + +impl Step for ToolStateCheck { + type Output = (); + + /// Checks tool state status. + /// + /// This is intended to be used in the `checktools.sh` script. To use + /// this, set `save-toolstates` in `config.toml` so that tool status will + /// be saved to a JSON file. Then, run `x.py test --no-fail-fast` for all + /// of the tools to populate the JSON file. After that is done, this + /// command can be run to check for any status failures, and exits with an + /// error if there are any. + /// + /// This also handles publishing the results to the `history` directory of + /// the toolstate repo + /// if the env var `TOOLSTATE_PUBLISH` is set. Note that there is a + /// *separate* step of updating the `latest.json` file and creating GitHub + /// issues and comments in `src/ci/publish_toolstate.sh`, which is only + /// performed on master. (The shell/python code is intended to be migrated + /// here eventually.) + /// + /// The rules for failure are: + /// * If the PR modifies a tool, the status must be test-pass. + /// NOTE: There is intent to change this, see + /// . + /// * All "stable" tools must be test-pass on the stable or beta branches. + /// * During beta promotion week, a PR is not allowed to "regress" a + /// stable tool. That is, the status is not allowed to get worse + /// (test-pass to test-fail or build-fail). + fn run(self, builder: &Builder<'_>) { + if builder.config.dry_run() { + return; + } + + let days_since_beta_promotion = days_since_beta_promotion(); + let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START; + let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable"); + let toolstates = builder.toolstates(); + + let mut did_error = false; + + for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { + if !toolstates.contains_key(*tool) { + did_error = true; + eprintln!("ERROR: Tool `{tool}` was not recorded in tool state."); + } + } + + if did_error { + crate::exit!(1); + } + + check_changed_files(&toolstates); + checkout_toolstate_repo(); + let old_toolstate = read_old_toolstate(); + + for (tool, _) in STABLE_TOOLS.iter() { + let state = toolstates[*tool]; + + if state != ToolState::TestPass { + if !is_nightly { + did_error = true; + eprintln!("ERROR: Tool `{tool}` should be test-pass but is {state}"); + } else if in_beta_week { + let old_state = old_toolstate + .iter() + .find(|ts| ts.tool == *tool) + .expect("latest.json missing tool") + .state(); + if state < old_state { + did_error = true; + eprintln!( + "ERROR: Tool `{tool}` has regressed from {old_state} to {state} during beta week." + ); + } else { + // This warning only appears in the logs, which most + // people won't read. It's mostly here for testing and + // debugging. + eprintln!( + "WARNING: Tool `{tool}` is not test-pass (is `{state}`), \ + this should be fixed before beta is branched." + ); + } + } + // `publish_toolstate.py` is responsible for updating + // `latest.json` and creating comments/issues warning people + // if there is a regression. That all happens in a separate CI + // job on the master branch once the PR has passed all tests + // on the `auto` branch. + } + } + + if did_error { + crate::exit!(1); + } + + if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { + commit_toolstate_change(&toolstates); + } + } + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.alias("check-tools") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(ToolStateCheck); + } +} + +impl Builder<'_> { + fn toolstates(&self) -> HashMap, ToolState> { + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = + t!(fs::OpenOptions::new().create(true).write(true).read(true).open(path)); + + serde_json::from_reader(&mut file).unwrap_or_default() + } else { + Default::default() + } + } + + /// Updates the actual toolstate of a tool. + /// + /// The toolstates are saved to the file specified by the key + /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be + /// done. The file is updated immediately after this function completes. + pub fn save_toolstate(&self, tool: &str, state: ToolState) { + use std::io::Write; + + // If we're in a dry run setting we don't want to save toolstates as + // that means if we e.g. panic down the line it'll look like we tested + // everything (but we actually haven't). + if self.config.dry_run() { + return; + } + // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking + // in all the places we could save toolstate and just do so here. + if tool == "clippy-driver" || tool == "rustfmt" { + return; + } + if let Some(ref path) = self.config.save_toolstates { + if let Some(parent) = path.parent() { + // Ensure the parent directory always exists + t!(std::fs::create_dir_all(parent)); + } + let mut file = + t!(fs::OpenOptions::new().create(true).read(true).write(true).open(path)); + + let mut current_toolstates: HashMap, ToolState> = + serde_json::from_reader(&mut file).unwrap_or_default(); + current_toolstates.insert(tool.into(), state); + t!(file.seek(SeekFrom::Start(0))); + t!(file.set_len(0)); + t!(serde_json::to_writer(&file, ¤t_toolstates)); + t!(writeln!(file)); // make sure this ends in a newline + } + } +} + +fn toolstate_repo() -> String { + env::var("TOOLSTATE_REPO") + .unwrap_or_else(|_| "https://github.com/rust-lang-nursery/rust-toolstate.git".to_string()) +} + +/// Directory where the toolstate repo is checked out. +const TOOLSTATE_DIR: &str = "rust-toolstate"; + +/// Checks out the toolstate repo into `TOOLSTATE_DIR`. +fn checkout_toolstate_repo() { + if let Ok(token) = env::var("TOOLSTATE_REPO_ACCESS_TOKEN") { + prepare_toolstate_config(&token); + } + if Path::new(TOOLSTATE_DIR).exists() { + eprintln!("Cleaning old toolstate directory..."); + t!(fs::remove_dir_all(TOOLSTATE_DIR)); + } + + let status = Command::new("git") + .arg("clone") + .arg("--depth=1") + .arg(toolstate_repo()) + .arg(TOOLSTATE_DIR) + .status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git clone unsuccessful (status: {status:?})"); + } +} + +/// Sets up config and authentication for modifying the toolstate repo. +fn prepare_toolstate_config(token: &str) { + fn git_config(key: &str, value: &str) { + let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status(); + let success = match status { + Ok(s) => s.success(), + Err(_) => false, + }; + if !success { + panic!("git config key={key} value={value} failed (status: {status:?})"); + } + } + + // If changing anything here, then please check that `src/ci/publish_toolstate.sh` is up to date + // as well. + git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); + git_config("user.name", "Rust Toolstate Update"); + git_config("credential.helper", "store"); + + let credential = format!("https://{token}:x-oauth-basic@github.com\n",); + let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials"); + t!(fs::write(&git_credential_path, credential)); +} + +/// Reads the latest toolstate from the toolstate repo. +fn read_old_toolstate() -> Vec { + let latest_path = Path::new(TOOLSTATE_DIR).join("_data").join("latest.json"); + let old_toolstate = t!(fs::read(latest_path)); + t!(serde_json::from_slice(&old_toolstate)) +} + +/// This function `commit_toolstate_change` provides functionality for pushing a change +/// to the `rust-toolstate` repository. +/// +/// The function relies on a GitHub bot user, which should have a Personal access +/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for +/// some reason you need to change the token, please update the Azure Pipelines +/// variable group. +/// +/// 1. Generate a new Personal access token: +/// +/// * Login to the bot account, and go to Settings -> Developer settings -> +/// Personal access tokens +/// * Click "Generate new token" +/// * Enable the "public_repo" permission, then click "Generate token" +/// * Copy the generated token (should be a 40-digit hexadecimal number). +/// Save it somewhere secure, as the token would be gone once you leave +/// the page. +/// +/// 2. Update the variable group in Azure Pipelines +/// +/// * Ping a member of the infrastructure team to do this. +/// +/// 4. Replace the email address below if the bot account identity is changed +/// +/// * See +/// if a private email by GitHub is wanted. +fn commit_toolstate_change(current_toolstate: &ToolstateData) { + let message = format!("({} CI update)", OS.expect("linux/windows only")); + let mut success = false; + for _ in 1..=5 { + // Upload the test results (the new commit-to-toolstate mapping) to the toolstate repo. + // This does *not* change the "current toolstate"; that only happens post-landing + // via `src/ci/docker/publish_toolstate.sh`. + publish_test_results(¤t_toolstate); + + // `git commit` failing means nothing to commit. + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("commit") + .arg("-a") + .arg("-m") + .arg(&message) + .status()); + if !status.success() { + success = true; + break; + } + + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("push") + .arg("origin") + .arg("master") + .status()); + // If we successfully push, exit. + if status.success() { + success = true; + break; + } + eprintln!("Sleeping for 3 seconds before retrying push"); + std::thread::sleep(std::time::Duration::from_secs(3)); + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("fetch") + .arg("origin") + .arg("master") + .status()); + assert!(status.success()); + let status = t!(Command::new("git") + .current_dir(TOOLSTATE_DIR) + .arg("reset") + .arg("--hard") + .arg("origin/master") + .status()); + assert!(status.success()); + } + + if !success { + panic!("Failed to update toolstate repository with new data"); + } +} + +/// Updates the "history" files with the latest results. +/// +/// These results will later be promoted to `latest.json` by the +/// `publish_toolstate.py` script if the PR passes all tests and is merged to +/// master. +fn publish_test_results(current_toolstate: &ToolstateData) { + let commit = t!(std::process::Command::new("git").arg("rev-parse").arg("HEAD").output()); + let commit = t!(String::from_utf8(commit.stdout)); + + let toolstate_serialized = t!(serde_json::to_string(¤t_toolstate)); + + let history_path = Path::new(TOOLSTATE_DIR) + .join("history") + .join(format!("{}.tsv", OS.expect("linux/windows only"))); + let mut file = t!(fs::read_to_string(&history_path)); + let end_of_first_line = file.find('\n').unwrap(); + file.insert_str(end_of_first_line, &format!("\n{}\t{}", commit.trim(), toolstate_serialized)); + t!(fs::write(&history_path, file)); +} + +#[derive(Debug, Deserialize)] +struct RepoState { + tool: String, + windows: ToolState, + linux: ToolState, +} + +impl RepoState { + fn state(&self) -> ToolState { + if cfg!(target_os = "linux") { + self.linux + } else if cfg!(windows) { + self.windows + } else { + unimplemented!() + } + } +} diff --git a/src/bootstrap/src/core/builder.rs b/src/bootstrap/src/core/builder.rs new file mode 100644 index 000000000..cd276674d --- /dev/null +++ b/src/bootstrap/src/core/builder.rs @@ -0,0 +1,2392 @@ +use std::any::{type_name, Any}; +use std::cell::{Cell, RefCell}; +use std::collections::BTreeSet; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fmt::{Debug, Write}; +use std::fs::{self, File}; +use std::hash::Hash; +use std::io::{BufRead, BufReader}; +use std::ops::Deref; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::time::{Duration, Instant}; + +use crate::core::build_steps::llvm; +use crate::core::build_steps::tool::{self, SourceType}; +use crate::core::build_steps::{check, clean, compile, dist, doc, install, run, setup, test}; +use crate::core::config::flags::{Color, Subcommand}; +use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection}; +use crate::utils::cache::{Cache, Interned, INTERNER}; +use crate::utils::helpers::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; +use crate::Crate; +use crate::EXTRA_CHECK_CFGS; +use crate::{Build, CLang, DocTests, GitRepo, Mode}; + +pub use crate::Compiler; +// FIXME: +// - use std::lazy for `Lazy` +// - use std::cell for `OnceCell` +// Once they get stabilized and reach beta. +use clap::ValueEnum; +use once_cell::sync::{Lazy, OnceCell}; + +#[cfg(test)] +#[path = "../tests/builder.rs"] +mod tests; + +pub struct Builder<'a> { + pub build: &'a Build, + pub top_stage: u32, + pub kind: Kind, + cache: Cache, + stack: RefCell>>, + time_spent_on_dependencies: Cell, + pub paths: Vec, +} + +impl<'a> Deref for Builder<'a> { + type Target = Build; + + fn deref(&self) -> &Self::Target { + self.build + } +} + +pub trait Step: 'static + Clone + Debug + PartialEq + Eq + Hash { + /// `PathBuf` when directories are created or to return a `Compiler` once + /// it's been assembled. + type Output: Clone; + + /// Whether this step is run by default as part of its respective phase. + /// `true` here can still be overwritten by `should_run` calling `default_condition`. + const DEFAULT: bool = false; + + /// If true, then this rule should be skipped if --target was specified, but --host was not + const ONLY_HOSTS: bool = false; + + /// Primary function to execute this rule. Can call `builder.ensure()` + /// with other steps to run those. + fn run(self, builder: &Builder<'_>) -> Self::Output; + + /// When bootstrap is passed a set of paths, this controls whether this rule + /// will execute. However, it does not get called in a "default" context + /// when we are not passed any paths; in that case, `make_run` is called + /// directly. + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_>; + + /// Builds up a "root" rule, either as a default rule or from a path passed + /// to us. + /// + /// When path is `None`, we are executing in a context where no paths were + /// passed. When `./x.py build` is run, for example, this rule could get + /// called if it is in the correct list below with a path of `None`. + fn make_run(_run: RunConfig<'_>) { + // It is reasonable to not have an implementation of make_run for rules + // who do not want to get called from the root context. This means that + // they are likely dependencies (e.g., sysroot creation) or similar, and + // as such calling them from ./x.py isn't logical. + unimplemented!() + } +} + +pub struct RunConfig<'a> { + pub builder: &'a Builder<'a>, + pub target: TargetSelection, + pub paths: Vec, +} + +impl RunConfig<'_> { + pub fn build_triple(&self) -> TargetSelection { + self.builder.build.build + } + + /// Return a list of crate names selected by `run.paths`. + #[track_caller] + pub fn cargo_crates_in_set(&self) -> Interned> { + let mut crates = Vec::new(); + for krate in &self.paths { + let path = krate.assert_single_path(); + let Some(crate_name) = self.builder.crate_paths.get(&path.path) else { + panic!("missing crate for path {}", path.path.display()) + }; + crates.push(crate_name.to_string()); + } + INTERNER.intern_list(crates) + } + + /// Given an `alias` selected by the `Step` and the paths passed on the command line, + /// return a list of the crates that should be built. + /// + /// Normally, people will pass *just* `library` if they pass it. + /// But it's possible (although strange) to pass something like `library std core`. + /// Build all crates anyway, as if they hadn't passed the other args. + pub fn make_run_crates(&self, alias: Alias) -> Interned> { + let has_alias = + self.paths.iter().any(|set| set.assert_single_path().path.ends_with(alias.as_str())); + if !has_alias { + return self.cargo_crates_in_set(); + } + + let crates = match alias { + Alias::Library => self.builder.in_tree_crates("sysroot", Some(self.target)), + Alias::Compiler => self.builder.in_tree_crates("rustc-main", Some(self.target)), + }; + + let crate_names = crates.into_iter().map(|krate| krate.name.to_string()).collect(); + INTERNER.intern_list(crate_names) + } +} + +#[derive(Debug, Copy, Clone)] +pub enum Alias { + Library, + Compiler, +} + +impl Alias { + fn as_str(self) -> &'static str { + match self { + Alias::Library => "library", + Alias::Compiler => "compiler", + } + } +} + +/// A description of the crates in this set, suitable for passing to `builder.info`. +/// +/// `crates` should be generated by [`RunConfig::cargo_crates_in_set`]. +pub fn crate_description(crates: &[impl AsRef]) -> String { + if crates.is_empty() { + return "".into(); + } + + let mut descr = String::from(" {"); + descr.push_str(crates[0].as_ref()); + for krate in &crates[1..] { + descr.push_str(", "); + descr.push_str(krate.as_ref()); + } + descr.push('}'); + descr +} + +struct StepDescription { + default: bool, + only_hosts: bool, + should_run: fn(ShouldRun<'_>) -> ShouldRun<'_>, + make_run: fn(RunConfig<'_>), + name: &'static str, + kind: Kind, +} + +#[derive(Clone, PartialOrd, Ord, PartialEq, Eq)] +pub struct TaskPath { + pub path: PathBuf, + pub kind: Option, +} + +impl Debug for TaskPath { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(kind) = &self.kind { + write!(f, "{}::", kind.as_str())?; + } + write!(f, "{}", self.path.display()) + } +} + +/// Collection of paths used to match a task rule. +#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq)] +pub enum PathSet { + /// A collection of individual paths or aliases. + /// + /// These are generally matched as a path suffix. For example, a + /// command-line value of `std` will match if `library/std` is in the + /// set. + /// + /// NOTE: the paths within a set should always be aliases of one another. + /// For example, `src/librustdoc` and `src/tools/rustdoc` should be in the same set, + /// but `library/core` and `library/std` generally should not, unless there's no way (for that Step) + /// to build them separately. + Set(BTreeSet), + /// A "suite" of paths. + /// + /// These can match as a path suffix (like `Set`), or as a prefix. For + /// example, a command-line value of `tests/ui/abi/variadic-ffi.rs` + /// will match `tests/ui`. A command-line value of `ui` would also + /// match `tests/ui`. + Suite(TaskPath), +} + +impl PathSet { + fn empty() -> PathSet { + PathSet::Set(BTreeSet::new()) + } + + fn one>(path: P, kind: Kind) -> PathSet { + let mut set = BTreeSet::new(); + set.insert(TaskPath { path: path.into(), kind: Some(kind) }); + PathSet::Set(set) + } + + fn has(&self, needle: &Path, module: Kind) -> bool { + match self { + PathSet::Set(set) => set.iter().any(|p| Self::check(p, needle, module)), + PathSet::Suite(suite) => Self::check(suite, needle, module), + } + } + + // internal use only + fn check(p: &TaskPath, needle: &Path, module: Kind) -> bool { + if let Some(p_kind) = &p.kind { + p.path.ends_with(needle) && *p_kind == module + } else { + p.path.ends_with(needle) + } + } + + /// Return all `TaskPath`s in `Self` that contain any of the `needles`, removing the + /// matched needles. + /// + /// This is used for `StepDescription::krate`, which passes all matching crates at once to + /// `Step::make_run`, rather than calling it many times with a single crate. + /// See `tests.rs` for examples. + fn intersection_removing_matches(&self, needles: &mut Vec<&Path>, module: Kind) -> PathSet { + let mut check = |p| { + for (i, n) in needles.iter().enumerate() { + let matched = Self::check(p, n, module); + if matched { + needles.remove(i); + return true; + } + } + false + }; + match self { + PathSet::Set(set) => PathSet::Set(set.iter().filter(|&p| check(p)).cloned().collect()), + PathSet::Suite(suite) => { + if check(suite) { + self.clone() + } else { + PathSet::empty() + } + } + } + } + + /// A convenience wrapper for Steps which know they have no aliases and all their sets contain only a single path. + /// + /// This can be used with [`ShouldRun::crate_or_deps`], [`ShouldRun::path`], or [`ShouldRun::alias`]. + #[track_caller] + pub fn assert_single_path(&self) -> &TaskPath { + match self { + PathSet::Set(set) => { + assert_eq!(set.len(), 1, "called assert_single_path on multiple paths"); + set.iter().next().unwrap() + } + PathSet::Suite(_) => unreachable!("called assert_single_path on a Suite path"), + } + } +} + +impl StepDescription { + fn from(kind: Kind) -> StepDescription { + StepDescription { + default: S::DEFAULT, + only_hosts: S::ONLY_HOSTS, + should_run: S::should_run, + make_run: S::make_run, + name: std::any::type_name::(), + kind, + } + } + + fn maybe_run(&self, builder: &Builder<'_>, mut pathsets: Vec) { + pathsets.retain(|set| !self.is_excluded(builder, set)); + + if pathsets.is_empty() { + return; + } + + // Determine the targets participating in this rule. + let targets = if self.only_hosts { &builder.hosts } else { &builder.targets }; + + for target in targets { + let run = RunConfig { builder, paths: pathsets.clone(), target: *target }; + (self.make_run)(run); + } + } + + fn is_excluded(&self, builder: &Builder<'_>, pathset: &PathSet) -> bool { + if builder.config.skip.iter().any(|e| pathset.has(&e, builder.kind)) { + if !matches!(builder.config.dry_run, DryRun::SelfCheck) { + println!("Skipping {pathset:?} because it is excluded"); + } + return true; + } + + if !builder.config.skip.is_empty() && !matches!(builder.config.dry_run, DryRun::SelfCheck) { + builder.verbose(&format!( + "{:?} not skipped for {:?} -- not in {:?}", + pathset, self.name, builder.config.skip + )); + } + false + } + + fn run(v: &[StepDescription], builder: &Builder<'_>, paths: &[PathBuf]) { + let should_runs = v + .iter() + .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind))) + .collect::>(); + + // sanity checks on rules + for (desc, should_run) in v.iter().zip(&should_runs) { + assert!( + !should_run.paths.is_empty(), + "{:?} should have at least one pathset", + desc.name + ); + } + + if paths.is_empty() || builder.config.include_default_paths { + for (desc, should_run) in v.iter().zip(&should_runs) { + if desc.default && should_run.is_really_default() { + desc.maybe_run(builder, should_run.paths.iter().cloned().collect()); + } + } + } + + // strip CurDir prefix if present + let mut paths: Vec<_> = + paths.into_iter().map(|p| p.strip_prefix(".").unwrap_or(p)).collect(); + + // Handle all test suite paths. + // (This is separate from the loop below to avoid having to handle multiple paths in `is_suite_path` somehow.) + paths.retain(|path| { + for (desc, should_run) in v.iter().zip(&should_runs) { + if let Some(suite) = should_run.is_suite_path(&path) { + desc.maybe_run(builder, vec![suite.clone()]); + return false; + } + } + true + }); + + if paths.is_empty() { + return; + } + + // Handle all PathSets. + for (desc, should_run) in v.iter().zip(&should_runs) { + let pathsets = should_run.pathset_for_paths_removing_matches(&mut paths, desc.kind); + if !pathsets.is_empty() { + desc.maybe_run(builder, pathsets); + } + } + + if !paths.is_empty() { + eprintln!("ERROR: no `{}` rules matched {:?}", builder.kind.as_str(), paths,); + eprintln!( + "HELP: run `x.py {} --help --verbose` to show a list of available paths", + builder.kind.as_str() + ); + eprintln!( + "NOTE: if you are adding a new Step to bootstrap itself, make sure you register it with `describe!`" + ); + crate::exit!(1); + } + } +} + +enum ReallyDefault<'a> { + Bool(bool), + Lazy(Lazy bool + 'a>>), +} + +pub struct ShouldRun<'a> { + pub builder: &'a Builder<'a>, + kind: Kind, + + // use a BTreeSet to maintain sort order + paths: BTreeSet, + + // If this is a default rule, this is an additional constraint placed on + // its run. Generally something like compiler docs being enabled. + is_really_default: ReallyDefault<'a>, +} + +impl<'a> ShouldRun<'a> { + fn new(builder: &'a Builder<'_>, kind: Kind) -> ShouldRun<'a> { + ShouldRun { + builder, + kind, + paths: BTreeSet::new(), + is_really_default: ReallyDefault::Bool(true), // by default no additional conditions + } + } + + pub fn default_condition(mut self, cond: bool) -> Self { + self.is_really_default = ReallyDefault::Bool(cond); + self + } + + pub fn lazy_default_condition(mut self, lazy_cond: Box bool + 'a>) -> Self { + self.is_really_default = ReallyDefault::Lazy(Lazy::new(lazy_cond)); + self + } + + pub fn is_really_default(&self) -> bool { + match &self.is_really_default { + ReallyDefault::Bool(val) => *val, + ReallyDefault::Lazy(lazy) => *lazy.deref(), + } + } + + /// Indicates it should run if the command-line selects the given crate or + /// any of its (local) dependencies. + /// + /// `make_run` will be called a single time with all matching command-line paths. + pub fn crate_or_deps(self, name: &str) -> Self { + let crates = self.builder.in_tree_crates(name, None); + self.crates(crates) + } + + /// Indicates it should run if the command-line selects any of the given crates. + /// + /// `make_run` will be called a single time with all matching command-line paths. + /// + /// Prefer [`ShouldRun::crate_or_deps`] to this function where possible. + pub(crate) fn crates(mut self, crates: Vec<&Crate>) -> Self { + for krate in crates { + let path = krate.local_path(self.builder); + self.paths.insert(PathSet::one(path, self.kind)); + } + self + } + + // single alias, which does not correspond to any on-disk path + pub fn alias(mut self, alias: &str) -> Self { + // exceptional case for `Kind::Setup` because its `library` + // and `compiler` options would otherwise naively match with + // `compiler` and `library` folders respectively. + assert!( + self.kind == Kind::Setup || !self.builder.src.join(alias).exists(), + "use `builder.path()` for real paths: {alias}" + ); + self.paths.insert(PathSet::Set( + std::iter::once(TaskPath { path: alias.into(), kind: Some(self.kind) }).collect(), + )); + self + } + + // single, non-aliased path + pub fn path(self, path: &str) -> Self { + self.paths(&[path]) + } + + /// Multiple aliases for the same job. + /// + /// This differs from [`path`] in that multiple calls to path will end up calling `make_run` + /// multiple times, whereas a single call to `paths` will only ever generate a single call to + /// `paths`. + /// + /// This is analogous to `all_krates`, although `all_krates` is gone now. Prefer [`path`] where possible. + /// + /// [`path`]: ShouldRun::path + pub fn paths(mut self, paths: &[&str]) -> Self { + static SUBMODULES_PATHS: OnceCell> = OnceCell::new(); + + let init_submodules_paths = |src: &PathBuf| { + let file = File::open(src.join(".gitmodules")).unwrap(); + + let mut submodules_paths = vec![]; + for line in BufReader::new(file).lines() { + if let Ok(line) = line { + let line = line.trim(); + + if line.starts_with("path") { + let actual_path = + line.split(' ').last().expect("Couldn't get value of path"); + submodules_paths.push(actual_path.to_owned()); + } + } + } + + submodules_paths + }; + + let submodules_paths = + SUBMODULES_PATHS.get_or_init(|| init_submodules_paths(&self.builder.src)); + + self.paths.insert(PathSet::Set( + paths + .iter() + .map(|p| { + // assert only if `p` isn't submodule + if submodules_paths.iter().find(|sm_p| p.contains(*sm_p)).is_none() { + assert!( + self.builder.src.join(p).exists(), + "`should_run.paths` should correspond to real on-disk paths - use `alias` if there is no relevant path: {}", + p + ); + } + + TaskPath { path: p.into(), kind: Some(self.kind) } + }) + .collect(), + )); + self + } + + /// Handles individual files (not directories) within a test suite. + fn is_suite_path(&self, requested_path: &Path) -> Option<&PathSet> { + self.paths.iter().find(|pathset| match pathset { + PathSet::Suite(suite) => requested_path.starts_with(&suite.path), + PathSet::Set(_) => false, + }) + } + + pub fn suite_path(mut self, suite: &str) -> Self { + self.paths.insert(PathSet::Suite(TaskPath { path: suite.into(), kind: Some(self.kind) })); + self + } + + // allows being more explicit about why should_run in Step returns the value passed to it + pub fn never(mut self) -> ShouldRun<'a> { + self.paths.insert(PathSet::empty()); + self + } + + /// Given a set of requested paths, return the subset which match the Step for this `ShouldRun`, + /// removing the matches from `paths`. + /// + /// NOTE: this returns multiple PathSets to allow for the possibility of multiple units of work + /// within the same step. For example, `test::Crate` allows testing multiple crates in the same + /// cargo invocation, which are put into separate sets because they aren't aliases. + /// + /// The reason we return PathSet instead of PathBuf is to allow for aliases that mean the same thing + /// (for now, just `all_krates` and `paths`, but we may want to add an `aliases` function in the future?) + fn pathset_for_paths_removing_matches( + &self, + paths: &mut Vec<&Path>, + kind: Kind, + ) -> Vec { + let mut sets = vec![]; + for pathset in &self.paths { + let subset = pathset.intersection_removing_matches(paths, kind); + if subset != PathSet::empty() { + sets.push(subset); + } + } + sets + } +} + +#[derive(Copy, Clone, PartialEq, Eq, PartialOrd, Ord, ValueEnum)] +pub enum Kind { + #[clap(alias = "b")] + Build, + #[clap(alias = "c")] + Check, + Clippy, + Fix, + Format, + #[clap(alias = "t")] + Test, + Bench, + #[clap(alias = "d")] + Doc, + Clean, + Dist, + Install, + #[clap(alias = "r")] + Run, + Setup, + Suggest, +} + +impl Kind { + pub fn parse(string: &str) -> Option { + // these strings, including the one-letter aliases, must match the x.py help text + Some(match string { + "build" | "b" => Kind::Build, + "check" | "c" => Kind::Check, + "clippy" => Kind::Clippy, + "fix" => Kind::Fix, + "fmt" => Kind::Format, + "test" | "t" => Kind::Test, + "bench" => Kind::Bench, + "doc" | "d" => Kind::Doc, + "clean" => Kind::Clean, + "dist" => Kind::Dist, + "install" => Kind::Install, + "run" | "r" => Kind::Run, + "setup" => Kind::Setup, + "suggest" => Kind::Suggest, + _ => return None, + }) + } + + pub fn as_str(&self) -> &'static str { + match self { + Kind::Build => "build", + Kind::Check => "check", + Kind::Clippy => "clippy", + Kind::Fix => "fix", + Kind::Format => "fmt", + Kind::Test => "test", + Kind::Bench => "bench", + Kind::Doc => "doc", + Kind::Clean => "clean", + Kind::Dist => "dist", + Kind::Install => "install", + Kind::Run => "run", + Kind::Setup => "setup", + Kind::Suggest => "suggest", + } + } + + pub fn description(&self) -> String { + match self { + Kind::Test => "Testing", + Kind::Bench => "Benchmarking", + Kind::Doc => "Documenting", + Kind::Run => "Running", + Kind::Suggest => "Suggesting", + _ => { + let title_letter = self.as_str()[0..1].to_ascii_uppercase(); + return format!("{title_letter}{}ing", &self.as_str()[1..]); + } + } + .to_owned() + } +} + +impl<'a> Builder<'a> { + fn get_step_descriptions(kind: Kind) -> Vec { + macro_rules! describe { + ($($rule:ty),+ $(,)?) => {{ + vec![$(StepDescription::from::<$rule>(kind)),+] + }}; + } + match kind { + Kind::Build => describe!( + compile::Std, + compile::Rustc, + compile::Assemble, + compile::CodegenBackend, + compile::StartupObjects, + tool::BuildManifest, + tool::Rustbook, + tool::ErrorIndex, + tool::UnstableBookGen, + tool::Tidy, + tool::Linkchecker, + tool::CargoTest, + tool::Compiletest, + tool::RemoteTestServer, + tool::RemoteTestClient, + tool::RustInstaller, + tool::Cargo, + tool::Rls, + tool::RustAnalyzer, + tool::RustAnalyzerProcMacroSrv, + tool::RustDemangler, + tool::Rustdoc, + tool::Clippy, + tool::CargoClippy, + llvm::Llvm, + llvm::Sanitizers, + tool::Rustfmt, + tool::Miri, + tool::CargoMiri, + llvm::Lld, + llvm::CrtBeginEnd, + tool::RustdocGUITest, + tool::OptimizedDist, + tool::CoverageDump, + ), + Kind::Check | Kind::Clippy | Kind::Fix => describe!( + check::Std, + check::Rustc, + check::Rustdoc, + check::CodegenBackend, + check::Clippy, + check::Miri, + check::CargoMiri, + check::MiroptTestTools, + check::Rls, + check::Rustfmt, + check::RustAnalyzer, + check::Bootstrap + ), + Kind::Test => describe!( + crate::core::build_steps::toolstate::ToolStateCheck, + test::ExpandYamlAnchors, + test::Tidy, + test::Ui, + test::RunPassValgrind, + test::Coverage, + test::CoverageMap, + test::CoverageRun, + test::MirOpt, + test::Codegen, + test::CodegenUnits, + test::Assembly, + test::Incremental, + test::Debuginfo, + test::UiFullDeps, + test::CodegenCranelift, + test::CodegenGCC, + test::Rustdoc, + test::CoverageRunRustdoc, + test::Pretty, + test::Crate, + test::CrateLibrustc, + test::CrateRustdoc, + test::CrateRustdocJsonTypes, + test::CrateBootstrap, + test::Linkcheck, + test::TierCheck, + test::Cargotest, + test::Cargo, + test::RustAnalyzer, + test::ErrorIndex, + test::Distcheck, + test::RunMakeFullDeps, + test::Nomicon, + test::Reference, + test::RustdocBook, + test::RustByExample, + test::TheBook, + test::UnstableBook, + test::RustcBook, + test::LintDocs, + test::RustcGuide, + test::EmbeddedBook, + test::EditionGuide, + test::Rustfmt, + test::Miri, + test::Clippy, + test::RustDemangler, + test::CompiletestTest, + test::RustdocJSStd, + test::RustdocJSNotStd, + test::RustdocGUI, + test::RustdocTheme, + test::RustdocUi, + test::RustdocJson, + test::HtmlCheck, + test::RustInstaller, + // Run bootstrap close to the end as it's unlikely to fail + test::Bootstrap, + // Run run-make last, since these won't pass without make on Windows + test::RunMake, + ), + Kind::Bench => describe!(test::Crate, test::CrateLibrustc), + Kind::Doc => describe!( + doc::UnstableBook, + doc::UnstableBookGen, + doc::TheBook, + doc::Standalone, + doc::Std, + doc::Rustc, + doc::Rustdoc, + doc::Rustfmt, + doc::ErrorIndex, + doc::Nomicon, + doc::Reference, + doc::RustdocBook, + doc::RustByExample, + doc::RustcBook, + doc::Cargo, + doc::CargoBook, + doc::Clippy, + doc::ClippyBook, + doc::Miri, + doc::EmbeddedBook, + doc::EditionGuide, + doc::StyleGuide, + doc::Tidy, + doc::Bootstrap, + ), + Kind::Dist => describe!( + dist::Docs, + dist::RustcDocs, + dist::JsonDocs, + dist::Mingw, + dist::Rustc, + dist::CodegenBackend, + dist::Std, + dist::RustcDev, + dist::Analysis, + dist::Src, + dist::Cargo, + dist::Rls, + dist::RustAnalyzer, + dist::Rustfmt, + dist::RustDemangler, + dist::Clippy, + dist::Miri, + dist::LlvmTools, + dist::RustDev, + dist::Bootstrap, + dist::Extended, + // It seems that PlainSourceTarball somehow changes how some of the tools + // perceive their dependencies (see #93033) which would invalidate fingerprints + // and force us to rebuild tools after vendoring dependencies. + // To work around this, create the Tarball after building all the tools. + dist::PlainSourceTarball, + dist::BuildManifest, + dist::ReproducibleArtifacts, + ), + Kind::Install => describe!( + install::Docs, + install::Std, + install::Cargo, + install::RustAnalyzer, + install::Rustfmt, + install::RustDemangler, + install::Clippy, + install::Miri, + install::LlvmTools, + install::Src, + install::Rustc + ), + Kind::Run => describe!( + run::ExpandYamlAnchors, + run::BuildManifest, + run::BumpStage0, + run::ReplaceVersionPlaceholder, + run::Miri, + run::CollectLicenseMetadata, + run::GenerateCopyright, + run::GenerateWindowsSys, + run::GenerateCompletions, + ), + Kind::Setup => describe!(setup::Profile, setup::Hook, setup::Link, setup::Vscode), + Kind::Clean => describe!(clean::CleanAll, clean::Rustc, clean::Std), + // special-cased in Build::build() + Kind::Format | Kind::Suggest => vec![], + } + } + + pub fn get_help(build: &Build, kind: Kind) -> Option { + let step_descriptions = Builder::get_step_descriptions(kind); + if step_descriptions.is_empty() { + return None; + } + + let builder = Self::new_internal(build, kind, vec![]); + let builder = &builder; + // The "build" kind here is just a placeholder, it will be replaced with something else in + // the following statement. + let mut should_run = ShouldRun::new(builder, Kind::Build); + for desc in step_descriptions { + should_run.kind = desc.kind; + should_run = (desc.should_run)(should_run); + } + let mut help = String::from("Available paths:\n"); + let mut add_path = |path: &Path| { + t!(write!(help, " ./x.py {} {}\n", kind.as_str(), path.display())); + }; + for pathset in should_run.paths { + match pathset { + PathSet::Set(set) => { + for path in set { + add_path(&path.path); + } + } + PathSet::Suite(path) => { + add_path(&path.path.join("...")); + } + } + } + Some(help) + } + + fn new_internal(build: &Build, kind: Kind, paths: Vec) -> Builder<'_> { + Builder { + build, + top_stage: build.config.stage, + kind, + cache: Cache::new(), + stack: RefCell::new(Vec::new()), + time_spent_on_dependencies: Cell::new(Duration::new(0, 0)), + paths, + } + } + + pub fn new(build: &Build) -> Builder<'_> { + let paths = &build.config.paths; + let (kind, paths) = match build.config.cmd { + Subcommand::Build => (Kind::Build, &paths[..]), + Subcommand::Check { .. } => (Kind::Check, &paths[..]), + Subcommand::Clippy { .. } => (Kind::Clippy, &paths[..]), + Subcommand::Fix => (Kind::Fix, &paths[..]), + Subcommand::Doc { .. } => (Kind::Doc, &paths[..]), + Subcommand::Test { .. } => (Kind::Test, &paths[..]), + Subcommand::Bench { .. } => (Kind::Bench, &paths[..]), + Subcommand::Dist => (Kind::Dist, &paths[..]), + Subcommand::Install => (Kind::Install, &paths[..]), + Subcommand::Run { .. } => (Kind::Run, &paths[..]), + Subcommand::Clean { .. } => (Kind::Clean, &paths[..]), + Subcommand::Format { .. } => (Kind::Format, &[][..]), + Subcommand::Suggest { .. } => (Kind::Suggest, &[][..]), + Subcommand::Setup { profile: ref path } => ( + Kind::Setup, + path.as_ref().map_or([].as_slice(), |path| std::slice::from_ref(path)), + ), + }; + + Self::new_internal(build, kind, paths.to_owned()) + } + + pub fn execute_cli(&self) { + self.run_step_descriptions(&Builder::get_step_descriptions(self.kind), &self.paths); + } + + pub fn default_doc(&self, paths: &[PathBuf]) { + self.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), paths); + } + + pub fn doc_rust_lang_org_channel(&self) -> String { + let channel = match &*self.config.channel { + "stable" => &self.version, + "beta" => "beta", + "nightly" | "dev" => "nightly", + // custom build of rustdoc maybe? link to the latest stable docs just in case + _ => "stable", + }; + "https://doc.rust-lang.org/".to_owned() + channel + } + + fn run_step_descriptions(&self, v: &[StepDescription], paths: &[PathBuf]) { + StepDescription::run(v, self, paths); + } + + /// Obtain a compiler at a given stage and for a given host. Explicitly does + /// not take `Compiler` since all `Compiler` instances are meant to be + /// obtained through this function, since it ensures that they are valid + /// (i.e., built and assembled). + pub fn compiler(&self, stage: u32, host: TargetSelection) -> Compiler { + self.ensure(compile::Assemble { target_compiler: Compiler { stage, host } }) + } + + /// Similar to `compiler`, except handles the full-bootstrap option to + /// silently use the stage1 compiler instead of a stage2 compiler if one is + /// requested. + /// + /// Note that this does *not* have the side effect of creating + /// `compiler(stage, host)`, unlike `compiler` above which does have such + /// a side effect. The returned compiler here can only be used to compile + /// new artifacts, it can't be used to rely on the presence of a particular + /// sysroot. + /// + /// See `force_use_stage1` and `force_use_stage2` for documentation on what each argument is. + pub fn compiler_for( + &self, + stage: u32, + host: TargetSelection, + target: TargetSelection, + ) -> Compiler { + if self.build.force_use_stage2(stage) { + self.compiler(2, self.config.build) + } else if self.build.force_use_stage1(stage, target) { + self.compiler(1, self.config.build) + } else { + self.compiler(stage, host) + } + } + + pub fn sysroot(&self, compiler: Compiler) -> Interned { + self.ensure(compile::Sysroot::new(compiler)) + } + + /// Returns the libdir where the standard library and other artifacts are + /// found for a compiler's sysroot. + pub fn sysroot_libdir(&self, compiler: Compiler, target: TargetSelection) -> Interned { + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] + struct Libdir { + compiler: Compiler, + target: TargetSelection, + } + impl Step for Libdir { + type Output = Interned; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.never() + } + + fn run(self, builder: &Builder<'_>) -> Interned { + let lib = builder.sysroot_libdir_relative(self.compiler); + let sysroot = builder + .sysroot(self.compiler) + .join(lib) + .join("rustlib") + .join(self.target.triple) + .join("lib"); + // Avoid deleting the rustlib/ directory we just copied + // (in `impl Step for Sysroot`). + if !builder.download_rustc() { + builder.verbose(&format!( + "Removing sysroot {} to avoid caching bugs", + sysroot.display() + )); + let _ = fs::remove_dir_all(&sysroot); + t!(fs::create_dir_all(&sysroot)); + } + + if self.compiler.stage == 0 { + // The stage 0 compiler for the build triple is always pre-built. + // Ensure that `libLLVM.so` ends up in the target libdir, so that ui-fulldeps tests can use it when run. + dist::maybe_install_llvm_target( + builder, + self.compiler.host, + &builder.sysroot(self.compiler), + ); + } + + INTERNER.intern_path(sysroot) + } + } + self.ensure(Libdir { compiler, target }) + } + + pub fn sysroot_codegen_backends(&self, compiler: Compiler) -> PathBuf { + self.sysroot_libdir(compiler, compiler.host).with_file_name("codegen-backends") + } + + /// Returns the compiler's libdir where it stores the dynamic libraries that + /// it itself links against. + /// + /// For example this returns `/lib` on Unix and `/bin` on + /// Windows. + pub fn rustc_libdir(&self, compiler: Compiler) -> PathBuf { + if compiler.is_snapshot(self) { + self.rustc_snapshot_libdir() + } else { + match self.config.libdir_relative() { + Some(relative_libdir) if compiler.stage >= 1 => { + self.sysroot(compiler).join(relative_libdir) + } + _ => self.sysroot(compiler).join(libdir(compiler.host)), + } + } + } + + /// Returns the compiler's relative libdir where it stores the dynamic libraries that + /// it itself links against. + /// + /// For example this returns `lib` on Unix and `bin` on + /// Windows. + pub fn libdir_relative(&self, compiler: Compiler) -> &Path { + if compiler.is_snapshot(self) { + libdir(self.config.build).as_ref() + } else { + match self.config.libdir_relative() { + Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, + _ => libdir(compiler.host).as_ref(), + } + } + } + + /// Returns the compiler's relative libdir where the standard library and other artifacts are + /// found for a compiler's sysroot. + /// + /// For example this returns `lib` on Unix and Windows. + pub fn sysroot_libdir_relative(&self, compiler: Compiler) -> &Path { + match self.config.libdir_relative() { + Some(relative_libdir) if compiler.stage >= 1 => relative_libdir, + _ if compiler.stage == 0 => &self.build.initial_libdir, + _ => Path::new("lib"), + } + } + + pub fn rustc_lib_paths(&self, compiler: Compiler) -> Vec { + let mut dylib_dirs = vec![self.rustc_libdir(compiler)]; + + // Ensure that the downloaded LLVM libraries can be found. + if self.config.llvm_from_ci { + let ci_llvm_lib = self.out.join(&*compiler.host.triple).join("ci-llvm").join("lib"); + dylib_dirs.push(ci_llvm_lib); + } + + dylib_dirs + } + + /// Adds the compiler's directory of dynamic libraries to `cmd`'s dynamic + /// library lookup path. + pub fn add_rustc_lib_path(&self, compiler: Compiler, cmd: &mut Command) { + // Windows doesn't need dylib path munging because the dlls for the + // compiler live next to the compiler and the system will find them + // automatically. + if cfg!(windows) { + return; + } + + add_dylib_path(self.rustc_lib_paths(compiler), cmd); + } + + /// Gets a path to the compiler specified. + pub fn rustc(&self, compiler: Compiler) -> PathBuf { + if compiler.is_snapshot(self) { + self.initial_rustc.clone() + } else { + self.sysroot(compiler).join("bin").join(exe("rustc", compiler.host)) + } + } + + /// Gets the paths to all of the compiler's codegen backends. + fn codegen_backends(&self, compiler: Compiler) -> impl Iterator { + fs::read_dir(self.sysroot_codegen_backends(compiler)) + .into_iter() + .flatten() + .filter_map(Result::ok) + .map(|entry| entry.path()) + } + + pub fn rustdoc(&self, compiler: Compiler) -> PathBuf { + self.ensure(tool::Rustdoc { compiler }) + } + + pub fn rustdoc_cmd(&self, compiler: Compiler) -> Command { + let mut cmd = Command::new(&self.bootstrap_out.join("rustdoc")); + cmd.env("RUSTC_STAGE", compiler.stage.to_string()) + .env("RUSTC_SYSROOT", self.sysroot(compiler)) + // Note that this is *not* the sysroot_libdir because rustdoc must be linked + // equivalently to rustc. + .env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)) + .env("CFG_RELEASE_CHANNEL", &self.config.channel) + .env("RUSTDOC_REAL", self.rustdoc(compiler)) + .env("RUSTC_BOOTSTRAP", "1"); + + cmd.arg("-Wrustdoc::invalid_codeblock_attributes"); + + if self.config.deny_warnings { + cmd.arg("-Dwarnings"); + } + cmd.arg("-Znormalize-docs"); + + // Remove make-related flags that can cause jobserver problems. + cmd.env_remove("MAKEFLAGS"); + cmd.env_remove("MFLAGS"); + + if let Some(linker) = self.linker(compiler.host) { + cmd.env("RUSTDOC_LINKER", linker); + } + cmd + } + + /// Return the path to `llvm-config` for the target, if it exists. + /// + /// Note that this returns `None` if LLVM is disabled, or if we're in a + /// check build or dry-run, where there's no need to build all of LLVM. + fn llvm_config(&self, target: TargetSelection) -> Option { + if self.config.llvm_enabled() && self.kind != Kind::Check && !self.config.dry_run() { + let llvm::LlvmResult { llvm_config, .. } = self.ensure(llvm::Llvm { target }); + if llvm_config.is_file() { + return Some(llvm_config); + } + } + None + } + + /// Like `cargo`, but only passes flags that are valid for all commands. + pub fn bare_cargo( + &self, + compiler: Compiler, + mode: Mode, + target: TargetSelection, + cmd: &str, + ) -> Command { + let mut cargo = Command::new(&self.initial_cargo); + // Run cargo from the source root so it can find .cargo/config. + // This matters when using vendoring and the working directory is outside the repository. + cargo.current_dir(&self.src); + + let out_dir = self.stage_out(compiler, mode); + cargo.env("CARGO_TARGET_DIR", &out_dir).arg(cmd); + + // Found with `rg "init_env_logger\("`. If anyone uses `init_env_logger` + // from out of tree it shouldn't matter, since x.py is only used for + // building in-tree. + let color_logs = ["RUSTDOC_LOG_COLOR", "RUSTC_LOG_COLOR", "RUST_LOG_COLOR"]; + match self.build.config.color { + Color::Always => { + cargo.arg("--color=always"); + for log in &color_logs { + cargo.env(log, "always"); + } + } + Color::Never => { + cargo.arg("--color=never"); + for log in &color_logs { + cargo.env(log, "never"); + } + } + Color::Auto => {} // nothing to do + } + + if cmd != "install" { + cargo.arg("--target").arg(target.rustc_target_arg()); + } else { + assert_eq!(target, compiler.host); + } + + if self.config.rust_optimize.is_release() { + // FIXME: cargo bench/install do not accept `--release` + if cmd != "bench" && cmd != "install" { + cargo.arg("--release"); + } + } + + // Remove make-related flags to ensure Cargo can correctly set things up + cargo.env_remove("MAKEFLAGS"); + cargo.env_remove("MFLAGS"); + + cargo + } + + /// Prepares an invocation of `cargo` to be run. + /// + /// This will create a `Command` that represents a pending execution of + /// Cargo. This cargo will be configured to use `compiler` as the actual + /// rustc compiler, its output will be scoped by `mode`'s output directory, + /// it will pass the `--target` flag for the specified `target`, and will be + /// executing the Cargo command `cmd`. + pub fn cargo( + &self, + compiler: Compiler, + mode: Mode, + source_type: SourceType, + target: TargetSelection, + cmd: &str, + ) -> Cargo { + let mut cargo = self.bare_cargo(compiler, mode, target, cmd); + let out_dir = self.stage_out(compiler, mode); + + let mut hostflags = HostFlags::default(); + + // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, + // so we need to explicitly clear out if they've been updated. + for backend in self.codegen_backends(compiler) { + self.clear_if_dirty(&out_dir, &backend); + } + + if cmd == "doc" || cmd == "rustdoc" { + let my_out = match mode { + // This is the intended out directory for compiler documentation. + Mode::Rustc | Mode::ToolRustc => self.compiler_doc_out(target), + Mode::Std => { + if self.config.cmd.json() { + out_dir.join(target.triple).join("json-doc") + } else { + out_dir.join(target.triple).join("doc") + } + } + _ => panic!("doc mode {mode:?} not expected"), + }; + let rustdoc = self.rustdoc(compiler); + self.clear_if_dirty(&my_out, &rustdoc); + } + + let profile_var = |name: &str| { + let profile = if self.config.rust_optimize.is_release() { "RELEASE" } else { "DEV" }; + format!("CARGO_PROFILE_{}_{}", profile, name) + }; + + // See comment in rustc_llvm/build.rs for why this is necessary, largely llvm-config + // needs to not accidentally link to libLLVM in stage0/lib. + cargo.env("REAL_LIBRARY_PATH_VAR", &helpers::dylib_path_var()); + if let Some(e) = env::var_os(helpers::dylib_path_var()) { + cargo.env("REAL_LIBRARY_PATH", e); + } + + // Set a flag for `check`/`clippy`/`fix`, so that certain build + // scripts can do less work (i.e. not building/requiring LLVM). + if cmd == "check" || cmd == "clippy" || cmd == "fix" { + // If we've not yet built LLVM, or it's stale, then bust + // the rustc_llvm cache. That will always work, even though it + // may mean that on the next non-check build we'll need to rebuild + // rustc_llvm. But if LLVM is stale, that'll be a tiny amount + // of work comparatively, and we'd likely need to rebuild it anyway, + // so that's okay. + if crate::core::build_steps::llvm::prebuilt_llvm_config(self, target).is_err() { + cargo.env("RUST_CHECK", "1"); + } + } + + let stage = if compiler.stage == 0 && self.local_rebuild { + // Assume the local-rebuild rustc already has stage1 features. + 1 + } else { + compiler.stage + }; + + let mut rustflags = Rustflags::new(target); + if stage != 0 { + if let Ok(s) = env::var("CARGOFLAGS_NOT_BOOTSTRAP") { + cargo.args(s.split_whitespace()); + } + rustflags.env("RUSTFLAGS_NOT_BOOTSTRAP"); + } else { + if let Ok(s) = env::var("CARGOFLAGS_BOOTSTRAP") { + cargo.args(s.split_whitespace()); + } + rustflags.env("RUSTFLAGS_BOOTSTRAP"); + if cmd == "clippy" { + // clippy overwrites sysroot if we pass it to cargo. + // Pass it directly to clippy instead. + // NOTE: this can't be fixed in clippy because we explicitly don't set `RUSTC`, + // so it has no way of knowing the sysroot. + rustflags.arg("--sysroot"); + rustflags.arg( + self.sysroot(compiler) + .as_os_str() + .to_str() + .expect("sysroot must be valid UTF-8"), + ); + // Only run clippy on a very limited subset of crates (in particular, not build scripts). + cargo.arg("-Zunstable-options"); + // Explicitly does *not* set `--cfg=bootstrap`, since we're using a nightly clippy. + let host_version = Command::new("rustc").arg("--version").output().map_err(|_| ()); + let output = host_version.and_then(|output| { + if output.status.success() { + Ok(output) + } else { + Err(()) + } + }).unwrap_or_else(|_| { + eprintln!( + "ERROR: `x.py clippy` requires a host `rustc` toolchain with the `clippy` component" + ); + eprintln!("HELP: try `rustup component add clippy`"); + crate::exit!(1); + }); + if !t!(std::str::from_utf8(&output.stdout)).contains("nightly") { + rustflags.arg("--cfg=bootstrap"); + } + } else { + rustflags.arg("--cfg=bootstrap"); + } + } + + let use_new_symbol_mangling = match self.config.rust_new_symbol_mangling { + Some(setting) => { + // If an explicit setting is given, use that + setting + } + None => { + if mode == Mode::Std { + // The standard library defaults to the legacy scheme + false + } else { + // The compiler and tools default to the new scheme + true + } + } + }; + + // By default, windows-rs depends on a native library that doesn't get copied into the + // sysroot. Passing this cfg enables raw-dylib support instead, which makes the native + // library unnecessary. This can be removed when windows-rs enables raw-dylib + // unconditionally. + if let Mode::Rustc | Mode::ToolRustc = mode { + rustflags.arg("--cfg=windows_raw_dylib"); + } + + if use_new_symbol_mangling { + rustflags.arg("-Csymbol-mangling-version=v0"); + } else { + rustflags.arg("-Csymbol-mangling-version=legacy"); + rustflags.arg("-Zunstable-options"); + } + + // #[cfg(bootstrap)] + let use_new_check_cfg_syntax = self.local_rebuild; + + // Enable compile-time checking of `cfg` names, values and Cargo `features`. + // + // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like + // backtrace, core_simd, std_float, ...), those dependencies have their own + // features but cargo isn't involved in the #[path] process and so cannot pass the + // complete list of features, so for that reason we don't enable checking of + // features for std crates. + if use_new_check_cfg_syntax { + cargo.arg("-Zcheck-cfg"); + if mode == Mode::Std { + rustflags.arg("--check-cfg=cfg(feature,values(any()))"); + } + } else { + cargo.arg(if mode != Mode::Std { + "-Zcheck-cfg=names,values,output,features" + } else { + "-Zcheck-cfg=names,values,output" + }); + } + + // Add extra cfg not defined in/by rustc + // + // Note: Although it would seems that "-Zunstable-options" to `rustflags` is useless as + // cargo would implicitly add it, it was discover that sometimes bootstrap only use + // `rustflags` without `cargo` making it required. + rustflags.arg("-Zunstable-options"); + for (restricted_mode, name, values) in EXTRA_CHECK_CFGS { + if *restricted_mode == None || *restricted_mode == Some(mode) { + // Creating a string of the values by concatenating each value: + // ',"tvos","watchos"' or '' (nothing) when there are no values + let values = match values { + Some(values) => values + .iter() + .map(|val| [",", "\"", val, "\""]) + .flatten() + .collect::(), + None => String::new(), + }; + if use_new_check_cfg_syntax { + let values = values.strip_prefix(",").unwrap_or(&values); // remove the first `,` + rustflags.arg(&format!("--check-cfg=cfg({name},values({values}))")); + } else { + rustflags.arg(&format!("--check-cfg=values({name}{values})")); + } + } + } + + // FIXME(rust-lang/cargo#5754) we shouldn't be using special command arguments + // to the host invocation here, but rather Cargo should know what flags to pass rustc + // itself. + if stage == 0 { + hostflags.arg("--cfg=bootstrap"); + } + // Cargo doesn't pass RUSTFLAGS to proc_macros: + // https://github.com/rust-lang/cargo/issues/4423 + // Thus, if we are on stage 0, we explicitly set `--cfg=bootstrap`. + // We also declare that the flag is expected, which we need to do to not + // get warnings about it being unexpected. + hostflags.arg("-Zunstable-options"); + if use_new_check_cfg_syntax { + hostflags.arg("--check-cfg=cfg(bootstrap)"); + } else { + hostflags.arg("--check-cfg=values(bootstrap)"); + } + + // FIXME: It might be better to use the same value for both `RUSTFLAGS` and `RUSTDOCFLAGS`, + // but this breaks CI. At the very least, stage0 `rustdoc` needs `--cfg bootstrap`. See + // #71458. + let mut rustdocflags = rustflags.clone(); + rustdocflags.propagate_cargo_env("RUSTDOCFLAGS"); + if stage == 0 { + rustdocflags.env("RUSTDOCFLAGS_BOOTSTRAP"); + } else { + rustdocflags.env("RUSTDOCFLAGS_NOT_BOOTSTRAP"); + } + + if let Ok(s) = env::var("CARGOFLAGS") { + cargo.args(s.split_whitespace()); + } + + match mode { + Mode::Std | Mode::ToolBootstrap | Mode::ToolStd => {} + Mode::Rustc | Mode::Codegen | Mode::ToolRustc => { + // Build proc macros both for the host and the target + if target != compiler.host && cmd != "check" { + cargo.arg("-Zdual-proc-macros"); + rustflags.arg("-Zdual-proc-macros"); + } + } + } + + // This tells Cargo (and in turn, rustc) to output more complete + // dependency information. Most importantly for rustbuild, this + // includes sysroot artifacts, like libstd, which means that we don't + // need to track those in rustbuild (an error prone process!). This + // feature is currently unstable as there may be some bugs and such, but + // it represents a big improvement in rustbuild's reliability on + // rebuilds, so we're using it here. + // + // For some additional context, see #63470 (the PR originally adding + // this), as well as #63012 which is the tracking issue for this + // feature on the rustc side. + cargo.arg("-Zbinary-dep-depinfo"); + let allow_features = match mode { + Mode::ToolBootstrap | Mode::ToolStd => { + // Restrict the allowed features so we don't depend on nightly + // accidentally. + // + // binary-dep-depinfo is used by rustbuild itself for all + // compilations. + // + // Lots of tools depend on proc_macro2 and proc-macro-error. + // Those have build scripts which assume nightly features are + // available if the `rustc` version is "nighty" or "dev". See + // bin/rustc.rs for why that is a problem. Instead of labeling + // those features for each individual tool that needs them, + // just blanket allow them here. + // + // If this is ever removed, be sure to add something else in + // its place to keep the restrictions in place (or make a way + // to unset RUSTC_BOOTSTRAP). + "binary-dep-depinfo,proc_macro_span,proc_macro_span_shrink,proc_macro_diagnostic" + .to_string() + } + Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => String::new(), + }; + + cargo.arg("-j").arg(self.jobs().to_string()); + + // FIXME: Temporary fix for https://github.com/rust-lang/cargo/issues/3005 + // Force cargo to output binaries with disambiguating hashes in the name + let mut metadata = if compiler.stage == 0 { + // Treat stage0 like a special channel, whether it's a normal prior- + // release rustc or a local rebuild with the same version, so we + // never mix these libraries by accident. + "bootstrap".to_string() + } else { + self.config.channel.to_string() + }; + // We want to make sure that none of the dependencies between + // std/test/rustc unify with one another. This is done for weird linkage + // reasons but the gist of the problem is that if librustc, libtest, and + // libstd all depend on libc from crates.io (which they actually do) we + // want to make sure they all get distinct versions. Things get really + // weird if we try to unify all these dependencies right now, namely + // around how many times the library is linked in dynamic libraries and + // such. If rustc were a static executable or if we didn't ship dylibs + // this wouldn't be a problem, but we do, so it is. This is in general + // just here to make sure things build right. If you can remove this and + // things still build right, please do! + match mode { + Mode::Std => metadata.push_str("std"), + // When we're building rustc tools, they're built with a search path + // that contains things built during the rustc build. For example, + // bitflags is built during the rustc build, and is a dependency of + // rustdoc as well. We're building rustdoc in a different target + // directory, though, which means that Cargo will rebuild the + // dependency. When we go on to build rustdoc, we'll look for + // bitflags, and find two different copies: one built during the + // rustc step and one that we just built. This isn't always a + // problem, somehow -- not really clear why -- but we know that this + // fixes things. + Mode::ToolRustc => metadata.push_str("tool-rustc"), + // Same for codegen backends. + Mode::Codegen => metadata.push_str("codegen"), + _ => {} + } + cargo.env("__CARGO_DEFAULT_LIB_METADATA", &metadata); + + if cmd == "clippy" { + rustflags.arg("-Zforce-unstable-if-unmarked"); + } + + rustflags.arg("-Zmacro-backtrace"); + + let want_rustdoc = self.doc_tests != DocTests::No; + + // We synthetically interpret a stage0 compiler used to build tools as a + // "raw" compiler in that it's the exact snapshot we download. Normally + // the stage0 build means it uses libraries build by the stage0 + // compiler, but for tools we just use the precompiled libraries that + // we've downloaded + let use_snapshot = mode == Mode::ToolBootstrap; + assert!(!use_snapshot || stage == 0 || self.local_rebuild); + + let maybe_sysroot = self.sysroot(compiler); + let sysroot = if use_snapshot { self.rustc_snapshot_sysroot() } else { &maybe_sysroot }; + let libdir = self.rustc_libdir(compiler); + + // Clear the output directory if the real rustc we're using has changed; + // Cargo cannot detect this as it thinks rustc is bootstrap/debug/rustc. + // + // Avoid doing this during dry run as that usually means the relevant + // compiler is not yet linked/copied properly. + // + // Only clear out the directory if we're compiling std; otherwise, we + // should let Cargo take care of things for us (via depdep info) + if !self.config.dry_run() && mode == Mode::Std && cmd == "build" { + self.clear_if_dirty(&out_dir, &self.rustc(compiler)); + } + + // Customize the compiler we're running. Specify the compiler to cargo + // as our shim and then pass it some various options used to configure + // how the actual compiler itself is called. + // + // These variables are primarily all read by + // src/bootstrap/bin/{rustc.rs,rustdoc.rs} + cargo + .env("RUSTBUILD_NATIVE_DIR", self.native_dir(target)) + .env("RUSTC_REAL", self.rustc(compiler)) + .env("RUSTC_STAGE", stage.to_string()) + .env("RUSTC_SYSROOT", &sysroot) + .env("RUSTC_LIBDIR", &libdir) + .env("RUSTDOC", self.bootstrap_out.join("rustdoc")) + .env( + "RUSTDOC_REAL", + if cmd == "doc" || cmd == "rustdoc" || (cmd == "test" && want_rustdoc) { + self.rustdoc(compiler) + } else { + PathBuf::from("/path/to/nowhere/rustdoc/not/required") + }, + ) + .env("RUSTC_ERROR_METADATA_DST", self.extended_error_dir()) + .env("RUSTC_BREAK_ON_ICE", "1"); + // Clippy support is a hack and uses the default `cargo-clippy` in path. + // Don't override RUSTC so that the `cargo-clippy` in path will be run. + if cmd != "clippy" { + cargo.env("RUSTC", self.bootstrap_out.join("rustc")); + } + + // Dealing with rpath here is a little special, so let's go into some + // detail. First off, `-rpath` is a linker option on Unix platforms + // which adds to the runtime dynamic loader path when looking for + // dynamic libraries. We use this by default on Unix platforms to ensure + // that our nightlies behave the same on Windows, that is they work out + // of the box. This can be disabled by setting `rpath = false` in `[rust]` + // table of `config.toml` + // + // Ok, so the astute might be wondering "why isn't `-C rpath` used + // here?" and that is indeed a good question to ask. This codegen + // option is the compiler's current interface to generating an rpath. + // Unfortunately it doesn't quite suffice for us. The flag currently + // takes no value as an argument, so the compiler calculates what it + // should pass to the linker as `-rpath`. This unfortunately is based on + // the **compile time** directory structure which when building with + // Cargo will be very different than the runtime directory structure. + // + // All that's a really long winded way of saying that if we use + // `-Crpath` then the executables generated have the wrong rpath of + // something like `$ORIGIN/deps` when in fact the way we distribute + // rustc requires the rpath to be `$ORIGIN/../lib`. + // + // So, all in all, to set up the correct rpath we pass the linker + // argument manually via `-C link-args=-Wl,-rpath,...`. Plus isn't it + // fun to pass a flag to a tool to pass a flag to pass a flag to a tool + // to change a flag in a binary? + if self.config.rpath_enabled(target) && helpers::use_host_linker(target) { + let libdir = self.sysroot_libdir_relative(compiler).to_str().unwrap(); + let rpath = if target.contains("apple") { + // Note that we need to take one extra step on macOS to also pass + // `-Wl,-instal_name,@rpath/...` to get things to work right. To + // do that we pass a weird flag to the compiler to get it to do + // so. Note that this is definitely a hack, and we should likely + // flesh out rpath support more fully in the future. + rustflags.arg("-Zosx-rpath-install-name"); + Some(format!("-Wl,-rpath,@loader_path/../{libdir}")) + } else if !target.contains("windows") + && !target.contains("aix") + && !target.contains("xous") + { + rustflags.arg("-Clink-args=-Wl,-z,origin"); + Some(format!("-Wl,-rpath,$ORIGIN/../{libdir}")) + } else { + None + }; + if let Some(rpath) = rpath { + rustflags.arg(&format!("-Clink-args={rpath}")); + } + } + + if let Some(host_linker) = self.linker(compiler.host) { + hostflags.arg(format!("-Clinker={}", host_linker.display())); + } + if self.is_fuse_ld_lld(compiler.host) { + hostflags.arg("-Clink-args=-fuse-ld=lld"); + } + + if let Some(target_linker) = self.linker(target) { + let target = crate::envify(&target.triple); + cargo.env(&format!("CARGO_TARGET_{target}_LINKER"), target_linker); + } + if self.is_fuse_ld_lld(target) { + rustflags.arg("-Clink-args=-fuse-ld=lld"); + } + self.lld_flags(target).for_each(|flag| { + rustdocflags.arg(&flag); + }); + + if !(["build", "check", "clippy", "fix", "rustc"].contains(&cmd)) && want_rustdoc { + cargo.env("RUSTDOC_LIBDIR", self.rustc_libdir(compiler)); + } + + let debuginfo_level = match mode { + Mode::Rustc | Mode::Codegen => self.config.rust_debuginfo_level_rustc, + Mode::Std => self.config.rust_debuginfo_level_std, + Mode::ToolBootstrap | Mode::ToolStd | Mode::ToolRustc => { + self.config.rust_debuginfo_level_tools + } + }; + cargo.env(profile_var("DEBUG"), debuginfo_level.to_string()); + if let Some(opt_level) = &self.config.rust_optimize.get_opt_level() { + cargo.env(profile_var("OPT_LEVEL"), opt_level); + } + if !self.config.dry_run() && self.cc.borrow()[&target].args().iter().any(|arg| arg == "-gz") + { + rustflags.arg("-Clink-arg=-gz"); + } + cargo.env( + profile_var("DEBUG_ASSERTIONS"), + if mode == Mode::Std { + self.config.rust_debug_assertions_std.to_string() + } else { + self.config.rust_debug_assertions.to_string() + }, + ); + cargo.env( + profile_var("OVERFLOW_CHECKS"), + if mode == Mode::Std { + self.config.rust_overflow_checks_std.to_string() + } else { + self.config.rust_overflow_checks.to_string() + }, + ); + + let split_debuginfo_is_stable = target.contains("linux") + || target.contains("apple") + || (target.contains("msvc") + && self.config.rust_split_debuginfo == SplitDebuginfo::Packed) + || (target.contains("windows") + && self.config.rust_split_debuginfo == SplitDebuginfo::Off); + + if !split_debuginfo_is_stable { + rustflags.arg("-Zunstable-options"); + } + match self.config.rust_split_debuginfo { + SplitDebuginfo::Packed => rustflags.arg("-Csplit-debuginfo=packed"), + SplitDebuginfo::Unpacked => rustflags.arg("-Csplit-debuginfo=unpacked"), + SplitDebuginfo::Off => rustflags.arg("-Csplit-debuginfo=off"), + }; + + if self.config.cmd.bless() { + // Bless `expect!` tests. + cargo.env("UPDATE_EXPECT", "1"); + } + + if !mode.is_tool() { + cargo.env("RUSTC_FORCE_UNSTABLE", "1"); + } + + if let Some(x) = self.crt_static(target) { + if x { + rustflags.arg("-Ctarget-feature=+crt-static"); + } else { + rustflags.arg("-Ctarget-feature=-crt-static"); + } + } + + if let Some(x) = self.crt_static(compiler.host) { + let sign = if x { "+" } else { "-" }; + hostflags.arg(format!("-Ctarget-feature={sign}crt-static")); + } + + if let Some(map_to) = self.build.debuginfo_map_to(GitRepo::Rustc) { + let map = format!("{}={}", self.build.src.display(), map_to); + cargo.env("RUSTC_DEBUGINFO_MAP", map); + + // `rustc` needs to know the virtual `/rustc/$hash` we're mapping to, + // in order to opportunistically reverse it later. + cargo.env("CFG_VIRTUAL_RUST_SOURCE_BASE_DIR", map_to); + } + + if self.config.rust_remap_debuginfo { + // FIXME: handle vendored sources + let registry_src = t!(home::cargo_home()).join("registry").join("src"); + let mut env_var = OsString::new(); + for entry in t!(std::fs::read_dir(registry_src)) { + if !env_var.is_empty() { + env_var.push("\t"); + } + env_var.push(t!(entry).path()); + env_var.push("=/rust/deps"); + } + cargo.env("RUSTC_CARGO_REGISTRY_SRC_TO_REMAP", env_var); + } + + // Enable usage of unstable features + cargo.env("RUSTC_BOOTSTRAP", "1"); + self.add_rust_test_threads(&mut cargo); + + // Almost all of the crates that we compile as part of the bootstrap may + // have a build script, including the standard library. To compile a + // build script, however, it itself needs a standard library! This + // introduces a bit of a pickle when we're compiling the standard + // library itself. + // + // To work around this we actually end up using the snapshot compiler + // (stage0) for compiling build scripts of the standard library itself. + // The stage0 compiler is guaranteed to have a libstd available for use. + // + // For other crates, however, we know that we've already got a standard + // library up and running, so we can use the normal compiler to compile + // build scripts in that situation. + if mode == Mode::Std { + cargo + .env("RUSTC_SNAPSHOT", &self.initial_rustc) + .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_snapshot_libdir()); + } else { + cargo + .env("RUSTC_SNAPSHOT", self.rustc(compiler)) + .env("RUSTC_SNAPSHOT_LIBDIR", self.rustc_libdir(compiler)); + } + + // Tools that use compiler libraries may inherit the `-lLLVM` link + // requirement, but the `-L` library path is not propagated across + // separate Cargo projects. We can add LLVM's library path to the + // platform-specific environment variable as a workaround. + if mode == Mode::ToolRustc || mode == Mode::Codegen { + if let Some(llvm_config) = self.llvm_config(target) { + let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); + add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cargo); + } + } + + // Compile everything except libraries and proc macros with the more + // efficient initial-exec TLS model. This doesn't work with `dlopen`, + // so we can't use it by default in general, but we can use it for tools + // and our own internal libraries. + if !mode.must_support_dlopen() && !target.triple.starts_with("powerpc-") { + cargo.env("RUSTC_TLS_MODEL_INITIAL_EXEC", "1"); + } + + // Ignore incremental modes except for stage0, since we're + // not guaranteeing correctness across builds if the compiler + // is changing under your feet. + if self.config.incremental && compiler.stage == 0 { + cargo.env("CARGO_INCREMENTAL", "1"); + } else { + // Don't rely on any default setting for incr. comp. in Cargo + cargo.env("CARGO_INCREMENTAL", "0"); + } + + if let Some(ref on_fail) = self.config.on_fail { + cargo.env("RUSTC_ON_FAIL", on_fail); + } + + if self.config.print_step_timings { + cargo.env("RUSTC_PRINT_STEP_TIMINGS", "1"); + } + + if self.config.print_step_rusage { + cargo.env("RUSTC_PRINT_STEP_RUSAGE", "1"); + } + + if self.config.backtrace_on_ice { + cargo.env("RUSTC_BACKTRACE_ON_ICE", "1"); + } + + cargo.env("RUSTC_VERBOSE", self.verbosity.to_string()); + + // Downstream forks of the Rust compiler might want to use a custom libc to add support for + // targets that are not yet available upstream. Adding a patch to replace libc with a + // custom one would cause compilation errors though, because Cargo would interpret the + // custom libc as part of the workspace, and apply the check-cfg lints on it. + // + // The libc build script emits check-cfg flags only when this environment variable is set, + // so this line allows the use of custom libcs. + cargo.env("LIBC_CHECK_CFG", "1"); + + if source_type == SourceType::InTree { + let mut lint_flags = Vec::new(); + // When extending this list, add the new lints to the RUSTFLAGS of the + // build_bootstrap function of src/bootstrap/bootstrap.py as well as + // some code doesn't go through this `rustc` wrapper. + lint_flags.push("-Wrust_2018_idioms"); + lint_flags.push("-Wunused_lifetimes"); + lint_flags.push("-Wsemicolon_in_expressions_from_macros"); + + if self.config.deny_warnings { + lint_flags.push("-Dwarnings"); + rustdocflags.arg("-Dwarnings"); + } + + // This does not use RUSTFLAGS due to caching issues with Cargo. + // Clippy is treated as an "in tree" tool, but shares the same + // cache as other "submodule" tools. With these options set in + // RUSTFLAGS, that causes *every* shared dependency to be rebuilt. + // By injecting this into the rustc wrapper, this circumvents + // Cargo's fingerprint detection. This is fine because lint flags + // are always ignored in dependencies. Eventually this should be + // fixed via better support from Cargo. + cargo.env("RUSTC_LINT_FLAGS", lint_flags.join(" ")); + + rustdocflags.arg("-Wrustdoc::invalid_codeblock_attributes"); + } + + if mode == Mode::Rustc { + rustflags.arg("-Zunstable-options"); + rustflags.arg("-Wrustc::internal"); + } + + // Throughout the build Cargo can execute a number of build scripts + // compiling C/C++ code and we need to pass compilers, archivers, flags, etc + // obtained previously to those build scripts. + // Build scripts use either the `cc` crate or `configure/make` so we pass + // the options through environment variables that are fetched and understood by both. + // + // FIXME: the guard against msvc shouldn't need to be here + if target.contains("msvc") { + if let Some(ref cl) = self.config.llvm_clang_cl { + cargo.env("CC", cl).env("CXX", cl); + } + } else { + let ccache = self.config.ccache.as_ref(); + let ccacheify = |s: &Path| { + let ccache = match ccache { + Some(ref s) => s, + None => return s.display().to_string(), + }; + // FIXME: the cc-rs crate only recognizes the literal strings + // `ccache` and `sccache` when doing caching compilations, so we + // mirror that here. It should probably be fixed upstream to + // accept a new env var or otherwise work with custom ccache + // vars. + match &ccache[..] { + "ccache" | "sccache" => format!("{} {}", ccache, s.display()), + _ => s.display().to_string(), + } + }; + let triple_underscored = target.triple.replace("-", "_"); + let cc = ccacheify(&self.cc(target)); + cargo.env(format!("CC_{triple_underscored}"), &cc); + + let cflags = self.cflags(target, GitRepo::Rustc, CLang::C).join(" "); + cargo.env(format!("CFLAGS_{triple_underscored}"), &cflags); + + if let Some(ar) = self.ar(target) { + let ranlib = format!("{} s", ar.display()); + cargo + .env(format!("AR_{triple_underscored}"), ar) + .env(format!("RANLIB_{triple_underscored}"), ranlib); + } + + if let Ok(cxx) = self.cxx(target) { + let cxx = ccacheify(&cxx); + let cxxflags = self.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" "); + cargo + .env(format!("CXX_{triple_underscored}"), &cxx) + .env(format!("CXXFLAGS_{triple_underscored}"), cxxflags); + } + } + + // If Control Flow Guard is enabled, pass the `control-flow-guard` flag to rustc + // when compiling the standard library, since this might be linked into the final outputs + // produced by rustc. Since this mitigation is only available on Windows, only enable it + // for the standard library in case the compiler is run on a non-Windows platform. + // This is not needed for stage 0 artifacts because these will only be used for building + // the stage 1 compiler. + if cfg!(windows) + && mode == Mode::Std + && self.config.control_flow_guard + && compiler.stage >= 1 + { + rustflags.arg("-Ccontrol-flow-guard"); + } + + // For `cargo doc` invocations, make rustdoc print the Rust version into the docs + // This replaces spaces with tabs because RUSTDOCFLAGS does not + // support arguments with regular spaces. Hopefully someday Cargo will + // have space support. + let rust_version = self.rust_version().replace(' ', "\t"); + rustdocflags.arg("--crate-version").arg(&rust_version); + + // Environment variables *required* throughout the build + // + // FIXME: should update code to not require this env var + cargo.env("CFG_COMPILER_HOST_TRIPLE", target.triple); + + // Set this for all builds to make sure doc builds also get it. + cargo.env("CFG_RELEASE_CHANNEL", &self.config.channel); + + // This one's a bit tricky. As of the time of this writing the compiler + // links to the `winapi` crate on crates.io. This crate provides raw + // bindings to Windows system functions, sort of like libc does for + // Unix. This crate also, however, provides "import libraries" for the + // MinGW targets. There's an import library per dll in the windows + // distribution which is what's linked to. These custom import libraries + // are used because the winapi crate can reference Windows functions not + // present in the MinGW import libraries. + // + // For example MinGW may ship libdbghelp.a, but it may not have + // references to all the functions in the dbghelp dll. Instead the + // custom import library for dbghelp in the winapi crates has all this + // information. + // + // Unfortunately for us though the import libraries are linked by + // default via `-ldylib=winapi_foo`. That is, they're linked with the + // `dylib` type with a `winapi_` prefix (so the winapi ones don't + // conflict with the system MinGW ones). This consequently means that + // the binaries we ship of things like rustc_codegen_llvm (aka the rustc_codegen_llvm + // DLL) when linked against *again*, for example with procedural macros + // or plugins, will trigger the propagation logic of `-ldylib`, passing + // `-lwinapi_foo` to the linker again. This isn't actually available in + // our distribution, however, so the link fails. + // + // To solve this problem we tell winapi to not use its bundled import + // libraries. This means that it will link to the system MinGW import + // libraries by default, and the `-ldylib=foo` directives will still get + // passed to the final linker, but they'll look like `-lfoo` which can + // be resolved because MinGW has the import library. The downside is we + // don't get newer functions from Windows, but we don't use any of them + // anyway. + if !mode.is_tool() { + cargo.env("WINAPI_NO_BUNDLED_LIBRARIES", "1"); + } + + for _ in 0..self.verbosity { + cargo.arg("-v"); + } + + match (mode, self.config.rust_codegen_units_std, self.config.rust_codegen_units) { + (Mode::Std, Some(n), _) | (_, _, Some(n)) => { + cargo.env(profile_var("CODEGEN_UNITS"), n.to_string()); + } + _ => { + // Don't set anything + } + } + + if self.config.locked_deps { + cargo.arg("--locked"); + } + if self.config.vendor || self.is_sudo { + cargo.arg("--frozen"); + } + + // Try to use a sysroot-relative bindir, in case it was configured absolutely. + cargo.env("RUSTC_INSTALL_BINDIR", self.config.bindir_relative()); + + self.ci_env.force_coloring_in_ci(&mut cargo); + + // When we build Rust dylibs they're all intended for intermediate + // usage, so make sure we pass the -Cprefer-dynamic flag instead of + // linking all deps statically into the dylib. + if matches!(mode, Mode::Std | Mode::Rustc) { + rustflags.arg("-Cprefer-dynamic"); + } + + // When building incrementally we default to a lower ThinLTO import limit + // (unless explicitly specified otherwise). This will produce a somewhat + // slower code but give way better compile times. + { + let limit = match self.config.rust_thin_lto_import_instr_limit { + Some(limit) => Some(limit), + None if self.config.incremental => Some(10), + _ => None, + }; + + if let Some(limit) = limit { + if stage == 0 || self.config.default_codegen_backend().unwrap_or_default() == "llvm" + { + rustflags.arg(&format!("-Cllvm-args=-import-instr-limit={limit}")); + } + } + } + + if matches!(mode, Mode::Std) { + if let Some(mir_opt_level) = self.config.rust_validate_mir_opts { + rustflags.arg("-Zvalidate-mir"); + rustflags.arg(&format!("-Zmir-opt-level={mir_opt_level}")); + } + // Always enable inlining MIR when building the standard library. + // Without this flag, MIR inlining is disabled when incremental compilation is enabled. + // That causes some mir-opt tests which inline functions from the standard library to + // break when incremental compilation is enabled. So this overrides the "no inlining + // during incremental builds" heuristic for the standard library. + rustflags.arg("-Zinline-mir"); + } + + // set rustc args passed from command line + let rustc_args = + self.config.cmd.rustc_args().iter().map(|s| s.to_string()).collect::>(); + if !rustc_args.is_empty() { + cargo.env("RUSTFLAGS", &rustc_args.join(" ")); + } + + Cargo { command: cargo, rustflags, rustdocflags, hostflags, allow_features } + } + + /// Ensure that a given step is built, returning its output. This will + /// cache the step, so it is safe (and good!) to call this as often as + /// needed to ensure that all dependencies are built. + pub fn ensure(&'a self, step: S) -> S::Output { + { + let mut stack = self.stack.borrow_mut(); + for stack_step in stack.iter() { + // should skip + if stack_step.downcast_ref::().map_or(true, |stack_step| *stack_step != step) { + continue; + } + let mut out = String::new(); + out += &format!("\n\nCycle in build detected when adding {step:?}\n"); + for el in stack.iter().rev() { + out += &format!("\t{el:?}\n"); + } + panic!("{}", out); + } + if let Some(out) = self.cache.get(&step) { + self.verbose_than(1, &format!("{}c {:?}", " ".repeat(stack.len()), step)); + + return out; + } + self.verbose_than(1, &format!("{}> {:?}", " ".repeat(stack.len()), step)); + stack.push(Box::new(step.clone())); + } + + #[cfg(feature = "build-metrics")] + self.metrics.enter_step(&step, self); + + let (out, dur) = { + let start = Instant::now(); + let zero = Duration::new(0, 0); + let parent = self.time_spent_on_dependencies.replace(zero); + let out = step.clone().run(self); + let dur = start.elapsed(); + let deps = self.time_spent_on_dependencies.replace(parent + dur); + (out, dur - deps) + }; + + if self.config.print_step_timings && !self.config.dry_run() { + let step_string = format!("{step:?}"); + let brace_index = step_string.find("{").unwrap_or(0); + let type_string = type_name::(); + println!( + "[TIMING] {} {} -- {}.{:03}", + &type_string.strip_prefix("bootstrap::").unwrap_or(type_string), + &step_string[brace_index..], + dur.as_secs(), + dur.subsec_millis() + ); + } + + #[cfg(feature = "build-metrics")] + self.metrics.exit_step(self); + + { + let mut stack = self.stack.borrow_mut(); + let cur_step = stack.pop().expect("step stack empty"); + assert_eq!(cur_step.downcast_ref(), Some(&step)); + } + self.verbose_than(1, &format!("{}< {:?}", " ".repeat(self.stack.borrow().len()), step)); + self.cache.put(step, out.clone()); + out + } + + /// Ensure that a given step is built *only if it's supposed to be built by default*, returning + /// its output. This will cache the step, so it's safe (and good!) to call this as often as + /// needed to ensure that all dependencies are build. + pub(crate) fn ensure_if_default>>( + &'a self, + step: S, + kind: Kind, + ) -> S::Output { + let desc = StepDescription::from::(kind); + let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); + + // Avoid running steps contained in --skip + for pathset in &should_run.paths { + if desc.is_excluded(self, pathset) { + return None; + } + } + + // Only execute if it's supposed to run as default + if desc.default && should_run.is_really_default() { self.ensure(step) } else { None } + } + + /// Checks if any of the "should_run" paths is in the `Builder` paths. + pub(crate) fn was_invoked_explicitly(&'a self, kind: Kind) -> bool { + let desc = StepDescription::from::(kind); + let should_run = (desc.should_run)(ShouldRun::new(self, desc.kind)); + + for path in &self.paths { + if should_run.paths.iter().any(|s| s.has(path, desc.kind)) + && !desc.is_excluded( + self, + &PathSet::Suite(TaskPath { path: path.clone(), kind: Some(desc.kind) }), + ) + { + return true; + } + } + + false + } + + pub(crate) fn maybe_open_in_browser(&self, path: impl AsRef) { + if self.was_invoked_explicitly::(Kind::Doc) { + self.open_in_browser(path); + } + } + + pub(crate) fn open_in_browser(&self, path: impl AsRef) { + if self.config.dry_run() || !self.config.cmd.open() { + return; + } + + let path = path.as_ref(); + self.info(&format!("Opening doc {}", path.display())); + if let Err(err) = opener::open(path) { + self.info(&format!("{err}\n")); + } + } +} + +/// Represents flag values in `String` form with whitespace delimiter to pass it to the compiler later. +/// +/// `-Z crate-attr` flags will be applied recursively on the target code using the `rustc_parse::parser::Parser`. +/// See `rustc_builtin_macros::cmdline_attrs::inject` for more information. +#[derive(Debug, Clone)] +struct Rustflags(String, TargetSelection); + +impl Rustflags { + fn new(target: TargetSelection) -> Rustflags { + let mut ret = Rustflags(String::new(), target); + ret.propagate_cargo_env("RUSTFLAGS"); + ret + } + + /// By default, cargo will pick up on various variables in the environment. However, bootstrap + /// reuses those variables to pass additional flags to rustdoc, so by default they get overridden. + /// Explicitly add back any previous value in the environment. + /// + /// `prefix` is usually `RUSTFLAGS` or `RUSTDOCFLAGS`. + fn propagate_cargo_env(&mut self, prefix: &str) { + // Inherit `RUSTFLAGS` by default ... + self.env(prefix); + + // ... and also handle target-specific env RUSTFLAGS if they're configured. + let target_specific = format!("CARGO_TARGET_{}_{}", crate::envify(&self.1.triple), prefix); + self.env(&target_specific); + } + + fn env(&mut self, env: &str) { + if let Ok(s) = env::var(env) { + for part in s.split(' ') { + self.arg(part); + } + } + } + + fn arg(&mut self, arg: &str) -> &mut Self { + assert_eq!(arg.split(' ').count(), 1); + if !self.0.is_empty() { + self.0.push(' '); + } + self.0.push_str(arg); + self + } +} + +/// Flags that are passed to the `rustc` shim binary. +/// These flags will only be applied when compiling host code, i.e. when +/// `--target` is unset. +#[derive(Debug, Default)] +pub struct HostFlags { + rustc: Vec, +} + +impl HostFlags { + const SEPARATOR: &'static str = " "; + + /// Adds a host rustc flag. + fn arg>(&mut self, flag: S) { + let value = flag.into().trim().to_string(); + assert!(!value.contains(Self::SEPARATOR)); + self.rustc.push(value); + } + + /// Encodes all the flags into a single string. + fn encode(self) -> String { + self.rustc.join(Self::SEPARATOR) + } +} + +#[derive(Debug)] +pub struct Cargo { + command: Command, + rustflags: Rustflags, + rustdocflags: Rustflags, + hostflags: HostFlags, + allow_features: String, +} + +impl Cargo { + pub fn rustdocflag(&mut self, arg: &str) -> &mut Cargo { + self.rustdocflags.arg(arg); + self + } + pub fn rustflag(&mut self, arg: &str) -> &mut Cargo { + self.rustflags.arg(arg); + self + } + + pub fn arg(&mut self, arg: impl AsRef) -> &mut Cargo { + self.command.arg(arg.as_ref()); + self + } + + pub fn args(&mut self, args: I) -> &mut Cargo + where + I: IntoIterator, + S: AsRef, + { + for arg in args { + self.arg(arg.as_ref()); + } + self + } + + pub fn env(&mut self, key: impl AsRef, value: impl AsRef) -> &mut Cargo { + // These are managed through rustflag/rustdocflag interfaces. + assert_ne!(key.as_ref(), "RUSTFLAGS"); + assert_ne!(key.as_ref(), "RUSTDOCFLAGS"); + self.command.env(key.as_ref(), value.as_ref()); + self + } + + pub fn add_rustc_lib_path(&mut self, builder: &Builder<'_>, compiler: Compiler) { + builder.add_rustc_lib_path(compiler, &mut self.command); + } + + pub fn current_dir(&mut self, dir: &Path) -> &mut Cargo { + self.command.current_dir(dir); + self + } + + /// Adds nightly-only features that this invocation is allowed to use. + /// + /// By default, all nightly features are allowed. Once this is called, it + /// will be restricted to the given set. + pub fn allow_features(&mut self, features: &str) -> &mut Cargo { + if !self.allow_features.is_empty() { + self.allow_features.push(','); + } + self.allow_features.push_str(features); + self + } +} + +impl From for Command { + fn from(mut cargo: Cargo) -> Command { + let rustflags = &cargo.rustflags.0; + if !rustflags.is_empty() { + cargo.command.env("RUSTFLAGS", rustflags); + } + + let rustdocflags = &cargo.rustdocflags.0; + if !rustdocflags.is_empty() { + cargo.command.env("RUSTDOCFLAGS", rustdocflags); + } + + let encoded_hostflags = cargo.hostflags.encode(); + if !encoded_hostflags.is_empty() { + cargo.command.env("RUSTC_HOST_FLAGS", encoded_hostflags); + } + + if !cargo.allow_features.is_empty() { + cargo.command.env("RUSTC_ALLOW_FEATURES", cargo.allow_features); + } + + cargo.command + } +} diff --git a/src/bootstrap/src/core/config/config.rs b/src/bootstrap/src/core/config/config.rs new file mode 100644 index 000000000..0a9175aa3 --- /dev/null +++ b/src/bootstrap/src/core/config/config.rs @@ -0,0 +1,2208 @@ +//! Serialized configuration of a build. +//! +//! This module implements parsing `config.toml` configuration files to tweak +//! how the build runs. + +#[cfg(test)] +#[path = "../../tests/config.rs"] +mod tests; + +use std::cell::{Cell, RefCell}; +use std::cmp; +use std::collections::{HashMap, HashSet}; +use std::env; +use std::fmt::{self, Display}; +use std::fs; +use std::io::IsTerminal; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::str::FromStr; + +use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX; +use crate::core::build_steps::llvm; +use crate::core::config::flags::{Color, Flags, Warnings}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::channel::{self, GitInfo}; +use crate::utils::helpers::{exe, output, t}; +use build_helper::exit; +use once_cell::sync::OnceCell; +use semver::Version; +use serde::{Deserialize, Deserializer}; +use serde_derive::Deserialize; + +pub use crate::core::config::flags::Subcommand; +use build_helper::git::GitConfig; + +macro_rules! check_ci_llvm { + ($name:expr) => { + assert!( + $name.is_none(), + "setting {} is incompatible with download-ci-llvm.", + stringify!($name) + ); + }; +} + +#[derive(Clone, Default)] +pub enum DryRun { + /// This isn't a dry run. + #[default] + Disabled, + /// This is a dry run enabled by bootstrap itself, so it can verify that no work is done. + SelfCheck, + /// This is a dry run enabled by the `--dry-run` flag. + UserSelected, +} + +#[derive(Copy, Clone, Default, PartialEq, Eq)] +pub enum DebuginfoLevel { + #[default] + None, + LineTablesOnly, + Limited, + Full, +} + +// NOTE: can't derive(Deserialize) because the intermediate trip through toml::Value only +// deserializes i64, and derive() only generates visit_u64 +impl<'de> Deserialize<'de> for DebuginfoLevel { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + use serde::de::Error; + + Ok(match Deserialize::deserialize(deserializer)? { + StringOrInt::String("none") | StringOrInt::Int(0) => DebuginfoLevel::None, + StringOrInt::String("line-tables-only") => DebuginfoLevel::LineTablesOnly, + StringOrInt::String("limited") | StringOrInt::Int(1) => DebuginfoLevel::Limited, + StringOrInt::String("full") | StringOrInt::Int(2) => DebuginfoLevel::Full, + StringOrInt::Int(n) => { + let other = serde::de::Unexpected::Signed(n); + return Err(D::Error::invalid_value(other, &"expected 0, 1, or 2")); + } + StringOrInt::String(s) => { + let other = serde::de::Unexpected::Str(s); + return Err(D::Error::invalid_value( + other, + &"expected none, line-tables-only, limited, or full", + )); + } + }) + } +} + +/// Suitable for passing to `-C debuginfo` +impl Display for DebuginfoLevel { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + use DebuginfoLevel::*; + f.write_str(match self { + None => "0", + LineTablesOnly => "line-tables-only", + Limited => "1", + Full => "2", + }) + } +} + +/// Global configuration for the entire build and/or bootstrap. +/// +/// This structure is parsed from `config.toml`, and some of the fields are inferred from `git` or build-time parameters. +/// +/// Note that this structure is not decoded directly into, but rather it is +/// filled out from the decoded forms of the structs below. For documentation +/// each field, see the corresponding fields in +/// `config.example.toml`. +#[derive(Default, Clone)] +pub struct Config { + pub changelog_seen: Option, // FIXME: Deprecated field. Remove it at 2024. + pub change_id: Option, + pub ccache: Option, + /// Call Build::ninja() instead of this. + pub ninja_in_file: bool, + pub verbose: usize, + pub submodules: Option, + pub compiler_docs: bool, + pub library_docs_private_items: bool, + pub docs_minification: bool, + pub docs: bool, + pub locked_deps: bool, + pub vendor: bool, + pub target_config: HashMap, + pub full_bootstrap: bool, + pub extended: bool, + pub tools: Option>, + pub sanitizers: bool, + pub profiler: bool, + pub omit_git_hash: bool, + pub skip: Vec, + pub include_default_paths: bool, + pub rustc_error_format: Option, + pub json_output: bool, + pub test_compare_mode: bool, + pub color: Color, + pub patch_binaries_for_nix: Option, + pub stage0_metadata: Stage0Metadata, + pub android_ndk: Option, + + pub stdout_is_tty: bool, + pub stderr_is_tty: bool, + + pub on_fail: Option, + pub stage: u32, + pub keep_stage: Vec, + pub keep_stage_std: Vec, + pub src: PathBuf, + /// defaults to `config.toml` + pub config: Option, + pub jobs: Option, + pub cmd: Subcommand, + pub incremental: bool, + pub dry_run: DryRun, + /// Arguments appearing after `--` to be forwarded to tools, + /// e.g. `--fix-broken` or test arguments. + pub free_args: Vec, + + /// `None` if we shouldn't download CI compiler artifacts, or the commit to download if we should. + #[cfg(not(test))] + download_rustc_commit: Option, + #[cfg(test)] + pub download_rustc_commit: Option, + + pub deny_warnings: bool, + pub backtrace_on_ice: bool, + + // llvm codegen options + pub llvm_assertions: bool, + pub llvm_tests: bool, + pub llvm_plugins: bool, + pub llvm_optimize: bool, + pub llvm_thin_lto: bool, + pub llvm_release_debuginfo: bool, + pub llvm_static_stdcpp: bool, + /// `None` if `llvm_from_ci` is true and we haven't yet downloaded llvm. + #[cfg(not(test))] + llvm_link_shared: Cell>, + #[cfg(test)] + pub llvm_link_shared: Cell>, + pub llvm_clang_cl: Option, + pub llvm_targets: Option, + pub llvm_experimental_targets: Option, + pub llvm_link_jobs: Option, + pub llvm_version_suffix: Option, + pub llvm_use_linker: Option, + pub llvm_allow_old_toolchain: bool, + pub llvm_polly: bool, + pub llvm_clang: bool, + pub llvm_enable_warnings: bool, + pub llvm_from_ci: bool, + pub llvm_build_config: HashMap, + + pub use_lld: bool, + pub lld_enabled: bool, + pub llvm_tools_enabled: bool, + + pub llvm_cflags: Option, + pub llvm_cxxflags: Option, + pub llvm_ldflags: Option, + pub llvm_use_libcxx: bool, + + // rust codegen options + pub rust_optimize: RustOptimize, + pub rust_codegen_units: Option, + pub rust_codegen_units_std: Option, + pub rust_debug_assertions: bool, + pub rust_debug_assertions_std: bool, + pub rust_overflow_checks: bool, + pub rust_overflow_checks_std: bool, + pub rust_debug_logging: bool, + pub rust_debuginfo_level_rustc: DebuginfoLevel, + pub rust_debuginfo_level_std: DebuginfoLevel, + pub rust_debuginfo_level_tools: DebuginfoLevel, + pub rust_debuginfo_level_tests: DebuginfoLevel, + pub rust_split_debuginfo: SplitDebuginfo, + pub rust_rpath: bool, + pub rustc_parallel: bool, + pub rustc_default_linker: Option, + pub rust_optimize_tests: bool, + pub rust_dist_src: bool, + pub rust_codegen_backends: Vec>, + pub rust_verify_llvm_ir: bool, + pub rust_thin_lto_import_instr_limit: Option, + pub rust_remap_debuginfo: bool, + pub rust_new_symbol_mangling: Option, + pub rust_profile_use: Option, + pub rust_profile_generate: Option, + pub rust_lto: RustcLto, + pub rust_validate_mir_opts: Option, + pub llvm_profile_use: Option, + pub llvm_profile_generate: bool, + pub llvm_libunwind_default: Option, + pub enable_bolt_settings: bool, + + pub reproducible_artifacts: Vec, + + pub build: TargetSelection, + pub hosts: Vec, + pub targets: Vec, + pub local_rebuild: bool, + pub jemalloc: bool, + pub control_flow_guard: bool, + + // dist misc + pub dist_sign_folder: Option, + pub dist_upload_addr: Option, + pub dist_compression_formats: Option>, + pub dist_compression_profile: String, + pub dist_include_mingw_linker: bool, + + // libstd features + pub backtrace: bool, // support for RUST_BACKTRACE + + // misc + pub low_priority: bool, + pub channel: String, + pub description: Option, + pub verbose_tests: bool, + pub save_toolstates: Option, + pub print_step_timings: bool, + pub print_step_rusage: bool, + pub missing_tools: bool, + + // Fallback musl-root for all targets + pub musl_root: Option, + pub prefix: Option, + pub sysconfdir: Option, + pub datadir: Option, + pub docdir: Option, + pub bindir: PathBuf, + pub libdir: Option, + pub mandir: Option, + pub codegen_tests: bool, + pub nodejs: Option, + pub npm: Option, + pub gdb: Option, + pub python: Option, + pub reuse: Option, + pub cargo_native_static: bool, + pub configure_args: Vec, + pub out: PathBuf, + pub rust_info: channel::GitInfo, + + // These are either the stage0 downloaded binaries or the locally installed ones. + pub initial_cargo: PathBuf, + pub initial_rustc: PathBuf, + + #[cfg(not(test))] + initial_rustfmt: RefCell, + #[cfg(test)] + pub initial_rustfmt: RefCell, + + pub paths: Vec, +} + +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Metadata { + pub compiler: CompilerMetadata, + pub config: Stage0Config, + pub checksums_sha256: HashMap, + pub rustfmt: Option, +} +#[derive(Default, Deserialize, Clone)] +pub struct CompilerMetadata { + pub date: String, + pub version: String, +} + +#[derive(Default, Deserialize, Clone)] +pub struct Stage0Config { + pub dist_server: String, + pub artifacts_server: String, + pub artifacts_with_llvm_assertions_server: String, + pub git_merge_commit_email: String, + pub git_repository: String, + pub nightly_branch: String, +} +#[derive(Default, Deserialize, Clone)] +pub struct RustfmtMetadata { + pub date: String, + pub version: String, +} + +#[derive(Clone, Debug, Default)] +pub enum RustfmtState { + SystemToolchain(PathBuf), + Downloaded(PathBuf), + Unavailable, + #[default] + LazyEvaluated, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq)] +pub enum LlvmLibunwind { + #[default] + No, + InTree, + System, +} + +impl FromStr for LlvmLibunwind { + type Err = String; + + fn from_str(value: &str) -> Result { + match value { + "no" => Ok(Self::No), + "in-tree" => Ok(Self::InTree), + "system" => Ok(Self::System), + invalid => Err(format!("Invalid value '{invalid}' for rust.llvm-libunwind config.")), + } + } +} + +#[derive(Default, Copy, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub enum SplitDebuginfo { + Packed, + Unpacked, + #[default] + Off, +} + +impl std::str::FromStr for SplitDebuginfo { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "packed" => Ok(SplitDebuginfo::Packed), + "unpacked" => Ok(SplitDebuginfo::Unpacked), + "off" => Ok(SplitDebuginfo::Off), + _ => Err(()), + } + } +} + +impl SplitDebuginfo { + /// Returns the default `-Csplit-debuginfo` value for the current target. See the comment for + /// `rust.split-debuginfo` in `config.example.toml`. + fn default_for_platform(target: &str) -> Self { + if target.contains("apple") { + SplitDebuginfo::Unpacked + } else if target.contains("windows") { + SplitDebuginfo::Packed + } else { + SplitDebuginfo::Off + } + } +} + +/// LTO mode used for compiling rustc itself. +#[derive(Default, Clone, PartialEq, Debug)] +pub enum RustcLto { + Off, + #[default] + ThinLocal, + Thin, + Fat, +} + +impl std::str::FromStr for RustcLto { + type Err = String; + + fn from_str(s: &str) -> Result { + match s { + "thin-local" => Ok(RustcLto::ThinLocal), + "thin" => Ok(RustcLto::Thin), + "fat" => Ok(RustcLto::Fat), + "off" => Ok(RustcLto::Off), + _ => Err(format!("Invalid value for rustc LTO: {s}")), + } + } +} + +#[derive(Copy, Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash)] +pub struct TargetSelection { + pub triple: Interned, + file: Option>, + synthetic: bool, +} + +/// Newtype over `Vec` so we can implement custom parsing logic +#[derive(Clone, Default, PartialEq, Eq, PartialOrd, Ord, Hash, Debug)] +pub struct TargetSelectionList(Vec); + +pub fn target_selection_list(s: &str) -> Result { + Ok(TargetSelectionList( + s.split(",").filter(|s| !s.is_empty()).map(TargetSelection::from_user).collect(), + )) +} + +impl TargetSelection { + pub fn from_user(selection: &str) -> Self { + let path = Path::new(selection); + + let (triple, file) = if path.exists() { + let triple = path + .file_stem() + .expect("Target specification file has no file stem") + .to_str() + .expect("Target specification file stem is not UTF-8"); + + (triple, Some(selection)) + } else { + (selection, None) + }; + + let triple = INTERNER.intern_str(triple); + let file = file.map(|f| INTERNER.intern_str(f)); + + Self { triple, file, synthetic: false } + } + + pub fn create_synthetic(triple: &str, file: &str) -> Self { + Self { + triple: INTERNER.intern_str(triple), + file: Some(INTERNER.intern_str(file)), + synthetic: true, + } + } + + pub fn rustc_target_arg(&self) -> &str { + self.file.as_ref().unwrap_or(&self.triple) + } + + pub fn contains(&self, needle: &str) -> bool { + self.triple.contains(needle) + } + + pub fn starts_with(&self, needle: &str) -> bool { + self.triple.starts_with(needle) + } + + pub fn ends_with(&self, needle: &str) -> bool { + self.triple.ends_with(needle) + } + + // See src/bootstrap/synthetic_targets.rs + pub fn is_synthetic(&self) -> bool { + self.synthetic + } +} + +impl fmt::Display for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{}", self.triple)?; + if let Some(file) = self.file { + write!(f, "({file})")?; + } + Ok(()) + } +} + +impl fmt::Debug for TargetSelection { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{self}") + } +} + +impl PartialEq<&str> for TargetSelection { + fn eq(&self, other: &&str) -> bool { + self.triple == *other + } +} + +/// Per-target configuration stored in the global configuration structure. +#[derive(Default, Clone)] +pub struct Target { + /// Some(path to llvm-config) if using an external LLVM. + pub llvm_config: Option, + pub llvm_has_rust_patches: Option, + /// Some(path to FileCheck) if one was specified. + pub llvm_filecheck: Option, + pub llvm_libunwind: Option, + pub cc: Option, + pub cxx: Option, + pub ar: Option, + pub ranlib: Option, + pub default_linker: Option, + pub linker: Option, + pub sanitizers: Option, + pub profiler: Option, + pub rpath: Option, + pub crt_static: Option, + pub musl_root: Option, + pub musl_libdir: Option, + pub wasi_root: Option, + pub qemu_rootfs: Option, + pub no_std: bool, +} + +impl Target { + pub fn from_triple(triple: &str) -> Self { + let mut target: Self = Default::default(); + if triple.contains("-none") || triple.contains("nvptx") || triple.contains("switch") { + target.no_std = true; + } + target + } +} +/// Structure of the `config.toml` file that configuration is read from. +/// +/// This structure uses `Decodable` to automatically decode a TOML configuration +/// file into this format, and then this is traversed and written into the above +/// `Config` structure. +#[derive(Deserialize, Default)] +#[serde(deny_unknown_fields, rename_all = "kebab-case")] +pub(crate) struct TomlConfig { + changelog_seen: Option, // FIXME: Deprecated field. Remove it at 2024. + change_id: Option, + build: Option, + install: Option, + llvm: Option, + rust: Option, + target: Option>, + dist: Option, + profile: Option, +} + +/// Describes how to handle conflicts in merging two [`TomlConfig`] +#[derive(Copy, Clone, Debug)] +enum ReplaceOpt { + /// Silently ignore a duplicated value + IgnoreDuplicate, + /// Override the current value, even if it's `Some` + Override, + /// Exit with an error on duplicate values + ErrorOnDuplicate, +} + +trait Merge { + fn merge(&mut self, other: Self, replace: ReplaceOpt); +} + +impl Merge for TomlConfig { + fn merge( + &mut self, + TomlConfig { + build, + install, + llvm, + rust, + dist, + target, + profile: _, + changelog_seen, + change_id, + }: Self, + replace: ReplaceOpt, + ) { + fn do_merge(x: &mut Option, y: Option, replace: ReplaceOpt) { + if let Some(new) = y { + if let Some(original) = x { + original.merge(new, replace); + } else { + *x = Some(new); + } + } + } + self.changelog_seen.merge(changelog_seen, replace); + self.change_id.merge(change_id, replace); + do_merge(&mut self.build, build, replace); + do_merge(&mut self.install, install, replace); + do_merge(&mut self.llvm, llvm, replace); + do_merge(&mut self.rust, rust, replace); + do_merge(&mut self.dist, dist, replace); + assert!(target.is_none(), "merging target-specific config is not currently supported"); + } +} + +// We are using a decl macro instead of a derive proc macro here to reduce the compile time of +// rustbuild. +macro_rules! define_config { + ($(#[$attr:meta])* struct $name:ident { + $($field:ident: Option<$field_ty:ty> = $field_key:literal,)* + }) => { + $(#[$attr])* + struct $name { + $($field: Option<$field_ty>,)* + } + + impl Merge for $name { + fn merge(&mut self, other: Self, replace: ReplaceOpt) { + $( + match replace { + ReplaceOpt::IgnoreDuplicate => { + if self.$field.is_none() { + self.$field = other.$field; + } + }, + ReplaceOpt::Override => { + if other.$field.is_some() { + self.$field = other.$field; + } + } + ReplaceOpt::ErrorOnDuplicate => { + if other.$field.is_some() { + if self.$field.is_some() { + if cfg!(test) { + panic!("overriding existing option") + } else { + eprintln!("overriding existing option: `{}`", stringify!($field)); + exit!(2); + } + } else { + self.$field = other.$field; + } + } + } + } + )* + } + } + + // The following is a trimmed version of what serde_derive generates. All parts not relevant + // for toml deserialization have been removed. This reduces the binary size and improves + // compile time of rustbuild. + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + struct Field; + impl<'de> serde::de::Visitor<'de> for Field { + type Value = $name; + fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(concat!("struct ", stringify!($name))) + } + + #[inline] + fn visit_map(self, mut map: A) -> Result + where + A: serde::de::MapAccess<'de>, + { + $(let mut $field: Option<$field_ty> = None;)* + while let Some(key) = + match serde::de::MapAccess::next_key::(&mut map) { + Ok(val) => val, + Err(err) => { + return Err(err); + } + } + { + match &*key { + $($field_key => { + if $field.is_some() { + return Err(::duplicate_field( + $field_key, + )); + } + $field = match serde::de::MapAccess::next_value::<$field_ty>( + &mut map, + ) { + Ok(val) => Some(val), + Err(err) => { + return Err(err); + } + }; + })* + key => { + return Err(serde::de::Error::unknown_field(key, FIELDS)); + } + } + } + Ok($name { $($field),* }) + } + } + const FIELDS: &'static [&'static str] = &[ + $($field_key,)* + ]; + Deserializer::deserialize_struct( + deserializer, + stringify!($name), + FIELDS, + Field, + ) + } + } + } +} + +impl Merge for Option { + fn merge(&mut self, other: Self, replace: ReplaceOpt) { + match replace { + ReplaceOpt::IgnoreDuplicate => { + if self.is_none() { + *self = other; + } + } + ReplaceOpt::Override => { + if other.is_some() { + *self = other; + } + } + ReplaceOpt::ErrorOnDuplicate => { + if other.is_some() { + if self.is_some() { + if cfg!(test) { + panic!("overriding existing option") + } else { + eprintln!("overriding existing option"); + exit!(2); + } + } else { + *self = other; + } + } + } + } + } +} + +define_config! { + /// TOML representation of various global build decisions. + #[derive(Default)] + struct Build { + build: Option = "build", + host: Option> = "host", + target: Option> = "target", + build_dir: Option = "build-dir", + cargo: Option = "cargo", + rustc: Option = "rustc", + rustfmt: Option = "rustfmt", + docs: Option = "docs", + compiler_docs: Option = "compiler-docs", + library_docs_private_items: Option = "library-docs-private-items", + docs_minification: Option = "docs-minification", + submodules: Option = "submodules", + gdb: Option = "gdb", + nodejs: Option = "nodejs", + npm: Option = "npm", + python: Option = "python", + reuse: Option = "reuse", + locked_deps: Option = "locked-deps", + vendor: Option = "vendor", + full_bootstrap: Option = "full-bootstrap", + extended: Option = "extended", + tools: Option> = "tools", + verbose: Option = "verbose", + sanitizers: Option = "sanitizers", + profiler: Option = "profiler", + cargo_native_static: Option = "cargo-native-static", + low_priority: Option = "low-priority", + configure_args: Option> = "configure-args", + local_rebuild: Option = "local-rebuild", + print_step_timings: Option = "print-step-timings", + print_step_rusage: Option = "print-step-rusage", + check_stage: Option = "check-stage", + doc_stage: Option = "doc-stage", + build_stage: Option = "build-stage", + test_stage: Option = "test-stage", + install_stage: Option = "install-stage", + dist_stage: Option = "dist-stage", + bench_stage: Option = "bench-stage", + patch_binaries_for_nix: Option = "patch-binaries-for-nix", + // NOTE: only parsed by bootstrap.py, `--feature build-metrics` enables metrics unconditionally + metrics: Option = "metrics", + android_ndk: Option = "android-ndk", + } +} + +define_config! { + /// TOML representation of various global install decisions. + struct Install { + prefix: Option = "prefix", + sysconfdir: Option = "sysconfdir", + docdir: Option = "docdir", + bindir: Option = "bindir", + libdir: Option = "libdir", + mandir: Option = "mandir", + datadir: Option = "datadir", + } +} + +define_config! { + /// TOML representation of how the LLVM build is configured. + struct Llvm { + optimize: Option = "optimize", + thin_lto: Option = "thin-lto", + release_debuginfo: Option = "release-debuginfo", + assertions: Option = "assertions", + tests: Option = "tests", + plugins: Option = "plugins", + ccache: Option = "ccache", + static_libstdcpp: Option = "static-libstdcpp", + ninja: Option = "ninja", + targets: Option = "targets", + experimental_targets: Option = "experimental-targets", + link_jobs: Option = "link-jobs", + link_shared: Option = "link-shared", + version_suffix: Option = "version-suffix", + clang_cl: Option = "clang-cl", + cflags: Option = "cflags", + cxxflags: Option = "cxxflags", + ldflags: Option = "ldflags", + use_libcxx: Option = "use-libcxx", + use_linker: Option = "use-linker", + allow_old_toolchain: Option = "allow-old-toolchain", + polly: Option = "polly", + clang: Option = "clang", + enable_warnings: Option = "enable-warnings", + download_ci_llvm: Option = "download-ci-llvm", + build_config: Option> = "build-config", + } +} + +define_config! { + struct Dist { + sign_folder: Option = "sign-folder", + gpg_password_file: Option = "gpg-password-file", + upload_addr: Option = "upload-addr", + src_tarball: Option = "src-tarball", + missing_tools: Option = "missing-tools", + compression_formats: Option> = "compression-formats", + compression_profile: Option = "compression-profile", + include_mingw_linker: Option = "include-mingw-linker", + } +} + +#[derive(Clone, Debug, Deserialize)] +#[serde(untagged)] +pub enum StringOrBool { + String(String), + Bool(bool), +} + +impl Default for StringOrBool { + fn default() -> StringOrBool { + StringOrBool::Bool(false) + } +} + +impl StringOrBool { + fn is_string_or_true(&self) -> bool { + matches!(self, Self::String(_) | Self::Bool(true)) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum RustOptimize { + String(String), + Int(u8), + Bool(bool), +} + +impl Default for RustOptimize { + fn default() -> RustOptimize { + RustOptimize::Bool(false) + } +} + +impl<'de> Deserialize<'de> for RustOptimize { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + deserializer.deserialize_any(OptimizeVisitor) + } +} + +struct OptimizeVisitor; + +impl<'de> serde::de::Visitor<'de> for OptimizeVisitor { + type Value = RustOptimize; + + fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter.write_str(r#"one of: 0, 1, 2, 3, "s", "z", true, false"#) + } + + fn visit_str(self, value: &str) -> Result + where + E: serde::de::Error, + { + if ["s", "z"].iter().find(|x| **x == value).is_some() { + Ok(RustOptimize::String(value.to_string())) + } else { + Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) + } + } + + fn visit_i64(self, value: i64) -> Result + where + E: serde::de::Error, + { + if matches!(value, 0..=3) { + Ok(RustOptimize::Int(value as u8)) + } else { + Err(format_optimize_error_msg(value)).map_err(serde::de::Error::custom) + } + } + + fn visit_bool(self, value: bool) -> Result + where + E: serde::de::Error, + { + Ok(RustOptimize::Bool(value)) + } +} + +fn format_optimize_error_msg(v: impl std::fmt::Display) -> String { + format!( + r#"unrecognized option for rust optimize: "{v}", expected one of 0, 1, 2, 3, "s", "z", true, false"# + ) +} + +impl RustOptimize { + pub(crate) fn is_release(&self) -> bool { + match &self { + RustOptimize::Bool(true) | RustOptimize::String(_) => true, + RustOptimize::Int(i) => *i > 0, + RustOptimize::Bool(false) => false, + } + } + + pub(crate) fn get_opt_level(&self) -> Option { + match &self { + RustOptimize::String(s) => Some(s.clone()), + RustOptimize::Int(i) => Some(i.to_string()), + RustOptimize::Bool(_) => None, + } + } +} + +#[derive(Deserialize)] +#[serde(untagged)] +enum StringOrInt<'a> { + String(&'a str), + Int(i64), +} +define_config! { + /// TOML representation of how the Rust build is configured. + struct Rust { + optimize: Option = "optimize", + debug: Option = "debug", + codegen_units: Option = "codegen-units", + codegen_units_std: Option = "codegen-units-std", + debug_assertions: Option = "debug-assertions", + debug_assertions_std: Option = "debug-assertions-std", + overflow_checks: Option = "overflow-checks", + overflow_checks_std: Option = "overflow-checks-std", + debug_logging: Option = "debug-logging", + debuginfo_level: Option = "debuginfo-level", + debuginfo_level_rustc: Option = "debuginfo-level-rustc", + debuginfo_level_std: Option = "debuginfo-level-std", + debuginfo_level_tools: Option = "debuginfo-level-tools", + debuginfo_level_tests: Option = "debuginfo-level-tests", + split_debuginfo: Option = "split-debuginfo", + run_dsymutil: Option = "run-dsymutil", + backtrace: Option = "backtrace", + incremental: Option = "incremental", + parallel_compiler: Option = "parallel-compiler", + default_linker: Option = "default-linker", + channel: Option = "channel", + description: Option = "description", + musl_root: Option = "musl-root", + rpath: Option = "rpath", + verbose_tests: Option = "verbose-tests", + optimize_tests: Option = "optimize-tests", + codegen_tests: Option = "codegen-tests", + omit_git_hash: Option = "omit-git-hash", + dist_src: Option = "dist-src", + save_toolstates: Option = "save-toolstates", + codegen_backends: Option> = "codegen-backends", + lld: Option = "lld", + use_lld: Option = "use-lld", + llvm_tools: Option = "llvm-tools", + deny_warnings: Option = "deny-warnings", + backtrace_on_ice: Option = "backtrace-on-ice", + verify_llvm_ir: Option = "verify-llvm-ir", + thin_lto_import_instr_limit: Option = "thin-lto-import-instr-limit", + remap_debuginfo: Option = "remap-debuginfo", + jemalloc: Option = "jemalloc", + test_compare_mode: Option = "test-compare-mode", + llvm_libunwind: Option = "llvm-libunwind", + control_flow_guard: Option = "control-flow-guard", + new_symbol_mangling: Option = "new-symbol-mangling", + profile_generate: Option = "profile-generate", + profile_use: Option = "profile-use", + // ignored; this is set from an env var set by bootstrap.py + download_rustc: Option = "download-rustc", + lto: Option = "lto", + validate_mir_opts: Option = "validate-mir-opts", + } +} + +define_config! { + /// TOML representation of how each build target is configured. + struct TomlTarget { + cc: Option = "cc", + cxx: Option = "cxx", + ar: Option = "ar", + ranlib: Option = "ranlib", + default_linker: Option = "default-linker", + linker: Option = "linker", + llvm_config: Option = "llvm-config", + llvm_has_rust_patches: Option = "llvm-has-rust-patches", + llvm_filecheck: Option = "llvm-filecheck", + llvm_libunwind: Option = "llvm-libunwind", + sanitizers: Option = "sanitizers", + profiler: Option = "profiler", + rpath: Option = "rpath", + crt_static: Option = "crt-static", + musl_root: Option = "musl-root", + musl_libdir: Option = "musl-libdir", + wasi_root: Option = "wasi-root", + qemu_rootfs: Option = "qemu-rootfs", + no_std: Option = "no-std", + } +} + +impl Config { + pub fn default_opts() -> Config { + let mut config = Config::default(); + config.llvm_optimize = true; + config.ninja_in_file = true; + config.llvm_static_stdcpp = false; + config.backtrace = true; + config.rust_optimize = RustOptimize::Bool(true); + config.rust_optimize_tests = true; + config.submodules = None; + config.docs = true; + config.docs_minification = true; + config.rust_rpath = true; + config.channel = "dev".to_string(); + config.codegen_tests = true; + config.rust_dist_src = true; + config.rust_codegen_backends = vec![INTERNER.intern_str("llvm")]; + config.deny_warnings = true; + config.bindir = "bin".into(); + config.dist_include_mingw_linker = true; + config.dist_compression_profile = "fast".into(); + config.rustc_parallel = true; + + config.stdout_is_tty = std::io::stdout().is_terminal(); + config.stderr_is_tty = std::io::stderr().is_terminal(); + + // set by build.rs + config.build = TargetSelection::from_user(&env!("BUILD_TRIPLE")); + + let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")); + // Undo `src/bootstrap` + config.src = manifest_dir.parent().unwrap().parent().unwrap().to_owned(); + config.out = PathBuf::from("build"); + + config + } + + pub fn parse(args: &[String]) -> Config { + #[cfg(test)] + fn get_toml(_: &Path) -> TomlConfig { + TomlConfig::default() + } + + #[cfg(not(test))] + fn get_toml(file: &Path) -> TomlConfig { + let contents = + t!(fs::read_to_string(file), format!("config file {} not found", file.display())); + // Deserialize to Value and then TomlConfig to prevent the Deserialize impl of + // TomlConfig and sub types to be monomorphized 5x by toml. + toml::from_str(&contents) + .and_then(|table: toml::Value| TomlConfig::deserialize(table)) + .unwrap_or_else(|err| { + eprintln!("failed to parse TOML configuration '{}': {err}", file.display()); + exit!(2); + }) + } + Self::parse_inner(args, get_toml) + } + + fn parse_inner(args: &[String], get_toml: impl Fn(&Path) -> TomlConfig) -> Config { + let mut flags = Flags::parse(&args); + let mut config = Config::default_opts(); + + // Set flags. + config.paths = std::mem::take(&mut flags.paths); + config.skip = flags.skip.into_iter().chain(flags.exclude).collect(); + config.include_default_paths = flags.include_default_paths; + config.rustc_error_format = flags.rustc_error_format; + config.json_output = flags.json_output; + config.on_fail = flags.on_fail; + config.jobs = Some(threads_from_config(flags.jobs as u32)); + config.cmd = flags.cmd; + config.incremental = flags.incremental; + config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled }; + config.keep_stage = flags.keep_stage; + config.keep_stage_std = flags.keep_stage_std; + config.color = flags.color; + config.free_args = std::mem::take(&mut flags.free_args); + config.llvm_profile_use = flags.llvm_profile_use; + config.llvm_profile_generate = flags.llvm_profile_generate; + config.enable_bolt_settings = flags.enable_bolt_settings; + + // Infer the rest of the configuration. + + // Infer the source directory. This is non-trivial because we want to support a downloaded bootstrap binary, + // running on a completely machine from where it was compiled. + let mut cmd = Command::new("git"); + // NOTE: we cannot support running from outside the repository because the only path we have available + // is set at compile time, which can be wrong if bootstrap was downloaded from source. + // We still support running outside the repository if we find we aren't in a git directory. + cmd.arg("rev-parse").arg("--show-toplevel"); + // Discard stderr because we expect this to fail when building from a tarball. + let output = cmd + .stderr(std::process::Stdio::null()) + .output() + .ok() + .and_then(|output| if output.status.success() { Some(output) } else { None }); + if let Some(output) = output { + let git_root = String::from_utf8(output.stdout).unwrap(); + // We need to canonicalize this path to make sure it uses backslashes instead of forward slashes. + let git_root = PathBuf::from(git_root.trim()).canonicalize().unwrap(); + let s = git_root.to_str().unwrap(); + + // Bootstrap is quite bad at handling /? in front of paths + let src = match s.strip_prefix("\\\\?\\") { + Some(p) => PathBuf::from(p), + None => PathBuf::from(git_root), + }; + // If this doesn't have at least `stage0.json`, we guessed wrong. This can happen when, + // for example, the build directory is inside of another unrelated git directory. + // In that case keep the original `CARGO_MANIFEST_DIR` handling. + // + // NOTE: this implies that downloadable bootstrap isn't supported when the build directory is outside + // the source directory. We could fix that by setting a variable from all three of python, ./x, and x.ps1. + if src.join("src").join("stage0.json").exists() { + config.src = src; + } + } else { + // We're building from a tarball, not git sources. + // We don't support pre-downloaded bootstrap in this case. + } + + if cfg!(test) { + // Use the build directory of the original x.py invocation, so that we can set `initial_rustc` properly. + config.out = Path::new( + &env::var_os("CARGO_TARGET_DIR").expect("cargo test directly is not supported"), + ) + .parent() + .unwrap() + .to_path_buf(); + } + + let stage0_json = t!(std::fs::read(&config.src.join("src").join("stage0.json"))); + + config.stage0_metadata = t!(serde_json::from_slice::(&stage0_json)); + + // Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, then `config.toml` in the root directory. + let toml_path = flags + .config + .clone() + .or_else(|| env::var_os("RUST_BOOTSTRAP_CONFIG").map(PathBuf::from)); + let using_default_path = toml_path.is_none(); + let mut toml_path = toml_path.unwrap_or_else(|| PathBuf::from("config.toml")); + if using_default_path && !toml_path.exists() { + toml_path = config.src.join(toml_path); + } + + // Give a hard error if `--config` or `RUST_BOOTSTRAP_CONFIG` are set to a missing path, + // but not if `config.toml` hasn't been created. + let mut toml = if !using_default_path || toml_path.exists() { + config.config = Some(toml_path.clone()); + get_toml(&toml_path) + } else { + config.config = None; + TomlConfig::default() + }; + + if let Some(include) = &toml.profile { + // Allows creating alias for profile names, allowing + // profiles to be renamed while maintaining back compatibility + // Keep in sync with `profile_aliases` in bootstrap.py + let profile_aliases = HashMap::from([("user", "dist")]); + let include = match profile_aliases.get(include.as_str()) { + Some(alias) => alias, + None => include.as_str(), + }; + let mut include_path = config.src.clone(); + include_path.push("src"); + include_path.push("bootstrap"); + include_path.push("defaults"); + include_path.push(format!("config.{include}.toml")); + let included_toml = get_toml(&include_path); + toml.merge(included_toml, ReplaceOpt::IgnoreDuplicate); + } + + let mut override_toml = TomlConfig::default(); + for option in flags.set.iter() { + fn get_table(option: &str) -> Result { + toml::from_str(&option) + .and_then(|table: toml::Value| TomlConfig::deserialize(table)) + } + + let mut err = match get_table(option) { + Ok(v) => { + override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); + continue; + } + Err(e) => e, + }; + // We want to be able to set string values without quotes, + // like in `configure.py`. Try adding quotes around the right hand side + if let Some((key, value)) = option.split_once("=") { + if !value.contains('"') { + match get_table(&format!(r#"{key}="{value}""#)) { + Ok(v) => { + override_toml.merge(v, ReplaceOpt::ErrorOnDuplicate); + continue; + } + Err(e) => err = e, + } + } + } + eprintln!("failed to parse override `{option}`: `{err}"); + exit!(2) + } + toml.merge(override_toml, ReplaceOpt::Override); + + config.changelog_seen = toml.changelog_seen; + config.change_id = toml.change_id; + + let build = toml.build.unwrap_or_default(); + if let Some(file_build) = build.build { + config.build = TargetSelection::from_user(&file_build); + }; + + set(&mut config.out, flags.build_dir.or_else(|| build.build_dir.map(PathBuf::from))); + // NOTE: Bootstrap spawns various commands with different working directories. + // To avoid writing to random places on the file system, `config.out` needs to be an absolute path. + if !config.out.is_absolute() { + // `canonicalize` requires the path to already exist. Use our vendored copy of `absolute` instead. + config.out = crate::utils::helpers::absolute(&config.out); + } + + config.initial_rustc = if let Some(rustc) = build.rustc { + if !flags.skip_stage0_validation { + config.check_build_rustc_version(&rustc); + } + PathBuf::from(rustc) + } else { + config.download_beta_toolchain(); + config.out.join(config.build.triple).join("stage0/bin/rustc") + }; + + config.initial_cargo = build + .cargo + .map(|cargo| { + t!(PathBuf::from(cargo).canonicalize(), "`initial_cargo` not found on disk") + }) + .unwrap_or_else(|| config.out.join(config.build.triple).join("stage0/bin/cargo")); + + // NOTE: it's important this comes *after* we set `initial_rustc` just above. + if config.dry_run() { + let dir = config.out.join("tmp-dry-run"); + t!(fs::create_dir_all(&dir)); + config.out = dir; + } + + config.hosts = if let Some(TargetSelectionList(arg_host)) = flags.host { + arg_host + } else if let Some(file_host) = build.host { + file_host.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + vec![config.build] + }; + config.targets = if let Some(TargetSelectionList(arg_target)) = flags.target { + arg_target + } else if let Some(file_target) = build.target { + file_target.iter().map(|h| TargetSelection::from_user(h)).collect() + } else { + // If target is *not* configured, then default to the host + // toolchains. + config.hosts.clone() + }; + + config.nodejs = build.nodejs.map(PathBuf::from); + config.npm = build.npm.map(PathBuf::from); + config.gdb = build.gdb.map(PathBuf::from); + config.python = build.python.map(PathBuf::from); + config.reuse = build.reuse.map(PathBuf::from); + config.submodules = build.submodules; + config.android_ndk = build.android_ndk; + set(&mut config.low_priority, build.low_priority); + set(&mut config.compiler_docs, build.compiler_docs); + set(&mut config.library_docs_private_items, build.library_docs_private_items); + set(&mut config.docs_minification, build.docs_minification); + set(&mut config.docs, build.docs); + set(&mut config.locked_deps, build.locked_deps); + set(&mut config.vendor, build.vendor); + set(&mut config.full_bootstrap, build.full_bootstrap); + set(&mut config.extended, build.extended); + config.tools = build.tools; + set(&mut config.verbose, build.verbose); + set(&mut config.sanitizers, build.sanitizers); + set(&mut config.profiler, build.profiler); + set(&mut config.cargo_native_static, build.cargo_native_static); + set(&mut config.configure_args, build.configure_args); + set(&mut config.local_rebuild, build.local_rebuild); + set(&mut config.print_step_timings, build.print_step_timings); + set(&mut config.print_step_rusage, build.print_step_rusage); + config.patch_binaries_for_nix = build.patch_binaries_for_nix; + + config.verbose = cmp::max(config.verbose, flags.verbose as usize); + + if let Some(install) = toml.install { + config.prefix = install.prefix.map(PathBuf::from); + config.sysconfdir = install.sysconfdir.map(PathBuf::from); + config.datadir = install.datadir.map(PathBuf::from); + config.docdir = install.docdir.map(PathBuf::from); + set(&mut config.bindir, install.bindir.map(PathBuf::from)); + config.libdir = install.libdir.map(PathBuf::from); + config.mandir = install.mandir.map(PathBuf::from); + } + + // Store off these values as options because if they're not provided + // we'll infer default values for them later + let mut llvm_assertions = None; + let mut llvm_tests = None; + let mut llvm_plugins = None; + let mut debug = None; + let mut debug_assertions = None; + let mut debug_assertions_std = None; + let mut overflow_checks = None; + let mut overflow_checks_std = None; + let mut debug_logging = None; + let mut debuginfo_level = None; + let mut debuginfo_level_rustc = None; + let mut debuginfo_level_std = None; + let mut debuginfo_level_tools = None; + let mut debuginfo_level_tests = None; + let mut optimize = None; + let mut omit_git_hash = None; + + if let Some(rust) = toml.rust { + set(&mut config.channel, rust.channel); + + config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); + // This list is incomplete, please help by expanding it! + if config.download_rustc_commit.is_some() { + // We need the channel used by the downloaded compiler to match the one we set for rustdoc; + // otherwise rustdoc-ui tests break. + let ci_channel = t!(fs::read_to_string(config.src.join("src/ci/channel"))); + let ci_channel = ci_channel.trim_end(); + if config.channel != ci_channel + && !(config.channel == "dev" && ci_channel == "nightly") + { + panic!( + "setting rust.channel={} is incompatible with download-rustc", + config.channel + ); + } + } + + debug = rust.debug; + debug_assertions = rust.debug_assertions; + debug_assertions_std = rust.debug_assertions_std; + overflow_checks = rust.overflow_checks; + overflow_checks_std = rust.overflow_checks_std; + debug_logging = rust.debug_logging; + debuginfo_level = rust.debuginfo_level; + debuginfo_level_rustc = rust.debuginfo_level_rustc; + debuginfo_level_std = rust.debuginfo_level_std; + debuginfo_level_tools = rust.debuginfo_level_tools; + debuginfo_level_tests = rust.debuginfo_level_tests; + + config.rust_split_debuginfo = rust + .split_debuginfo + .as_deref() + .map(SplitDebuginfo::from_str) + .map(|v| v.expect("invalid value for rust.split_debuginfo")) + .unwrap_or(SplitDebuginfo::default_for_platform(&config.build.triple)); + optimize = rust.optimize; + omit_git_hash = rust.omit_git_hash; + config.rust_new_symbol_mangling = rust.new_symbol_mangling; + set(&mut config.rust_optimize_tests, rust.optimize_tests); + set(&mut config.codegen_tests, rust.codegen_tests); + set(&mut config.rust_rpath, rust.rpath); + set(&mut config.jemalloc, rust.jemalloc); + set(&mut config.test_compare_mode, rust.test_compare_mode); + set(&mut config.backtrace, rust.backtrace); + config.description = rust.description; + set(&mut config.rust_dist_src, rust.dist_src); + set(&mut config.verbose_tests, rust.verbose_tests); + // in the case "false" is set explicitly, do not overwrite the command line args + if let Some(true) = rust.incremental { + config.incremental = true; + } + set(&mut config.use_lld, rust.use_lld); + set(&mut config.lld_enabled, rust.lld); + set(&mut config.llvm_tools_enabled, rust.llvm_tools); + config.rustc_parallel = rust + .parallel_compiler + .unwrap_or(config.channel == "dev" || config.channel == "nightly"); + config.rustc_default_linker = rust.default_linker; + config.musl_root = rust.musl_root.map(PathBuf::from); + config.save_toolstates = rust.save_toolstates.map(PathBuf::from); + set( + &mut config.deny_warnings, + match flags.warnings { + Warnings::Deny => Some(true), + Warnings::Warn => Some(false), + Warnings::Default => rust.deny_warnings, + }, + ); + set(&mut config.backtrace_on_ice, rust.backtrace_on_ice); + set(&mut config.rust_verify_llvm_ir, rust.verify_llvm_ir); + config.rust_thin_lto_import_instr_limit = rust.thin_lto_import_instr_limit; + set(&mut config.rust_remap_debuginfo, rust.remap_debuginfo); + set(&mut config.control_flow_guard, rust.control_flow_guard); + config.llvm_libunwind_default = rust + .llvm_libunwind + .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); + + if let Some(ref backends) = rust.codegen_backends { + let available_backends = vec!["llvm", "cranelift", "gcc"]; + + config.rust_codegen_backends = backends.iter().map(|s| { + if let Some(backend) = s.strip_prefix(CODEGEN_BACKEND_PREFIX) { + if available_backends.contains(&backend) { + panic!("Invalid value '{s}' for 'rust.codegen-backends'. Instead, please use '{backend}'."); + } else { + println!("HELP: '{s}' for 'rust.codegen-backends' might fail. \ + Codegen backends are mostly defined without the '{CODEGEN_BACKEND_PREFIX}' prefix. \ + In this case, it would be referred to as '{backend}'."); + } + } + + INTERNER.intern_str(s) + }).collect(); + } + + config.rust_codegen_units = rust.codegen_units.map(threads_from_config); + config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); + config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); + config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); + config.rust_lto = rust + .lto + .as_deref() + .map(|value| RustcLto::from_str(value).unwrap()) + .unwrap_or_default(); + config.rust_validate_mir_opts = rust.validate_mir_opts; + } else { + config.rust_profile_use = flags.rust_profile_use; + config.rust_profile_generate = flags.rust_profile_generate; + } + + config.reproducible_artifacts = flags.reproducible_artifact; + + // rust_info must be set before is_ci_llvm_available() is called. + let default = config.channel == "dev"; + config.omit_git_hash = omit_git_hash.unwrap_or(default); + config.rust_info = GitInfo::new(config.omit_git_hash, &config.src); + + if let Some(llvm) = toml.llvm { + match llvm.ccache { + Some(StringOrBool::String(ref s)) => config.ccache = Some(s.to_string()), + Some(StringOrBool::Bool(true)) => { + config.ccache = Some("ccache".to_string()); + } + Some(StringOrBool::Bool(false)) | None => {} + } + set(&mut config.ninja_in_file, llvm.ninja); + llvm_assertions = llvm.assertions; + llvm_tests = llvm.tests; + llvm_plugins = llvm.plugins; + set(&mut config.llvm_optimize, llvm.optimize); + set(&mut config.llvm_thin_lto, llvm.thin_lto); + set(&mut config.llvm_release_debuginfo, llvm.release_debuginfo); + set(&mut config.llvm_static_stdcpp, llvm.static_libstdcpp); + if let Some(v) = llvm.link_shared { + config.llvm_link_shared.set(Some(v)); + } + config.llvm_targets = llvm.targets.clone(); + config.llvm_experimental_targets = llvm.experimental_targets.clone(); + config.llvm_link_jobs = llvm.link_jobs; + config.llvm_version_suffix = llvm.version_suffix.clone(); + config.llvm_clang_cl = llvm.clang_cl.clone(); + + config.llvm_cflags = llvm.cflags.clone(); + config.llvm_cxxflags = llvm.cxxflags.clone(); + config.llvm_ldflags = llvm.ldflags.clone(); + set(&mut config.llvm_use_libcxx, llvm.use_libcxx); + config.llvm_use_linker = llvm.use_linker.clone(); + config.llvm_allow_old_toolchain = llvm.allow_old_toolchain.unwrap_or(false); + config.llvm_polly = llvm.polly.unwrap_or(false); + config.llvm_clang = llvm.clang.unwrap_or(false); + config.llvm_enable_warnings = llvm.enable_warnings.unwrap_or(false); + config.llvm_build_config = llvm.build_config.clone().unwrap_or(Default::default()); + + let asserts = llvm_assertions.unwrap_or(false); + config.llvm_from_ci = config.parse_download_ci_llvm(llvm.download_ci_llvm, asserts); + + if config.llvm_from_ci { + // None of the LLVM options, except assertions, are supported + // when using downloaded LLVM. We could just ignore these but + // that's potentially confusing, so force them to not be + // explicitly set. The defaults and CI defaults don't + // necessarily match but forcing people to match (somewhat + // arbitrary) CI configuration locally seems bad/hard. + check_ci_llvm!(llvm.optimize); + check_ci_llvm!(llvm.thin_lto); + check_ci_llvm!(llvm.release_debuginfo); + // CI-built LLVM can be either dynamic or static. We won't know until we download it. + check_ci_llvm!(llvm.link_shared); + check_ci_llvm!(llvm.static_libstdcpp); + check_ci_llvm!(llvm.targets); + check_ci_llvm!(llvm.experimental_targets); + check_ci_llvm!(llvm.link_jobs); + check_ci_llvm!(llvm.clang_cl); + check_ci_llvm!(llvm.version_suffix); + check_ci_llvm!(llvm.cflags); + check_ci_llvm!(llvm.cxxflags); + check_ci_llvm!(llvm.ldflags); + check_ci_llvm!(llvm.use_libcxx); + check_ci_llvm!(llvm.use_linker); + check_ci_llvm!(llvm.allow_old_toolchain); + check_ci_llvm!(llvm.polly); + check_ci_llvm!(llvm.clang); + check_ci_llvm!(llvm.build_config); + check_ci_llvm!(llvm.plugins); + } + + // NOTE: can never be hit when downloading from CI, since we call `check_ci_llvm!(thin_lto)` above. + if config.llvm_thin_lto && llvm.link_shared.is_none() { + // If we're building with ThinLTO on, by default we want to link + // to LLVM shared, to avoid re-doing ThinLTO (which happens in + // the link step) with each stage. + config.llvm_link_shared.set(Some(true)); + } + } else { + config.llvm_from_ci = config.channel == "dev" + && crate::core::build_steps::llvm::is_ci_llvm_available(&config, false); + } + + if let Some(t) = toml.target { + for (triple, cfg) in t { + let mut target = Target::from_triple(&triple); + + if let Some(ref s) = cfg.llvm_config { + if config.download_rustc_commit.is_some() && triple == &*config.build.triple { + panic!( + "setting llvm_config for the host is incompatible with download-rustc" + ); + } + target.llvm_config = Some(config.src.join(s)); + } + target.llvm_has_rust_patches = cfg.llvm_has_rust_patches; + if let Some(ref s) = cfg.llvm_filecheck { + target.llvm_filecheck = Some(config.src.join(s)); + } + target.llvm_libunwind = cfg + .llvm_libunwind + .as_ref() + .map(|v| v.parse().expect("failed to parse rust.llvm-libunwind")); + if let Some(s) = cfg.no_std { + target.no_std = s; + } + target.cc = cfg.cc.map(PathBuf::from); + target.cxx = cfg.cxx.map(PathBuf::from); + target.ar = cfg.ar.map(PathBuf::from); + target.ranlib = cfg.ranlib.map(PathBuf::from); + target.linker = cfg.linker.map(PathBuf::from); + target.crt_static = cfg.crt_static; + target.musl_root = cfg.musl_root.map(PathBuf::from); + target.musl_libdir = cfg.musl_libdir.map(PathBuf::from); + target.wasi_root = cfg.wasi_root.map(PathBuf::from); + target.qemu_rootfs = cfg.qemu_rootfs.map(PathBuf::from); + target.sanitizers = cfg.sanitizers; + target.profiler = cfg.profiler; + target.rpath = cfg.rpath; + + config.target_config.insert(TargetSelection::from_user(&triple), target); + } + } + + if config.llvm_from_ci { + let triple = &config.build.triple; + let ci_llvm_bin = config.ci_llvm_root().join("bin"); + let build_target = config + .target_config + .entry(config.build) + .or_insert_with(|| Target::from_triple(&triple)); + + check_ci_llvm!(build_target.llvm_config); + check_ci_llvm!(build_target.llvm_filecheck); + build_target.llvm_config = Some(ci_llvm_bin.join(exe("llvm-config", config.build))); + build_target.llvm_filecheck = Some(ci_llvm_bin.join(exe("FileCheck", config.build))); + } + + if let Some(t) = toml.dist { + config.dist_sign_folder = t.sign_folder.map(PathBuf::from); + config.dist_upload_addr = t.upload_addr; + config.dist_compression_formats = t.compression_formats; + set(&mut config.dist_compression_profile, t.compression_profile); + set(&mut config.rust_dist_src, t.src_tarball); + set(&mut config.missing_tools, t.missing_tools); + set(&mut config.dist_include_mingw_linker, t.include_mingw_linker) + } + + if let Some(r) = build.rustfmt { + *config.initial_rustfmt.borrow_mut() = if r.exists() { + RustfmtState::SystemToolchain(r) + } else { + RustfmtState::Unavailable + }; + } + + // Now that we've reached the end of our configuration, infer the + // default values for all options that we haven't otherwise stored yet. + + config.llvm_assertions = llvm_assertions.unwrap_or(false); + config.llvm_tests = llvm_tests.unwrap_or(false); + config.llvm_plugins = llvm_plugins.unwrap_or(false); + config.rust_optimize = optimize.unwrap_or(RustOptimize::Bool(true)); + + let default = debug == Some(true); + config.rust_debug_assertions = debug_assertions.unwrap_or(default); + config.rust_debug_assertions_std = + debug_assertions_std.unwrap_or(config.rust_debug_assertions); + config.rust_overflow_checks = overflow_checks.unwrap_or(default); + config.rust_overflow_checks_std = + overflow_checks_std.unwrap_or(config.rust_overflow_checks); + + config.rust_debug_logging = debug_logging.unwrap_or(config.rust_debug_assertions); + + let with_defaults = |debuginfo_level_specific: Option<_>| { + debuginfo_level_specific.or(debuginfo_level).unwrap_or(if debug == Some(true) { + DebuginfoLevel::Limited + } else { + DebuginfoLevel::None + }) + }; + config.rust_debuginfo_level_rustc = with_defaults(debuginfo_level_rustc); + config.rust_debuginfo_level_std = with_defaults(debuginfo_level_std); + config.rust_debuginfo_level_tools = with_defaults(debuginfo_level_tools); + config.rust_debuginfo_level_tests = debuginfo_level_tests.unwrap_or(DebuginfoLevel::None); + + let download_rustc = config.download_rustc_commit.is_some(); + // See https://github.com/rust-lang/compiler-team/issues/326 + config.stage = match config.cmd { + Subcommand::Check { .. } => flags.stage.or(build.check_stage).unwrap_or(0), + // `download-rustc` only has a speed-up for stage2 builds. Default to stage2 unless explicitly overridden. + Subcommand::Doc { .. } => { + flags.stage.or(build.doc_stage).unwrap_or(if download_rustc { 2 } else { 0 }) + } + Subcommand::Build { .. } => { + flags.stage.or(build.build_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Test { .. } => { + flags.stage.or(build.test_stage).unwrap_or(if download_rustc { 2 } else { 1 }) + } + Subcommand::Bench { .. } => flags.stage.or(build.bench_stage).unwrap_or(2), + Subcommand::Dist { .. } => flags.stage.or(build.dist_stage).unwrap_or(2), + Subcommand::Install { .. } => flags.stage.or(build.install_stage).unwrap_or(2), + // These are all bootstrap tools, which don't depend on the compiler. + // The stage we pass shouldn't matter, but use 0 just in case. + Subcommand::Clean { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } + | Subcommand::Suggest { .. } => flags.stage.unwrap_or(0), + }; + + // CI should always run stage 2 builds, unless it specifically states otherwise + #[cfg(not(test))] + if flags.stage.is_none() && crate::CiEnv::current() != crate::CiEnv::None { + match config.cmd { + Subcommand::Test { .. } + | Subcommand::Doc { .. } + | Subcommand::Build { .. } + | Subcommand::Bench { .. } + | Subcommand::Dist { .. } + | Subcommand::Install { .. } => { + assert_eq!( + config.stage, 2, + "x.py should be run with `--stage 2` on CI, but was run with `--stage {}`", + config.stage, + ); + } + Subcommand::Clean { .. } + | Subcommand::Check { .. } + | Subcommand::Clippy { .. } + | Subcommand::Fix { .. } + | Subcommand::Run { .. } + | Subcommand::Setup { .. } + | Subcommand::Format { .. } + | Subcommand::Suggest { .. } => {} + } + } + + config + } + + pub(crate) fn dry_run(&self) -> bool { + match self.dry_run { + DryRun::Disabled => false, + DryRun::SelfCheck | DryRun::UserSelected => true, + } + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Exits if the command failed to execute at all, otherwise returns its + /// `status.success()`. + #[deprecated = "use `Builder::try_run` instead where possible"] + pub(crate) fn try_run(&self, cmd: &mut Command) -> Result<(), ()> { + if self.dry_run() { + return Ok(()); + } + self.verbose(&format!("running: {cmd:?}")); + build_helper::util::try_run(cmd, self.is_verbose()) + } + + /// A git invocation which runs inside the source directory. + /// + /// Use this rather than `Command::new("git")` in order to support out-of-tree builds. + pub(crate) fn git(&self) -> Command { + let mut git = Command::new("git"); + git.current_dir(&self.src); + git + } + + pub(crate) fn test_args(&self) -> Vec<&str> { + let mut test_args = match self.cmd { + Subcommand::Test { ref test_args, .. } | Subcommand::Bench { ref test_args, .. } => { + test_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + }; + test_args.extend(self.free_args.iter().map(|s| s.as_str())); + test_args + } + + pub(crate) fn args(&self) -> Vec<&str> { + let mut args = match self.cmd { + Subcommand::Run { ref args, .. } => { + args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + }; + args.extend(self.free_args.iter().map(|s| s.as_str())); + args + } + + /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. + /// Return the version it would have used for the given commit. + pub(crate) fn artifact_version_part(&self, commit: &str) -> String { + let (channel, version) = if self.rust_info.is_managed_git_subrepository() { + let mut channel = self.git(); + channel.arg("show").arg(format!("{commit}:src/ci/channel")); + let channel = output(&mut channel); + let mut version = self.git(); + version.arg("show").arg(format!("{commit}:src/version")); + let version = output(&mut version); + (channel.trim().to_owned(), version.trim().to_owned()) + } else { + let channel = fs::read_to_string(self.src.join("src/ci/channel")); + let version = fs::read_to_string(self.src.join("src/version")); + match (channel, version) { + (Ok(channel), Ok(version)) => { + (channel.trim().to_owned(), version.trim().to_owned()) + } + (channel, version) => { + let src = self.src.display(); + eprintln!("ERROR: failed to determine artifact channel and/or version"); + eprintln!( + "HELP: consider using a git checkout or ensure these files are readable" + ); + if let Err(channel) = channel { + eprintln!("reading {src}/src/ci/channel failed: {channel:?}"); + } + if let Err(version) = version { + eprintln!("reading {src}/src/version failed: {version:?}"); + } + panic!(); + } + } + }; + + match channel.as_str() { + "stable" => version, + "beta" => channel, + "nightly" => channel, + other => unreachable!("{:?} is not recognized as a valid channel", other), + } + } + + /// Try to find the relative path of `bindir`, otherwise return it in full. + pub fn bindir_relative(&self) -> &Path { + let bindir = &self.bindir; + if bindir.is_absolute() { + // Try to make it relative to the prefix. + if let Some(prefix) = &self.prefix { + if let Ok(stripped) = bindir.strip_prefix(prefix) { + return stripped; + } + } + } + bindir + } + + /// Try to find the relative path of `libdir`. + pub fn libdir_relative(&self) -> Option<&Path> { + let libdir = self.libdir.as_ref()?; + if libdir.is_relative() { + Some(libdir) + } else { + // Try to make it relative to the prefix. + libdir.strip_prefix(self.prefix.as_ref()?).ok() + } + } + + /// The absolute path to the downloaded LLVM artifacts. + pub(crate) fn ci_llvm_root(&self) -> PathBuf { + assert!(self.llvm_from_ci); + self.out.join(&*self.build.triple).join("ci-llvm") + } + + /// Directory where the extracted `rustc-dev` component is stored. + pub(crate) fn ci_rustc_dir(&self) -> PathBuf { + assert!(self.download_rustc()); + self.out.join(self.build.triple).join("ci-rustc") + } + + /// Determine whether llvm should be linked dynamically. + /// + /// If `false`, llvm should be linked statically. + /// This is computed on demand since LLVM might have to first be downloaded from CI. + pub(crate) fn llvm_link_shared(&self) -> bool { + let mut opt = self.llvm_link_shared.get(); + if opt.is_none() && self.dry_run() { + // just assume static for now - dynamic linking isn't supported on all platforms + return false; + } + + let llvm_link_shared = *opt.get_or_insert_with(|| { + if self.llvm_from_ci { + self.maybe_download_ci_llvm(); + let ci_llvm = self.ci_llvm_root(); + let link_type = t!( + std::fs::read_to_string(ci_llvm.join("link-type.txt")), + format!("CI llvm missing: {}", ci_llvm.display()) + ); + link_type == "dynamic" + } else { + // unclear how thought-through this default is, but it maintains compatibility with + // previous behavior + false + } + }); + self.llvm_link_shared.set(opt); + llvm_link_shared + } + + /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source. + pub(crate) fn download_rustc(&self) -> bool { + self.download_rustc_commit().is_some() + } + + pub(crate) fn download_rustc_commit(&self) -> Option<&str> { + static DOWNLOAD_RUSTC: OnceCell> = OnceCell::new(); + if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() { + // avoid trying to actually download the commit + return self.download_rustc_commit.as_deref(); + } + + DOWNLOAD_RUSTC + .get_or_init(|| match &self.download_rustc_commit { + None => None, + Some(commit) => { + self.download_ci_rustc(commit); + Some(commit.clone()) + } + }) + .as_deref() + } + + pub(crate) fn initial_rustfmt(&self) -> Option { + match &mut *self.initial_rustfmt.borrow_mut() { + RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), + RustfmtState::Unavailable => None, + r @ RustfmtState::LazyEvaluated => { + if self.dry_run() { + return Some(PathBuf::new()); + } + let path = self.maybe_download_rustfmt(); + *r = if let Some(p) = &path { + RustfmtState::Downloaded(p.clone()) + } else { + RustfmtState::Unavailable + }; + path + } + } + } + + pub fn verbose(&self, msg: &str) { + if self.verbose > 0 { + println!("{msg}"); + } + } + + pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).map(|t| t.sanitizers).flatten().unwrap_or(self.sanitizers) + } + + pub fn any_sanitizers_enabled(&self) -> bool { + self.target_config.values().any(|t| t.sanitizers == Some(true)) || self.sanitizers + } + + pub fn profiler_path(&self, target: TargetSelection) -> Option<&str> { + match self.target_config.get(&target)?.profiler.as_ref()? { + StringOrBool::String(s) => Some(s), + StringOrBool::Bool(_) => None, + } + } + + pub fn profiler_enabled(&self, target: TargetSelection) -> bool { + self.target_config + .get(&target) + .and_then(|t| t.profiler.as_ref()) + .map(StringOrBool::is_string_or_true) + .unwrap_or(self.profiler) + } + + pub fn any_profiler_enabled(&self) -> bool { + self.target_config.values().any(|t| matches!(&t.profiler, Some(p) if p.is_string_or_true())) + || self.profiler + } + + pub fn rpath_enabled(&self, target: TargetSelection) -> bool { + self.target_config.get(&target).map(|t| t.rpath).flatten().unwrap_or(self.rust_rpath) + } + + pub fn llvm_enabled(&self) -> bool { + self.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) + } + + pub fn llvm_libunwind(&self, target: TargetSelection) -> LlvmLibunwind { + self.target_config + .get(&target) + .and_then(|t| t.llvm_libunwind) + .or(self.llvm_libunwind_default) + .unwrap_or(if target.contains("fuchsia") { + LlvmLibunwind::InTree + } else { + LlvmLibunwind::No + }) + } + + pub fn submodules(&self, rust_info: &GitInfo) -> bool { + self.submodules.unwrap_or(rust_info.is_managed_git_subrepository()) + } + + pub fn default_codegen_backend(&self) -> Option> { + self.rust_codegen_backends.get(0).cloned() + } + + pub fn git_config(&self) -> GitConfig<'_> { + GitConfig { + git_repository: &self.stage0_metadata.config.git_repository, + nightly_branch: &self.stage0_metadata.config.nightly_branch, + } + } + + pub fn check_build_rustc_version(&self, rustc_path: &str) { + if self.dry_run() { + return; + } + + // check rustc version is same or lower with 1 apart from the building one + let mut cmd = Command::new(rustc_path); + cmd.arg("--version"); + let rustc_output = output(&mut cmd) + .lines() + .next() + .unwrap() + .split(' ') + .nth(1) + .unwrap() + .split('-') + .next() + .unwrap() + .to_owned(); + let rustc_version = Version::parse(&rustc_output.trim()).unwrap(); + let source_version = + Version::parse(&fs::read_to_string(self.src.join("src/version")).unwrap().trim()) + .unwrap(); + if !(source_version == rustc_version + || (source_version.major == rustc_version.major + && (source_version.minor == rustc_version.minor + || source_version.minor == rustc_version.minor + 1))) + { + let prev_version = format!("{}.{}.x", source_version.major, source_version.minor - 1); + eprintln!( + "Unexpected rustc version: {rustc_version}, we should use {prev_version}/{source_version} to build source with {source_version}" + ); + exit!(1); + } + } + + /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. + fn download_ci_rustc_commit(&self, download_rustc: Option) -> Option { + // If `download-rustc` is not set, default to rebuilding. + let if_unchanged = match download_rustc { + None | Some(StringOrBool::Bool(false)) => return None, + Some(StringOrBool::Bool(true)) => false, + Some(StringOrBool::String(s)) if s == "if-unchanged" => true, + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-rustc: {other}") + } + }; + + // Handle running from a directory other than the top level + let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + let compiler = format!("{top_level}/compiler/"); + let library = format!("{top_level}/library/"); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + self.git() + .arg("rev-list") + .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("ERROR: could not find commit hash for downloading rustc"); + println!("HELP: maybe your repository history is too shallow?"); + println!("HELP: consider disabling `download-rustc`"); + println!("HELP: or fetch enough history to include one upstream commit"); + crate::exit!(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let has_changes = !t!(self + .git() + .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) + .status()) + .success(); + if has_changes { + if if_unchanged { + if self.verbose > 0 { + println!( + "WARNING: saw changes to compiler/ or library/ since {commit}; \ + ignoring `download-rustc`" + ); + } + return None; + } + println!( + "WARNING: `download-rustc` is enabled, but there are changes to \ + compiler/ or library/" + ); + } + + Some(commit.to_string()) + } + + fn parse_download_ci_llvm( + &self, + download_ci_llvm: Option, + asserts: bool, + ) -> bool { + match download_ci_llvm { + None => self.channel == "dev" && llvm::is_ci_llvm_available(&self, asserts), + Some(StringOrBool::Bool(b)) => b, + Some(StringOrBool::String(s)) if s == "if-available" => { + llvm::is_ci_llvm_available(&self, asserts) + } + Some(StringOrBool::String(s)) if s == "if-unchanged" => { + // Git is needed to track modifications here, but tarball source is not available. + // If not modified here or built through tarball source, we maintain consistency + // with '"if available"'. + if !self.rust_info.is_from_tarball() + && self + .last_modified_commit(&["src/llvm-project"], "download-ci-llvm", true) + .is_none() + { + // there are some untracked changes in the the given paths. + false + } else { + llvm::is_ci_llvm_available(&self, asserts) + } + } + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-ci-llvm: {:?}", other) + } + } + } + + /// Returns the last commit in which any of `modified_paths` were changed, + /// or `None` if there are untracked changes in the working directory and `if_unchanged` is true. + pub fn last_modified_commit( + &self, + modified_paths: &[&str], + option_name: &str, + if_unchanged: bool, + ) -> Option { + // Handle running from a directory other than the top level + let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + self.git() + .arg("rev-list") + .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading components from CI"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `{option_name}`"); + println!("help: or fetch enough history to include one upstream commit"); + crate::exit!(1); + } + + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let mut git = self.git(); + git.args(&["diff-index", "--quiet", &commit, "--"]); + + for path in modified_paths { + git.arg(format!("{top_level}/{path}")); + } + + let has_changes = !t!(git.status()).success(); + if has_changes { + if if_unchanged { + if self.verbose > 0 { + println!( + "warning: saw changes to one of {modified_paths:?} since {commit}; \ + ignoring `{option_name}`" + ); + } + return None; + } + println!( + "warning: `{option_name}` is enabled, but there are changes to one of {modified_paths:?}" + ); + } + + Some(commit.to_string()) + } +} + +fn set(field: &mut T, val: Option) { + if let Some(v) = val { + *field = v; + } +} + +fn threads_from_config(v: u32) -> u32 { + match v { + 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, + n => n, + } +} diff --git a/src/bootstrap/src/core/config/flags.rs b/src/bootstrap/src/core/config/flags.rs new file mode 100644 index 000000000..2a301007a --- /dev/null +++ b/src/bootstrap/src/core/config/flags.rs @@ -0,0 +1,571 @@ +//! Command-line interface of the rustbuild build system. +//! +//! This module implements the command-line parsing of the build system which +//! has various flags to configure how it's run. + +use std::path::{Path, PathBuf}; + +use clap::{CommandFactory, Parser, ValueEnum}; + +use crate::core::build_steps::setup::Profile; +use crate::core::builder::{Builder, Kind}; +use crate::core::config::{target_selection_list, Config, TargetSelectionList}; +use crate::{Build, DocTests}; + +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Color { + Always, + Never, + #[default] + Auto, +} + +/// Whether to deny warnings, emit them as warnings, or use the default behavior +#[derive(Copy, Clone, Default, Debug, ValueEnum)] +pub enum Warnings { + Deny, + Warn, + #[default] + Default, +} + +/// Deserialized version of all flags for this compile. +#[derive(Debug, Parser)] +#[clap( + override_usage = "x.py [options] [...]", + disable_help_subcommand(true), + about = "", + next_line_help(false) +)] +pub struct Flags { + #[command(subcommand)] + pub cmd: Subcommand, + + #[arg(global(true), short, long, action = clap::ArgAction::Count)] + /// use verbose output (-vv for very verbose) + pub verbose: u8, // each extra -v after the first is passed to Cargo + #[arg(global(true), short, long)] + /// use incremental compilation + pub incremental: bool, + #[arg(global(true), long, value_hint = clap::ValueHint::FilePath, value_name = "FILE")] + /// TOML configuration file for build + pub config: Option, + #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] + /// Build directory, overrides `build.build-dir` in `config.toml` + pub build_dir: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "BUILD")] + /// build target of the stage0 compiler + pub build: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "HOST", value_parser = target_selection_list)] + /// host targets to build + pub host: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::Other, value_name = "TARGET", value_parser = target_selection_list)] + /// target targets to build + pub target: Option, + + #[arg(global(true), long, value_name = "PATH")] + /// build paths to exclude + pub exclude: Vec, // keeping for client backward compatibility + #[arg(global(true), long, value_name = "PATH")] + /// build paths to skip + pub skip: Vec, + #[arg(global(true), long)] + /// include default paths in addition to the provided ones + pub include_default_paths: bool, + + #[arg(global(true), value_hint = clap::ValueHint::Other, long)] + pub rustc_error_format: Option, + + #[arg(global(true), long, value_hint = clap::ValueHint::CommandString, value_name = "CMD")] + /// command to run on failure + pub on_fail: Option, + #[arg(global(true), long)] + /// dry run; don't build anything + pub dry_run: bool, + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the + /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.) + pub stage: Option, + + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage(s) to keep without recompiling + /// (pass multiple times to keep e.g., both stages 0 and 1) + pub keep_stage: Vec, + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")] + /// stage(s) of the standard library to keep without recompiling + /// (pass multiple times to keep e.g., both stages 0 and 1) + pub keep_stage_std: Vec, + #[arg(global(true), long, value_hint = clap::ValueHint::DirPath, value_name = "DIR")] + /// path to the root of the rust checkout + pub src: Option, + + #[arg( + global(true), + short, + long, + value_hint = clap::ValueHint::Other, + default_value_t = std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get), + value_name = "JOBS" + )] + /// number of jobs to run in parallel + pub jobs: usize, + // This overrides the deny-warnings configuration option, + // which passes -Dwarnings to the compiler invocations. + #[arg(global(true), long)] + #[clap(value_enum, default_value_t=Warnings::Default, value_name = "deny|warn")] + /// if value is deny, will deny warnings + /// if value is warn, will emit warnings + /// otherwise, use the default configured behaviour + pub warnings: Warnings, + + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "FORMAT")] + /// rustc error format + pub error_format: Option, + #[arg(global(true), long)] + /// use message-format=json + pub json_output: bool, + + #[arg(global(true), long, value_name = "STYLE")] + #[clap(value_enum, default_value_t = Color::Auto)] + /// whether to use color in cargo and rustc output + pub color: Color, + + /// whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml + #[arg(global(true), long, value_name = "VALUE")] + pub llvm_skip_rebuild: Option, + /// generate PGO profile with rustc build + #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub rust_profile_generate: Option, + /// use PGO profile for rustc build + #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub rust_profile_use: Option, + /// use PGO profile for LLVM build + #[arg(global(true), value_hint = clap::ValueHint::FilePath, long, value_name = "PROFILE")] + pub llvm_profile_use: Option, + // LLVM doesn't support a custom location for generating profile + // information. + // + // llvm_out/build/profiles/ is the location this writes to. + /// generate PGO profile with llvm built for rustc + #[arg(global(true), long)] + pub llvm_profile_generate: bool, + /// Enable BOLT link flags + #[arg(global(true), long)] + pub enable_bolt_settings: bool, + /// Skip stage0 compiler validation + #[arg(global(true), long)] + pub skip_stage0_validation: bool, + /// Additional reproducible artifacts that should be added to the reproducible artifacts archive. + #[arg(global(true), long)] + pub reproducible_artifact: Vec, + #[arg(global(true))] + /// paths for the subcommand + pub paths: Vec, + /// override options in config.toml + #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "section.option=value")] + pub set: Vec, + /// arguments passed to subcommands + #[arg(global(true), last(true), value_name = "ARGS")] + pub free_args: Vec, +} + +impl Flags { + pub fn parse(args: &[String]) -> Self { + let first = String::from("x.py"); + let it = std::iter::once(&first).chain(args.iter()); + // We need to check for ` -h -v`, in which case we list the paths + #[derive(Parser)] + #[clap(disable_help_flag(true))] + struct HelpVerboseOnly { + #[arg(short, long)] + help: bool, + #[arg(global(true), short, long, action = clap::ArgAction::Count)] + pub verbose: u8, + #[arg(value_enum)] + cmd: Kind, + } + if let Ok(HelpVerboseOnly { help: true, verbose: 1.., cmd: subcommand }) = + HelpVerboseOnly::try_parse_from(it.clone()) + { + println!("NOTE: updating submodules before printing available paths"); + let config = Config::parse(&[String::from("build")]); + let build = Build::new(config); + let paths = Builder::get_help(&build, subcommand); + if let Some(s) = paths { + println!("{s}"); + } else { + panic!("No paths available for subcommand `{}`", subcommand.as_str()); + } + crate::exit!(0); + } + + Flags::parse_from(it) + } +} + +#[derive(Debug, Clone, Default, clap::Subcommand)] +pub enum Subcommand { + #[clap(aliases = ["b"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to compile. For example, for a quick build of a usable + compiler: + ./x.py build --stage 1 library/std + This will build a compiler and standard library from the local source code. + Once this is done, build/$ARCH/stage1 contains a usable compiler. + If no arguments are passed then the default artifacts for that stage are + compiled. For example: + ./x.py build --stage 0 + ./x.py build ")] + /// Compile either the compiler or libraries + #[default] + Build, + #[clap(aliases = ["c"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to compile. For example: + ./x.py check library/std + If no arguments are passed then many artifacts are checked.")] + /// Compile either the compiler or libraries, using cargo check + Check { + #[arg(long)] + /// Check all targets + all_targets: bool, + }, + /// Run Clippy (uses rustup/cargo-installed clippy binary) + #[clap(long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to run clippy against. For example: + ./x.py clippy library/core + ./x.py clippy library/core library/proc_macro")] + Clippy { + #[arg(long)] + fix: bool, + /// clippy lints to allow + #[arg(global(true), short = 'A', action = clap::ArgAction::Append, value_name = "LINT")] + allow: Vec, + /// clippy lints to deny + #[arg(global(true), short = 'D', action = clap::ArgAction::Append, value_name = "LINT")] + deny: Vec, + /// clippy lints to warn on + #[arg(global(true), short = 'W', action = clap::ArgAction::Append, value_name = "LINT")] + warn: Vec, + /// clippy lints to forbid + #[arg(global(true), short = 'F', action = clap::ArgAction::Append, value_name = "LINT")] + forbid: Vec, + }, + /// Run cargo fix + #[clap(long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories to the crates + and/or artifacts to run `cargo fix` against. For example: + ./x.py fix library/core + ./x.py fix library/core library/proc_macro")] + Fix, + #[clap( + name = "fmt", + long_about = "\n + Arguments: + This subcommand optionally accepts a `--check` flag which succeeds if formatting is correct and + fails if it is not. For example: + ./x.py fmt + ./x.py fmt --check" + )] + /// Run rustfmt + Format { + /// check formatting instead of applying + #[arg(long)] + check: bool, + }, + #[clap(aliases = ["d"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to directories of documentation + to build. For example: + ./x.py doc src/doc/book + ./x.py doc src/doc/nomicon + ./x.py doc src/doc/book library/std + ./x.py doc library/std --json + ./x.py doc library/std --open + If no arguments are passed then everything is documented: + ./x.py doc + ./x.py doc --stage 1")] + /// Build documentation + Doc { + #[arg(long)] + /// open the docs in a browser + open: bool, + #[arg(long)] + /// render the documentation in JSON format in addition to the usual HTML format + json: bool, + }, + #[clap(aliases = ["t"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to test directories that + should be compiled and run. For example: + ./x.py test tests/ui + ./x.py test library/std --test-args hash_map + ./x.py test library/std --stage 0 --no-doc + ./x.py test tests/ui --bless + ./x.py test tests/ui --compare-mode next-solver + Note that `test tests/* --stage N` does NOT depend on `build compiler/rustc --stage N`; + just like `build library/std --stage N` it tests the compiler produced by the previous + stage. + Execute tool tests with a tool name argument: + ./x.py test tidy + If no arguments are passed then the complete artifacts for that stage are + compiled and tested. + ./x.py test + ./x.py test --stage 1")] + /// Build and run some test suites + Test { + #[arg(long)] + /// run all tests regardless of failure + no_fail_fast: bool, + #[arg(long, value_name = "SUBSTRING")] + /// skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times + skip: Vec, + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + /// extra arguments to be passed for the test tool being used + /// (e.g. libtest, compiletest or rustdoc) + test_args: Vec, + /// extra options to pass the compiler when running tests + #[arg(long, value_name = "ARGS", allow_hyphen_values(true))] + rustc_args: Vec, + #[arg(long)] + /// do not run doc tests + no_doc: bool, + #[arg(long)] + /// only run doc tests + doc: bool, + #[arg(long)] + /// whether to automatically update stderr/stdout files + bless: bool, + #[arg(long)] + /// comma-separated list of other files types to check (accepts py, py:lint, + /// py:fmt, shell) + extra_checks: Option, + #[arg(long)] + /// rerun tests even if the inputs are unchanged + force_rerun: bool, + #[arg(long)] + /// only run tests that result has been changed + only_modified: bool, + #[arg(long, value_name = "COMPARE MODE")] + /// mode describing what file the actual ui output will be compared to + compare_mode: Option, + #[arg(long, value_name = "check | build | run")] + /// force {check,build,run}-pass tests to this mode. + pass: Option, + #[arg(long, value_name = "auto | always | never")] + /// whether to execute run-* tests + run: Option, + #[arg(long)] + /// enable this to generate a Rustfix coverage file, which is saved in + /// `//rustfix_missing_coverage.txt` + rustfix_coverage: bool, + }, + /// Build and run some benchmarks + Bench { + #[arg(long, allow_hyphen_values(true))] + test_args: Vec, + }, + /// Clean out build directories + Clean { + #[arg(long)] + /// Clean the entire build directory (not used by default) + all: bool, + #[arg(long, value_name = "N")] + /// Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used. + stage: Option, + }, + /// Build distribution artifacts + Dist, + /// Install distribution artifacts + Install, + #[clap(aliases = ["r"], long_about = "\n + Arguments: + This subcommand accepts a number of paths to tools to build and run. For + example: + ./x.py run src/tools/expand-yaml-anchors + At least a tool needs to be called.")] + /// Run tools contained in this repository + Run { + /// arguments for the tool + #[arg(long, allow_hyphen_values(true))] + args: Vec, + }, + /// Set up the environment for development + #[clap(long_about = format!( + "\n +x.py setup creates a `config.toml` which changes the defaults for x.py itself, +as well as setting up a git pre-push hook, VS Code config and toolchain link. +Arguments: + This subcommand accepts a 'profile' to use for builds. For example: + ./x.py setup library + The profile is optional and you will be prompted interactively if it is not given. + The following profiles are available: +{} + To only set up the git hook, VS Code config or toolchain link, you may use + ./x.py setup hook + ./x.py setup vscode + ./x.py setup link", Profile::all_for_help(" ").trim_end()))] + Setup { + /// Either the profile for `config.toml` or another setup action. + /// May be omitted to set up interactively + #[arg(value_name = "|hook|vscode|link")] + profile: Option, + }, + /// Suggest a subset of tests to run, based on modified files + #[clap(long_about = "\n")] + Suggest { + /// run suggested tests + #[arg(long)] + run: bool, + }, +} + +impl Subcommand { + pub fn kind(&self) -> Kind { + match self { + Subcommand::Bench { .. } => Kind::Bench, + Subcommand::Build { .. } => Kind::Build, + Subcommand::Check { .. } => Kind::Check, + Subcommand::Clippy { .. } => Kind::Clippy, + Subcommand::Doc { .. } => Kind::Doc, + Subcommand::Fix { .. } => Kind::Fix, + Subcommand::Format { .. } => Kind::Format, + Subcommand::Test { .. } => Kind::Test, + Subcommand::Clean { .. } => Kind::Clean, + Subcommand::Dist { .. } => Kind::Dist, + Subcommand::Install { .. } => Kind::Install, + Subcommand::Run { .. } => Kind::Run, + Subcommand::Setup { .. } => Kind::Setup, + Subcommand::Suggest { .. } => Kind::Suggest, + } + } + + pub fn rustc_args(&self) -> Vec<&str> { + match *self { + Subcommand::Test { ref rustc_args, .. } => { + rustc_args.iter().flat_map(|s| s.split_whitespace()).collect() + } + _ => vec![], + } + } + + pub fn fail_fast(&self) -> bool { + match *self { + Subcommand::Test { no_fail_fast, .. } => !no_fail_fast, + _ => false, + } + } + + pub fn doc_tests(&self) -> DocTests { + match *self { + Subcommand::Test { doc, no_doc, .. } => { + if doc { + DocTests::Only + } else if no_doc { + DocTests::No + } else { + DocTests::Yes + } + } + _ => DocTests::Yes, + } + } + + pub fn bless(&self) -> bool { + match *self { + Subcommand::Test { bless, .. } => bless, + _ => false, + } + } + + pub fn extra_checks(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref extra_checks, .. } => extra_checks.as_ref().map(String::as_str), + _ => None, + } + } + + pub fn only_modified(&self) -> bool { + match *self { + Subcommand::Test { only_modified, .. } => only_modified, + _ => false, + } + } + + pub fn force_rerun(&self) -> bool { + match *self { + Subcommand::Test { force_rerun, .. } => force_rerun, + _ => false, + } + } + + pub fn rustfix_coverage(&self) -> bool { + match *self { + Subcommand::Test { rustfix_coverage, .. } => rustfix_coverage, + _ => false, + } + } + + pub fn compare_mode(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref compare_mode, .. } => compare_mode.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn pass(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref pass, .. } => pass.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn run(&self) -> Option<&str> { + match *self { + Subcommand::Test { ref run, .. } => run.as_ref().map(|s| &s[..]), + _ => None, + } + } + + pub fn open(&self) -> bool { + match *self { + Subcommand::Doc { open, .. } => open, + _ => false, + } + } + + pub fn json(&self) -> bool { + match *self { + Subcommand::Doc { json, .. } => json, + _ => false, + } + } +} + +/// Returns the shell completion for a given shell, if the result differs from the current +/// content of `path`. If `path` does not exist, always returns `Some`. +pub fn get_completion(shell: G, path: &Path) -> Option { + let mut cmd = Flags::command(); + let current = if !path.exists() { + String::new() + } else { + std::fs::read_to_string(path).unwrap_or_else(|_| { + eprintln!("couldn't read {}", path.display()); + crate::exit!(1) + }) + }; + let mut buf = Vec::new(); + clap_complete::generate(shell, &mut cmd, "x.py", &mut buf); + if buf == current.as_bytes() { + return None; + } + Some(String::from_utf8(buf).expect("completion script should be UTF-8")) +} diff --git a/src/bootstrap/src/core/config/mod.rs b/src/bootstrap/src/core/config/mod.rs new file mode 100644 index 000000000..9c6861826 --- /dev/null +++ b/src/bootstrap/src/core/config/mod.rs @@ -0,0 +1,4 @@ +pub(crate) mod config; +pub(crate) mod flags; + +pub use config::*; diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs new file mode 100644 index 000000000..3327aed96 --- /dev/null +++ b/src/bootstrap/src/core/download.rs @@ -0,0 +1,701 @@ +use std::{ + env, + ffi::{OsStr, OsString}, + fs::{self, File}, + io::{BufRead, BufReader, BufWriter, ErrorKind, Write}, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use build_helper::ci::CiEnv; +use once_cell::sync::OnceCell; +use xz2::bufread::XzDecoder; + +use crate::core::build_steps::llvm::detect_llvm_sha; +use crate::core::config::RustfmtMetadata; +use crate::utils::helpers::{check_run, exe, program_out_of_date}; +use crate::{t, Config}; + +static SHOULD_FIX_BINS_AND_DYLIBS: OnceCell = OnceCell::new(); + +/// `Config::try_run` wrapper for this module to avoid warnings on `try_run`, since we don't have access to a `builder` yet. +fn try_run(config: &Config, cmd: &mut Command) -> Result<(), ()> { + #[allow(deprecated)] + config.try_run(cmd) +} + +/// Generic helpers that are useful anywhere in bootstrap. +impl Config { + pub fn is_verbose(&self) -> bool { + self.verbose > 0 + } + + pub(crate) fn create(&self, path: &Path, s: &str) { + if self.dry_run() { + return; + } + t!(fs::write(path, s)); + } + + pub(crate) fn remove(&self, f: &Path) { + if self.dry_run() { + return; + } + fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f)); + } + + /// Create a temporary directory in `out` and return its path. + /// + /// NOTE: this temporary directory is shared between all steps; + /// if you need an empty directory, create a new subdirectory inside it. + pub(crate) fn tempdir(&self) -> PathBuf { + let tmp = self.out.join("tmp"); + t!(fs::create_dir_all(&tmp)); + tmp + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Returns false if do not execute at all, otherwise returns its + /// `status.success()`. + pub(crate) fn check_run(&self, cmd: &mut Command) -> bool { + if self.dry_run() { + return true; + } + self.verbose(&format!("running: {cmd:?}")); + check_run(cmd, self.is_verbose()) + } + + /// Whether or not `fix_bin_or_dylib` needs to be run; can only be true + /// on NixOS + fn should_fix_bins_and_dylibs(&self) -> bool { + let val = *SHOULD_FIX_BINS_AND_DYLIBS.get_or_init(|| { + match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { + Err(_) => return false, + Ok(output) if !output.status.success() => return false, + Ok(output) => { + let mut os_name = output.stdout; + if os_name.last() == Some(&b'\n') { + os_name.pop(); + } + if os_name != b"Linux" { + return false; + } + } + } + + // If the user has asked binaries to be patched for Nix, then + // don't check for NixOS or `/lib`. + // NOTE: this intentionally comes after the Linux check: + // - patchelf only works with ELF files, so no need to run it on Mac or Windows + // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc. + if let Some(explicit_value) = self.patch_binaries_for_nix { + return explicit_value; + } + + // Use `/etc/os-release` instead of `/etc/NIXOS`. + // The latter one does not exist on NixOS when using tmpfs as root. + let is_nixos = match File::open("/etc/os-release") { + Err(e) if e.kind() == ErrorKind::NotFound => false, + Err(e) => panic!("failed to access /etc/os-release: {}", e), + Ok(os_release) => BufReader::new(os_release).lines().any(|l| { + let l = l.expect("reading /etc/os-release"); + matches!(l.trim(), "ID=nixos" | "ID='nixos'" | "ID=\"nixos\"") + }), + }; + if !is_nixos { + let in_nix_shell = env::var("IN_NIX_SHELL"); + if let Ok(in_nix_shell) = in_nix_shell { + eprintln!( + "The IN_NIX_SHELL environment variable is `{in_nix_shell}`; \ + you may need to set `patch-binaries-for-nix=true` in config.toml" + ); + } + } + is_nixos + }); + if val { + eprintln!("INFO: You seem to be using Nix."); + } + val + } + + /// Modifies the interpreter section of 'fname' to fix the dynamic linker, + /// or the RPATH section, to fix the dynamic library search path + /// + /// This is only required on NixOS and uses the PatchELF utility to + /// change the interpreter/RPATH of ELF executables. + /// + /// Please see for more information + fn fix_bin_or_dylib(&self, fname: &Path) { + assert_eq!(SHOULD_FIX_BINS_AND_DYLIBS.get(), Some(&true)); + println!("attempting to patch {}", fname.display()); + + // Only build `.nix-deps` once. + static NIX_DEPS_DIR: OnceCell = OnceCell::new(); + let mut nix_build_succeeded = true; + let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { + // Run `nix-build` to "build" each dependency (which will likely reuse + // the existing `/nix/store` copy, or at most download a pre-built copy). + // + // Importantly, we create a gc-root called `.nix-deps` in the `build/` + // directory, but still reference the actual `/nix/store` path in the rpath + // as it makes it significantly more robust against changes to the location of + // the `.nix-deps` location. + // + // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). + // zlib: Needed as a system dependency of `libLLVM-*.so`. + // patchelf: Needed for patching ELF binaries (see doc comment above). + let nix_deps_dir = self.out.join(".nix-deps"); + const NIX_EXPR: &str = " + with (import {}); + symlinkJoin { + name = \"rust-stage0-dependencies\"; + paths = [ + zlib + patchelf + stdenv.cc.bintools + ]; + } + "; + nix_build_succeeded = try_run( + self, + Command::new("nix-build").args(&[ + Path::new("-E"), + Path::new(NIX_EXPR), + Path::new("-o"), + &nix_deps_dir, + ]), + ) + .is_ok(); + nix_deps_dir + }); + if !nix_build_succeeded { + return; + } + + let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); + let rpath_entries = { + // ORIGIN is a relative default, all binary and dynamic libraries we ship + // appear to have this (even when `../lib` is redundant). + // NOTE: there are only two paths here, delimited by a `:` + let mut entries = OsString::from("$ORIGIN/../lib:"); + entries.push(t!(fs::canonicalize(nix_deps_dir)).join("lib")); + entries + }; + patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); + if !fname.extension().map_or(false, |ext| ext == "so") { + // Finally, set the correct .interp for binaries + let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); + // FIXME: can we support utf8 here? `args` doesn't accept Vec, only OsString ... + let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); + patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); + } + + let _ = try_run(self, patchelf.arg(fname)); + } + + fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) { + self.verbose(&format!("download {url}")); + // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. + let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); + // While bootstrap itself only supports http and https downloads, downstream forks might + // need to download components from other protocols. The match allows them adding more + // protocols without worrying about merge conflicts if we change the HTTP implementation. + match url.split_once("://").map(|(proto, _)| proto) { + Some("http") | Some("https") => { + self.download_http_with_retries(&tempfile, url, help_on_error) + } + Some(other) => panic!("unsupported protocol {other} in {url}"), + None => panic!("no protocol in {url}"), + } + t!(std::fs::rename(&tempfile, dest_path)); + } + + fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { + println!("downloading {url}"); + // Try curl. If that fails and we are on windows, fallback to PowerShell. + let mut curl = Command::new("curl"); + curl.args(&[ + "-y", + "30", + "-Y", + "10", // timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", + "30", // timeout if cannot connect within 30 seconds + "-o", + tempfile.to_str().unwrap(), + "--retry", + "3", + "-SRf", + ]); + // Don't print progress in CI; the \r wrapping looks bad and downloads don't take long enough for progress to be useful. + if CiEnv::is_ci() { + curl.arg("-s"); + } else { + curl.arg("--progress-bar"); + } + curl.arg(url); + if !self.check_run(&mut curl) { + if self.build.contains("windows-msvc") { + eprintln!("Fallback to PowerShell"); + for _ in 0..3 { + if try_run(self, Command::new("PowerShell.exe").args(&[ + "/nologo", + "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), + ), + ])).is_err() { + return; + } + eprintln!("\nspurious failure, trying again"); + } + } + if !help_on_error.is_empty() { + eprintln!("{help_on_error}"); + } + crate::exit!(1); + } + } + + fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) { + eprintln!("extracting {} to {}", tarball.display(), dst.display()); + if !dst.exists() { + t!(fs::create_dir_all(dst)); + } + + // `tarball` ends with `.tar.xz`; strip that suffix + // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` + let uncompressed_filename = + Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); + let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); + + // decompress the file + let data = t!(File::open(tarball), format!("file {} not found", tarball.display())); + let decompressor = XzDecoder::new(BufReader::new(data)); + + let mut tar = tar::Archive::new(decompressor); + + // `compile::Sysroot` needs to know the contents of the `rustc-dev` tarball to avoid adding + // it to the sysroot unless it was explicitly requested. But parsing the 100 MB tarball is slow. + // Cache the entries when we extract it so we only have to read it once. + let mut recorded_entries = + if dst.ends_with("ci-rustc") { recorded_entries(dst, pattern) } else { None }; + + for member in t!(tar.entries()) { + let mut member = t!(member); + let original_path = t!(member.path()).into_owned(); + // skip the top-level directory + if original_path == directory_prefix { + continue; + } + let mut short_path = t!(original_path.strip_prefix(directory_prefix)); + if !short_path.starts_with(pattern) { + continue; + } + short_path = t!(short_path.strip_prefix(pattern)); + let dst_path = dst.join(short_path); + self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); + if !t!(member.unpack_in(dst)) { + panic!("path traversal attack ??"); + } + if let Some(record) = &mut recorded_entries { + t!(writeln!(record, "{}", short_path.to_str().unwrap())); + } + let src_path = dst.join(original_path); + if src_path.is_dir() && dst_path.exists() { + continue; + } + t!(fs::rename(src_path, dst_path)); + } + let dst_dir = dst.join(directory_prefix); + if dst_dir.exists() { + t!(fs::remove_dir_all(&dst_dir), format!("failed to remove {}", dst_dir.display())); + } + } + + /// Returns whether the SHA256 checksum of `path` matches `expected`. + pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool { + use sha2::Digest; + + self.verbose(&format!("verifying {}", path.display())); + + if self.dry_run() { + return false; + } + + let mut hasher = sha2::Sha256::new(); + + let file = t!(File::open(path)); + let mut reader = BufReader::new(file); + + loop { + let buffer = t!(reader.fill_buf()); + let l = buffer.len(); + // break if EOF + if l == 0 { + break; + } + hasher.update(buffer); + reader.consume(l); + } + + let checksum = hex::encode(hasher.finalize().as_slice()); + let verified = checksum == expected; + + if !verified { + println!( + "invalid checksum: \n\ + found: {checksum}\n\ + expected: {expected}", + ); + } + + verified + } +} + +fn recorded_entries(dst: &Path, pattern: &str) -> Option> { + let name = if pattern == "rustc-dev" { + ".rustc-dev-contents" + } else if pattern.starts_with("rust-std") { + ".rust-std-contents" + } else { + return None; + }; + Some(BufWriter::new(t!(File::create(dst.join(name))))) +} + +enum DownloadSource { + CI, + Dist, +} + +/// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions. +impl Config { + pub(crate) fn maybe_download_rustfmt(&self) -> Option { + let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; + let channel = format!("{version}-{date}"); + + let host = self.build; + let bin_root = self.out.join(host.triple).join("rustfmt"); + let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); + let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); + if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { + return Some(rustfmt_path); + } + + self.download_component( + DownloadSource::Dist, + format!("rustfmt-{version}-{build}.tar.xz", build = host.triple), + "rustfmt-preview", + &date, + "rustfmt", + ); + self.download_component( + DownloadSource::Dist, + format!("rustc-{version}-{build}.tar.xz", build = host.triple), + "rustc", + &date, + "rustfmt", + ); + + if self.should_fix_bins_and_dylibs() { + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); + self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); + let lib_dir = bin_root.join("lib"); + for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { + let lib = t!(lib); + if lib.path().extension() == Some(OsStr::new("so")) { + self.fix_bin_or_dylib(&lib.path()); + } + } + } + + self.create(&rustfmt_stamp, &channel); + Some(rustfmt_path) + } + + pub(crate) fn ci_rust_std_contents(&self) -> Vec { + self.ci_component_contents(".rust-std-contents") + } + + pub(crate) fn ci_rustc_dev_contents(&self) -> Vec { + self.ci_component_contents(".rustc-dev-contents") + } + + fn ci_component_contents(&self, stamp_file: &str) -> Vec { + assert!(self.download_rustc()); + if self.dry_run() { + return vec![]; + } + + let ci_rustc_dir = self.ci_rustc_dir(); + let stamp_file = ci_rustc_dir.join(stamp_file); + let contents_file = t!(File::open(&stamp_file), stamp_file.display().to_string()); + t!(BufReader::new(contents_file).lines().collect()) + } + + pub(crate) fn download_ci_rustc(&self, commit: &str) { + self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); + + let version = self.artifact_version_part(commit); + // download-rustc doesn't need its own cargo, it can just use beta's. But it does need the + // `rustc_private` crates for tools. + let extra_components = ["rustc-dev"]; + + self.download_toolchain( + &version, + "ci-rustc", + &format!("{commit}-{}", self.llvm_assertions), + &extra_components, + Self::download_ci_component, + ); + } + + pub(crate) fn download_beta_toolchain(&self) { + self.verbose("downloading stage0 beta artifacts"); + + let date = &self.stage0_metadata.compiler.date; + let version = &self.stage0_metadata.compiler.version; + let extra_components = ["cargo"]; + + let download_beta_component = |config: &Config, filename, prefix: &_, date: &_| { + config.download_component(DownloadSource::Dist, filename, prefix, date, "stage0") + }; + + self.download_toolchain( + version, + "stage0", + date, + &extra_components, + download_beta_component, + ); + } + + fn download_toolchain( + &self, + version: &str, + sysroot: &str, + stamp_key: &str, + extra_components: &[&str], + download_component: fn(&Config, String, &str, &str), + ) { + let host = self.build.triple; + let bin_root = self.out.join(host).join(sysroot); + let rustc_stamp = bin_root.join(".rustc-stamp"); + + if !bin_root.join("bin").join(exe("rustc", self.build)).exists() + || program_out_of_date(&rustc_stamp, stamp_key) + { + if bin_root.exists() { + t!(fs::remove_dir_all(&bin_root)); + } + let filename = format!("rust-std-{version}-{host}.tar.xz"); + let pattern = format!("rust-std-{host}"); + download_component(self, filename, &pattern, stamp_key); + let filename = format!("rustc-{version}-{host}.tar.xz"); + download_component(self, filename, "rustc", stamp_key); + + for component in extra_components { + let filename = format!("{component}-{version}-{host}.tar.xz"); + download_component(self, filename, component, stamp_key); + } + + if self.should_fix_bins_and_dylibs() { + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); + self.fix_bin_or_dylib( + &bin_root.join("libexec").join("rust-analyzer-proc-macro-srv"), + ); + let lib_dir = bin_root.join("lib"); + for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { + let lib = t!(lib); + if lib.path().extension() == Some(OsStr::new("so")) { + self.fix_bin_or_dylib(&lib.path()); + } + } + } + + t!(fs::write(rustc_stamp, stamp_key)); + } + } + + /// Download a single component of a CI-built toolchain (not necessarily a published nightly). + // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident + fn download_ci_component(&self, filename: String, prefix: &str, commit_with_assertions: &str) { + Self::download_component( + self, + DownloadSource::CI, + filename, + prefix, + commit_with_assertions, + "ci-rustc", + ) + } + + fn download_component( + &self, + mode: DownloadSource, + filename: String, + prefix: &str, + key: &str, + destination: &str, + ) { + let cache_dst = self.out.join("cache"); + let cache_dir = cache_dst.join(key); + if !cache_dir.exists() { + t!(fs::create_dir_all(&cache_dir)); + } + + let bin_root = self.out.join(self.build.triple).join(destination); + let tarball = cache_dir.join(&filename); + let (base_url, url, should_verify) = match mode { + DownloadSource::CI => { + let dist_server = if self.llvm_assertions { + self.stage0_metadata.config.artifacts_with_llvm_assertions_server.clone() + } else { + self.stage0_metadata.config.artifacts_server.clone() + }; + let url = format!( + "{}/{filename}", + key.strip_suffix(&format!("-{}", self.llvm_assertions)).unwrap() + ); + (dist_server, url, false) + } + DownloadSource::Dist => { + let dist_server = env::var("RUSTUP_DIST_SERVER") + .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); + // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json + (dist_server, format!("dist/{key}/{filename}"), true) + } + }; + + // For the beta compiler, put special effort into ensuring the checksums are valid. + // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update + // this on each and every nightly ... + let checksum = if should_verify { + let error = format!( + "src/stage0.json doesn't contain a checksum for {url}. \ + Pre-built artifacts might not be available for this \ + target at this time, see https://doc.rust-lang.org/nightly\ + /rustc/platform-support.html for more information." + ); + let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); + if tarball.exists() { + if self.verify(&tarball, sha256) { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + self.verbose(&format!( + "ignoring cached file {} due to failed verification", + tarball.display() + )); + self.remove(&tarball); + } + } + Some(sha256) + } else if tarball.exists() { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + None + }; + + let mut help_on_error = ""; + if destination == "ci-rustc" { + help_on_error = "ERROR: failed to download pre-built rustc from CI + +NOTE: old builds get deleted after a certain time +HELP: if trying to compile an old commit of rustc, disable `download-rustc` in config.toml: + +[rust] +download-rustc = false +"; + } + self.download_file(&format!("{base_url}/{url}"), &tarball, help_on_error); + if let Some(sha256) = checksum { + if !self.verify(&tarball, sha256) { + panic!("failed to verify {}", tarball.display()); + } + } + + self.unpack(&tarball, &bin_root, prefix); + } + + pub(crate) fn maybe_download_ci_llvm(&self) { + if !self.llvm_from_ci { + return; + } + let llvm_root = self.ci_llvm_root(); + let llvm_stamp = llvm_root.join(".llvm-stamp"); + let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository()); + let key = format!("{}{}", llvm_sha, self.llvm_assertions); + if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { + self.download_ci_llvm(&llvm_sha); + if self.should_fix_bins_and_dylibs() { + for entry in t!(fs::read_dir(llvm_root.join("bin"))) { + self.fix_bin_or_dylib(&t!(entry).path()); + } + } + + // Update the timestamp of llvm-config to force rustc_llvm to be + // rebuilt. This is a hacky workaround for a deficiency in Cargo where + // the rerun-if-changed directive doesn't handle changes very well. + // https://github.com/rust-lang/cargo/issues/10791 + // Cargo only compares the timestamp of the file relative to the last + // time `rustc_llvm` build script ran. However, the timestamps of the + // files in the tarball are in the past, so it doesn't trigger a + // rebuild. + let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); + let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); + t!(filetime::set_file_times(&llvm_config, now, now)); + + if self.should_fix_bins_and_dylibs() { + let llvm_lib = llvm_root.join("lib"); + for entry in t!(fs::read_dir(&llvm_lib)) { + let lib = t!(entry).path(); + if lib.extension().map_or(false, |ext| ext == "so") { + self.fix_bin_or_dylib(&lib); + } + } + } + + t!(fs::write(llvm_stamp, key)); + } + } + + fn download_ci_llvm(&self, llvm_sha: &str) { + let llvm_assertions = self.llvm_assertions; + + let cache_prefix = format!("llvm-{llvm_sha}-{llvm_assertions}"); + let cache_dst = self.out.join("cache"); + let rustc_cache = cache_dst.join(cache_prefix); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + let base = if llvm_assertions { + &self.stage0_metadata.config.artifacts_with_llvm_assertions_server + } else { + &self.stage0_metadata.config.artifacts_server + }; + let version = self.artifact_version_part(llvm_sha); + let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + let help_on_error = "ERROR: failed to download llvm from ci + + HELP: old builds get deleted after a certain time + HELP: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: + + [llvm] + download-ci-llvm = false + "; + self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); + } + let llvm_root = self.ci_llvm_root(); + self.unpack(&tarball, &llvm_root, "rust-dev"); + } +} diff --git a/src/bootstrap/src/core/metadata.rs b/src/bootstrap/src/core/metadata.rs new file mode 100644 index 000000000..580208232 --- /dev/null +++ b/src/bootstrap/src/core/metadata.rs @@ -0,0 +1,101 @@ +use std::path::PathBuf; +use std::process::Command; + +use serde_derive::Deserialize; + +use crate::utils::cache::INTERNER; +use crate::utils::helpers::output; +use crate::{t, Build, Crate}; + +/// For more information, see the output of +/// +#[derive(Debug, Deserialize)] +struct Output { + packages: Vec, +} + +/// For more information, see the output of +/// +#[derive(Debug, Deserialize)] +struct Package { + name: String, + source: Option, + manifest_path: String, + dependencies: Vec, + targets: Vec, +} + +/// For more information, see the output of +/// +#[derive(Debug, Deserialize)] +struct Dependency { + name: String, + source: Option, +} + +#[derive(Debug, Deserialize)] +struct Target { + kind: Vec, +} + +/// Collects and stores package metadata of each workspace members into `build`, +/// by executing `cargo metadata` commands. +pub fn build(build: &mut Build) { + for package in workspace_members(build) { + if package.source.is_none() { + let name = INTERNER.intern_string(package.name); + let mut path = PathBuf::from(package.manifest_path); + path.pop(); + let deps = package + .dependencies + .into_iter() + .filter(|dep| dep.source.is_none()) + .map(|dep| INTERNER.intern_string(dep.name)) + .collect(); + let has_lib = package.targets.iter().any(|t| t.kind.iter().any(|k| k == "lib")); + let krate = Crate { name, deps, path, has_lib }; + let relative_path = krate.local_path(build); + build.crates.insert(name, krate); + let existing_path = build.crate_paths.insert(relative_path, name); + assert!( + existing_path.is_none(), + "multiple crates with the same path: {}", + existing_path.unwrap() + ); + } + } +} + +/// Invokes `cargo metadata` to get package metadata of each workspace member. +/// +/// Note that `src/tools/cargo` is no longer a workspace member but we still +/// treat it as one here, by invoking an additional `cargo metadata` command. +fn workspace_members(build: &Build) -> impl Iterator { + let collect_metadata = |manifest_path| { + let mut cargo = Command::new(&build.initial_cargo); + cargo + // Will read the libstd Cargo.toml + // which uses the unstable `public-dependency` feature. + .env("RUSTC_BOOTSTRAP", "1") + .arg("metadata") + .arg("--format-version") + .arg("1") + .arg("--no-deps") + .arg("--manifest-path") + .arg(build.src.join(manifest_path)); + let metadata_output = output(&mut cargo); + let Output { packages, .. } = t!(serde_json::from_str(&metadata_output)); + packages + }; + + // Collects `metadata.packages` from all workspaces. + let packages = collect_metadata("Cargo.toml"); + let cargo_packages = collect_metadata("src/tools/cargo/Cargo.toml"); + let ra_packages = collect_metadata("src/tools/rust-analyzer/Cargo.toml"); + let bootstrap_packages = collect_metadata("src/bootstrap/Cargo.toml"); + + // We only care about the root package from `src/tool/cargo` workspace. + let cargo_package = cargo_packages.into_iter().find(|pkg| pkg.name == "cargo").into_iter(); + + packages.into_iter().chain(cargo_package).chain(ra_packages).chain(bootstrap_packages) +} diff --git a/src/bootstrap/src/core/mod.rs b/src/bootstrap/src/core/mod.rs new file mode 100644 index 000000000..9e18d6704 --- /dev/null +++ b/src/bootstrap/src/core/mod.rs @@ -0,0 +1,6 @@ +pub(crate) mod build_steps; +pub(crate) mod builder; +pub(crate) mod config; +pub(crate) mod download; +pub(crate) mod metadata; +pub(crate) mod sanity; diff --git a/src/bootstrap/src/core/sanity.rs b/src/bootstrap/src/core/sanity.rs new file mode 100644 index 000000000..eec3be66a --- /dev/null +++ b/src/bootstrap/src/core/sanity.rs @@ -0,0 +1,267 @@ +//! Sanity checking performed by rustbuild before actually executing anything. +//! +//! This module contains the implementation of ensuring that the build +//! environment looks reasonable before progressing. This will verify that +//! various programs like git and python exist, along with ensuring that all C +//! compilers for cross-compiling are found. +//! +//! In theory if we get past this phase it's a bug if a build fails, but in +//! practice that's likely not true! + +use std::collections::HashMap; +use std::env; +use std::ffi::{OsStr, OsString}; +use std::fs; +use std::path::PathBuf; +use std::process::Command; + +use crate::core::config::Target; +use crate::utils::cache::INTERNER; +use crate::utils::helpers::output; +use crate::Build; + +pub struct Finder { + cache: HashMap>, + path: OsString, +} + +impl Finder { + pub fn new() -> Self { + Self { cache: HashMap::new(), path: env::var_os("PATH").unwrap_or_default() } + } + + pub fn maybe_have>(&mut self, cmd: S) -> Option { + let cmd: OsString = cmd.into(); + let path = &self.path; + self.cache + .entry(cmd.clone()) + .or_insert_with(|| { + for path in env::split_paths(path) { + let target = path.join(&cmd); + let mut cmd_exe = cmd.clone(); + cmd_exe.push(".exe"); + + if target.is_file() // some/path/git + || path.join(&cmd_exe).exists() // some/path/git.exe + || target.join(&cmd_exe).exists() + // some/path/git/git.exe + { + return Some(target); + } + } + None + }) + .clone() + } + + pub fn must_have>(&mut self, cmd: S) -> PathBuf { + self.maybe_have(&cmd).unwrap_or_else(|| { + panic!("\n\ncouldn't find required command: {:?}\n\n", cmd.as_ref()); + }) + } +} + +pub fn check(build: &mut Build) { + let skip_target_sanity = + env::var_os("BOOTSTRAP_SKIP_TARGET_SANITY").is_some_and(|s| s == "1" || s == "true"); + + let path = env::var_os("PATH").unwrap_or_default(); + // On Windows, quotes are invalid characters for filename paths, and if + // one is present as part of the PATH then that can lead to the system + // being unable to identify the files properly. See + // https://github.com/rust-lang/rust/issues/34959 for more details. + if cfg!(windows) && path.to_string_lossy().contains('\"') { + panic!("PATH contains invalid character '\"'"); + } + + let mut cmd_finder = Finder::new(); + // If we've got a git directory we're gonna need git to update + // submodules and learn about various other aspects. + if build.rust_info().is_managed_git_subrepository() { + cmd_finder.must_have("git"); + } + + // We need cmake, but only if we're actually building LLVM or sanitizers. + let building_llvm = build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) + && build + .hosts + .iter() + .map(|host| { + build + .config + .target_config + .get(host) + .map(|config| config.llvm_config.is_none()) + .unwrap_or(true) + }) + .any(|build_llvm_ourselves| build_llvm_ourselves); + + let need_cmake = building_llvm || build.config.any_sanitizers_enabled(); + if need_cmake && cmd_finder.maybe_have("cmake").is_none() { + eprintln!( + " +Couldn't find required command: cmake + +You should install cmake, or set `download-ci-llvm = true` in the +`[llvm]` section of `config.toml` to download LLVM rather +than building it. +" + ); + crate::exit!(1); + } + + build.config.python = build + .config + .python + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| env::var_os("BOOTSTRAP_PYTHON").map(PathBuf::from)) // set by bootstrap.py + .or_else(|| cmd_finder.maybe_have("python")) + .or_else(|| cmd_finder.maybe_have("python3")) + .or_else(|| cmd_finder.maybe_have("python2")); + + build.config.nodejs = build + .config + .nodejs + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("node")) + .or_else(|| cmd_finder.maybe_have("nodejs")); + + build.config.npm = build + .config + .npm + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("npm")); + + build.config.gdb = build + .config + .gdb + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("gdb")); + + build.config.reuse = build + .config + .reuse + .take() + .map(|p| cmd_finder.must_have(p)) + .or_else(|| cmd_finder.maybe_have("reuse")); + + // We're gonna build some custom C code here and there, host triples + // also build some C++ shims for LLVM so we need a C++ compiler. + for target in &build.targets { + // On emscripten we don't actually need the C compiler to just + // build the target artifacts, only for testing. For the sake + // of easier bot configuration, just skip detection. + if target.contains("emscripten") { + continue; + } + + // We don't use a C compiler on wasm32 + if target.contains("wasm32") { + continue; + } + + // Some environments don't want or need these tools, such as when testing Miri. + // FIXME: it would be better to refactor this code to split necessary setup from pure sanity + // checks, and have a regular flag for skipping the latter. Also see + // . + if skip_target_sanity { + continue; + } + + if !build.config.dry_run() { + cmd_finder.must_have(build.cc(*target)); + if let Some(ar) = build.ar(*target) { + cmd_finder.must_have(ar); + } + } + } + + for host in &build.hosts { + if !build.config.dry_run() { + cmd_finder.must_have(build.cxx(*host).unwrap()); + } + } + + if build.config.rust_codegen_backends.contains(&INTERNER.intern_str("llvm")) { + // Externally configured LLVM requires FileCheck to exist + let filecheck = build.llvm_filecheck(build.build); + if !filecheck.starts_with(&build.out) && !filecheck.exists() && build.config.codegen_tests { + panic!("FileCheck executable {filecheck:?} does not exist"); + } + } + + for target in &build.targets { + build + .config + .target_config + .entry(*target) + .or_insert_with(|| Target::from_triple(&target.triple)); + + if (target.contains("-none-") || target.contains("nvptx")) + && build.no_std(*target) == Some(false) + { + panic!("All the *-none-* and nvptx* targets are no-std targets") + } + + // Some environments don't want or need these tools, such as when testing Miri. + // FIXME: it would be better to refactor this code to split necessary setup from pure sanity + // checks, and have a regular flag for skipping the latter. Also see + // . + if skip_target_sanity { + continue; + } + + // Make sure musl-root is valid. + if target.contains("musl") && !target.contains("unikraft") { + // If this is a native target (host is also musl) and no musl-root is given, + // fall back to the system toolchain in /usr before giving up + if build.musl_root(*target).is_none() && build.config.build == *target { + let target = build.config.target_config.entry(*target).or_default(); + target.musl_root = Some("/usr".into()); + } + match build.musl_libdir(*target) { + Some(libdir) => { + if fs::metadata(libdir.join("libc.a")).is_err() { + panic!("couldn't find libc.a in musl libdir: {}", libdir.display()); + } + } + None => panic!( + "when targeting MUSL either the rust.musl-root \ + option or the target.$TARGET.musl-root option must \ + be specified in config.toml" + ), + } + } + + if need_cmake && target.contains("msvc") { + // There are three builds of cmake on windows: MSVC, MinGW, and + // Cygwin. The Cygwin build does not have generators for Visual + // Studio, so detect that here and error. + let out = output(Command::new("cmake").arg("--help")); + if !out.contains("Visual Studio") { + panic!( + " +cmake does not support Visual Studio generators. + +This is likely due to it being an msys/cygwin build of cmake, +rather than the required windows version, built using MinGW +or Visual Studio. + +If you are building under msys2 try installing the mingw-w64-x86_64-cmake +package instead of cmake: + +$ pacman -R cmake && pacman -S mingw-w64-x86_64-cmake +" + ); + } + } + } + + if let Some(ref s) = build.config.ccache { + cmd_finder.must_have(s); + } +} diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs new file mode 100644 index 000000000..33b8f1a7c --- /dev/null +++ b/src/bootstrap/src/lib.rs @@ -0,0 +1,1874 @@ +//! Implementation of rustbuild, the Rust build system. +//! +//! This module, and its descendants, are the implementation of the Rust build +//! system. Most of this build system is backed by Cargo but the outer layer +//! here serves as the ability to orchestrate calling Cargo, sequencing Cargo +//! builds, building artifacts like LLVM, etc. The goals of rustbuild are: +//! +//! * To be an easily understandable, easily extensible, and maintainable build +//! system. +//! * Leverage standard tools in the Rust ecosystem to build the compiler, aka +//! crates.io and Cargo. +//! * A standard interface to build across all platforms, including MSVC +//! +//! ## Further information +//! +//! More documentation can be found in each respective module below, and you can +//! also check out the `src/bootstrap/README.md` file for more information. + +use std::cell::{Cell, RefCell}; +use std::collections::{HashMap, HashSet}; +use std::env; +use std::fmt::Display; +use std::fs::{self, File}; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::{Command, Output, Stdio}; +use std::str; + +use build_helper::ci::{gha, CiEnv}; +use build_helper::exit; +use build_helper::util::fail; +use filetime::FileTime; +use once_cell::sync::OnceCell; +use termcolor::{ColorChoice, StandardStream, WriteColor}; +use utils::channel::GitInfo; + +use crate::core::builder; +use crate::core::builder::Kind; +use crate::core::config::flags; +use crate::core::config::{DryRun, Target}; +use crate::core::config::{LlvmLibunwind, TargetSelection}; +use crate::utils::cache::{Interned, INTERNER}; +use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, OutputMode}; +use crate::utils::helpers::{self, dir_is_empty, exe, libdir, mtime, output, symlink_dir}; + +mod core; +mod utils; + +pub use crate::core::builder::PathSet; +pub use crate::core::config::flags::Subcommand; +pub use crate::core::config::Config; + +const LLVM_TOOLS: &[&str] = &[ + "llvm-cov", // used to generate coverage report + "llvm-nm", // used to inspect binaries; it shows symbol names, their sizes and visibility + "llvm-objcopy", // used to transform ELFs into binary format which flashing tools consume + "llvm-objdump", // used to disassemble programs + "llvm-profdata", // used to inspect and merge files generated by profiles + "llvm-readobj", // used to get information from ELFs/objects that the other tools don't provide + "llvm-size", // used to prints the size of the linker sections of a program + "llvm-strip", // used to discard symbols from binary files to reduce their size + "llvm-ar", // used for creating and modifying archive files + "llvm-as", // used to convert LLVM assembly to LLVM bitcode + "llvm-dis", // used to disassemble LLVM bitcode + "llc", // used to compile LLVM bytecode + "opt", // used to optimize LLVM bytecode +]; + +/// LLD file names for all flavors. +const LLD_FILE_NAMES: &[&str] = &["ld.lld", "ld64.lld", "lld-link", "wasm-ld"]; + +/// Keeps track of major changes made to the bootstrap configuration. +/// +/// These values also represent the IDs of the PRs that caused major changes. +/// You can visit `https://github.com/rust-lang/rust/pull/{any-id-from-the-list}` to +/// check for more details regarding each change. +/// +/// If you make any major changes (such as adding new values or changing default values), +/// please ensure that the associated PR ID is added to the end of this list. +/// This is necessary because the list must be sorted by the merge date. +pub const CONFIG_CHANGE_HISTORY: &[usize] = &[115898, 116998, 117435, 116881]; + +/// Extra --check-cfg to add when building +/// (Mode restriction, config name, config values (if any)) +const EXTRA_CHECK_CFGS: &[(Option, &str, Option<&[&'static str]>)] = &[ + (None, "bootstrap", None), + (Some(Mode::Rustc), "parallel_compiler", None), + (Some(Mode::ToolRustc), "parallel_compiler", None), + (Some(Mode::Codegen), "parallel_compiler", None), + (Some(Mode::Std), "stdarch_intel_sde", None), + (Some(Mode::Std), "no_fp_fmt_parse", None), + (Some(Mode::Std), "no_global_oom_handling", None), + (Some(Mode::Std), "no_rc", None), + (Some(Mode::Std), "no_sync", None), + (Some(Mode::Std), "freebsd12", None), + (Some(Mode::Std), "freebsd13", None), + (Some(Mode::Std), "backtrace_in_libstd", None), + /* Extra values not defined in the built-in targets yet, but used in std */ + (Some(Mode::Std), "target_env", Some(&["libnx"])), + // (Some(Mode::Std), "target_os", Some(&[])), + (Some(Mode::Std), "target_arch", Some(&["asmjs", "spirv", "nvptx", "xtensa"])), + /* Extra names used by dependencies */ + // FIXME: Used by serde_json, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "no_btreemap_remove_entry", None), + (Some(Mode::ToolRustc), "no_btreemap_remove_entry", None), + // FIXME: Used by crossbeam-utils, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "crossbeam_loom", None), + (Some(Mode::ToolRustc), "crossbeam_loom", None), + // FIXME: Used by proc-macro2, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "span_locations", None), + (Some(Mode::ToolRustc), "span_locations", None), + // FIXME: Used by rustix, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "rustix_use_libc", None), + (Some(Mode::ToolRustc), "rustix_use_libc", None), + // FIXME: Used by filetime, but we should not be triggering on external dependencies. + (Some(Mode::Rustc), "emulate_second_only_system", None), + (Some(Mode::ToolRustc), "emulate_second_only_system", None), + // Needed to avoid the need to copy windows.lib into the sysroot. + (Some(Mode::Rustc), "windows_raw_dylib", None), + (Some(Mode::ToolRustc), "windows_raw_dylib", None), +]; + +/// A structure representing a Rust compiler. +/// +/// Each compiler has a `stage` that it is associated with and a `host` that +/// corresponds to the platform the compiler runs on. This structure is used as +/// a parameter to many methods below. +#[derive(Eq, PartialOrd, Ord, PartialEq, Clone, Copy, Hash, Debug)] +pub struct Compiler { + stage: u32, + host: TargetSelection, +} + +#[derive(PartialEq, Eq, Copy, Clone, Debug)] +pub enum DocTests { + /// Run normal tests and doc tests (default). + Yes, + /// Do not run any doc tests. + No, + /// Only run doc tests. + Only, +} + +pub enum GitRepo { + Rustc, + Llvm, +} + +/// Global configuration for the build system. +/// +/// This structure transitively contains all configuration for the build system. +/// All filesystem-encoded configuration is in `config`, all flags are in +/// `flags`, and then parsed or probed information is listed in the keys below. +/// +/// This structure is a parameter of almost all methods in the build system, +/// although most functions are implemented as free functions rather than +/// methods specifically on this structure itself (to make it easier to +/// organize). +#[derive(Clone)] +pub struct Build { + /// User-specified configuration from `config.toml`. + config: Config, + + // Version information + version: String, + + // Properties derived from the above configuration + src: PathBuf, + out: PathBuf, + bootstrap_out: PathBuf, + cargo_info: GitInfo, + rust_analyzer_info: GitInfo, + clippy_info: GitInfo, + miri_info: GitInfo, + rustfmt_info: GitInfo, + in_tree_llvm_info: GitInfo, + local_rebuild: bool, + fail_fast: bool, + doc_tests: DocTests, + verbosity: usize, + + // Targets for which to build + build: TargetSelection, + hosts: Vec, + targets: Vec, + + initial_rustc: PathBuf, + initial_cargo: PathBuf, + initial_lld: PathBuf, + initial_libdir: PathBuf, + initial_sysroot: PathBuf, + + // Runtime state filled in later on + // C/C++ compilers and archiver for all targets + cc: RefCell>, + cxx: RefCell>, + ar: RefCell>, + ranlib: RefCell>, + // Miscellaneous + // allow bidirectional lookups: both name -> path and path -> name + crates: HashMap, Crate>, + crate_paths: HashMap>, + is_sudo: bool, + ci_env: CiEnv, + delayed_failures: RefCell>, + prerelease_version: Cell>, + + #[cfg(feature = "build-metrics")] + metrics: crate::utils::metrics::BuildMetrics, +} + +#[derive(Debug, Clone)] +struct Crate { + name: Interned, + deps: HashSet>, + path: PathBuf, + has_lib: bool, +} + +impl Crate { + fn local_path(&self, build: &Build) -> PathBuf { + self.path.strip_prefix(&build.config.src).unwrap().into() + } +} + +/// When building Rust various objects are handled differently. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum DependencyType { + /// Libraries originating from proc-macros. + Host, + /// Typical Rust libraries. + Target, + /// Non Rust libraries and objects shipped to ease usage of certain targets. + TargetSelfContained, +} + +/// The various "modes" of invoking Cargo. +/// +/// These entries currently correspond to the various output directories of the +/// build system, with each mod generating output in a different directory. +#[derive(Debug, Hash, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Mode { + /// Build the standard library, placing output in the "stageN-std" directory. + Std, + + /// Build librustc, and compiler libraries, placing output in the "stageN-rustc" directory. + Rustc, + + /// Build a codegen backend for rustc, placing the output in the "stageN-codegen" directory. + Codegen, + + /// Build a tool, placing output in the "stage0-bootstrap-tools" + /// directory. This is for miscellaneous sets of tools that are built + /// using the bootstrap stage0 compiler in its entirety (target libraries + /// and all). Typically these tools compile with stable Rust. + ToolBootstrap, + + /// Build a tool which uses the locally built std, placing output in the + /// "stageN-tools" directory. Its usage is quite rare, mainly used by + /// compiletest which needs libtest. + ToolStd, + + /// Build a tool which uses the locally built rustc and the target std, + /// placing the output in the "stageN-tools" directory. This is used for + /// anything that needs a fully functional rustc, such as rustdoc, clippy, + /// cargo, rls, rustfmt, miri, etc. + ToolRustc, +} + +impl Mode { + pub fn is_tool(&self) -> bool { + matches!(self, Mode::ToolBootstrap | Mode::ToolRustc | Mode::ToolStd) + } + + pub fn must_support_dlopen(&self) -> bool { + matches!(self, Mode::Std | Mode::Codegen) + } +} + +pub enum CLang { + C, + Cxx, +} + +macro_rules! forward { + ( $( $fn:ident( $($param:ident: $ty:ty),* ) $( -> $ret:ty)? ),+ $(,)? ) => { + impl Build { + $( fn $fn(&self, $($param: $ty),* ) $( -> $ret)? { + self.config.$fn( $($param),* ) + } )+ + } + } +} + +forward! { + verbose(msg: &str), + is_verbose() -> bool, + create(path: &Path, s: &str), + remove(f: &Path), + tempdir() -> PathBuf, + llvm_link_shared() -> bool, + download_rustc() -> bool, + initial_rustfmt() -> Option, +} + +impl Build { + /// Creates a new set of build configuration from the `flags` on the command + /// line and the filesystem `config`. + /// + /// By default all build output will be placed in the current directory. + pub fn new(mut config: Config) -> Build { + let src = config.src.clone(); + let out = config.out.clone(); + + #[cfg(unix)] + // keep this consistent with the equivalent check in x.py: + // https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/bootstrap.py#L796-L797 + let is_sudo = match env::var_os("SUDO_USER") { + Some(_sudo_user) => { + // SAFETY: getuid() system call is always successful and no return value is reserved + // to indicate an error. + // + // For more context, see https://man7.org/linux/man-pages/man2/geteuid.2.html + let uid = unsafe { libc::getuid() }; + uid == 0 + } + None => false, + }; + #[cfg(not(unix))] + let is_sudo = false; + + let omit_git_hash = config.omit_git_hash; + let rust_info = GitInfo::new(omit_git_hash, &src); + let cargo_info = GitInfo::new(omit_git_hash, &src.join("src/tools/cargo")); + let rust_analyzer_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rust-analyzer")); + let clippy_info = GitInfo::new(omit_git_hash, &src.join("src/tools/clippy")); + let miri_info = GitInfo::new(omit_git_hash, &src.join("src/tools/miri")); + let rustfmt_info = GitInfo::new(omit_git_hash, &src.join("src/tools/rustfmt")); + + // we always try to use git for LLVM builds + let in_tree_llvm_info = GitInfo::new(false, &src.join("src/llvm-project")); + + let initial_target_libdir_str = if config.dry_run() { + "/dummy/lib/path/to/lib/".to_string() + } else { + output( + Command::new(&config.initial_rustc) + .arg("--target") + .arg(config.build.rustc_target_arg()) + .arg("--print") + .arg("target-libdir"), + ) + }; + let initial_target_dir = Path::new(&initial_target_libdir_str).parent().unwrap(); + let initial_lld = initial_target_dir.join("bin").join("rust-lld"); + + let initial_sysroot = if config.dry_run() { + "/dummy".to_string() + } else { + output(Command::new(&config.initial_rustc).arg("--print").arg("sysroot")) + } + .trim() + .to_string(); + + let initial_libdir = initial_target_dir + .parent() + .unwrap() + .parent() + .unwrap() + .strip_prefix(&initial_sysroot) + .unwrap() + .to_path_buf(); + + let version = std::fs::read_to_string(src.join("src").join("version")) + .expect("failed to read src/version"); + let version = version.trim(); + + let bootstrap_out = std::env::current_exe() + .expect("could not determine path to running process") + .parent() + .unwrap() + .to_path_buf(); + if !bootstrap_out.join(exe("rustc", config.build)).exists() && !cfg!(test) { + // this restriction can be lifted whenever https://github.com/rust-lang/rfcs/pull/3028 is implemented + panic!( + "`rustc` not found in {}, run `cargo build --bins` before `cargo run`", + bootstrap_out.display() + ) + } + + if rust_info.is_from_tarball() && config.description.is_none() { + config.description = Some("built from a source tarball".to_owned()); + } + + let mut build = Build { + initial_rustc: config.initial_rustc.clone(), + initial_cargo: config.initial_cargo.clone(), + initial_lld, + initial_libdir, + initial_sysroot: initial_sysroot.into(), + local_rebuild: config.local_rebuild, + fail_fast: config.cmd.fail_fast(), + doc_tests: config.cmd.doc_tests(), + verbosity: config.verbose, + + build: config.build, + hosts: config.hosts.clone(), + targets: config.targets.clone(), + + config, + version: version.to_string(), + src, + out, + bootstrap_out, + + cargo_info, + rust_analyzer_info, + clippy_info, + miri_info, + rustfmt_info, + in_tree_llvm_info, + cc: RefCell::new(HashMap::new()), + cxx: RefCell::new(HashMap::new()), + ar: RefCell::new(HashMap::new()), + ranlib: RefCell::new(HashMap::new()), + crates: HashMap::new(), + crate_paths: HashMap::new(), + is_sudo, + ci_env: CiEnv::current(), + delayed_failures: RefCell::new(Vec::new()), + prerelease_version: Cell::new(None), + + #[cfg(feature = "build-metrics")] + metrics: crate::utils::metrics::BuildMetrics::init(), + }; + + // If local-rust is the same major.minor as the current version, then force a + // local-rebuild + let local_version_verbose = + output(Command::new(&build.initial_rustc).arg("--version").arg("--verbose")); + let local_release = local_version_verbose + .lines() + .filter_map(|x| x.strip_prefix("release:")) + .next() + .unwrap() + .trim(); + if local_release.split('.').take(2).eq(version.split('.').take(2)) { + build.verbose(&format!("auto-detected local-rebuild {local_release}")); + build.local_rebuild = true; + } + + build.verbose("finding compilers"); + utils::cc_detect::find(&build); + // When running `setup`, the profile is about to change, so any requirements we have now may + // be different on the next invocation. Don't check for them until the next time x.py is + // run. This is ok because `setup` never runs any build commands, so it won't fail if commands are missing. + // + // Similarly, for `setup` we don't actually need submodules or cargo metadata. + if !matches!(build.config.cmd, Subcommand::Setup { .. }) { + build.verbose("running sanity check"); + crate::core::sanity::check(&mut build); + + // Make sure we update these before gathering metadata so we don't get an error about missing + // Cargo.toml files. + let rust_submodules = ["src/tools/cargo", "library/backtrace", "library/stdarch"]; + for s in rust_submodules { + build.update_submodule(Path::new(s)); + } + // Now, update all existing submodules. + build.update_existing_submodules(); + + build.verbose("learning about cargo"); + crate::core::metadata::build(&mut build); + } + + // Make a symbolic link so we can use a consistent directory in the documentation. + let build_triple = build.out.join(&build.build.triple); + t!(fs::create_dir_all(&build_triple)); + let host = build.out.join("host"); + if host.is_symlink() { + // Left over from a previous build; overwrite it. + // This matters if `build.build` has changed between invocations. + #[cfg(windows)] + t!(fs::remove_dir(&host)); + #[cfg(not(windows))] + t!(fs::remove_file(&host)); + } + t!( + symlink_dir(&build.config, &build_triple, &host), + format!("symlink_dir({} => {}) failed", host.display(), build_triple.display()) + ); + + build + } + + // modified from `check_submodule` and `update_submodule` in bootstrap.py + /// Given a path to the directory of a submodule, update it. + /// + /// `relative_path` should be relative to the root of the git repository, not an absolute path. + pub(crate) fn update_submodule(&self, relative_path: &Path) { + if !self.config.submodules(&self.rust_info()) { + return; + } + + let absolute_path = self.config.src.join(relative_path); + + // NOTE: The check for the empty directory is here because when running x.py the first time, + // the submodule won't be checked out. Check it out now so we can build it. + if !GitInfo::new(false, &absolute_path).is_managed_git_subrepository() + && !dir_is_empty(&absolute_path) + { + return; + } + + // check_submodule + let checked_out_hash = + output(Command::new("git").args(&["rev-parse", "HEAD"]).current_dir(&absolute_path)); + // update_submodules + let recorded = output( + Command::new("git") + .args(&["ls-tree", "HEAD"]) + .arg(relative_path) + .current_dir(&self.config.src), + ); + let actual_hash = recorded + .split_whitespace() + .nth(2) + .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); + + // update_submodule + if actual_hash == checked_out_hash.trim_end() { + // already checked out + return; + } + + println!("Updating submodule {}", relative_path.display()); + self.run( + Command::new("git") + .args(&["submodule", "-q", "sync"]) + .arg(relative_path) + .current_dir(&self.config.src), + ); + + // Try passing `--progress` to start, then run git again without if that fails. + let update = |progress: bool| { + // Git is buggy and will try to fetch submodules from the tracking branch for *this* repository, + // even though that has no relation to the upstream for the submodule. + let current_branch = { + let output = self + .config + .git() + .args(["symbolic-ref", "--short", "HEAD"]) + .stderr(Stdio::inherit()) + .output(); + let output = t!(output); + if output.status.success() { + Some(String::from_utf8(output.stdout).unwrap().trim().to_owned()) + } else { + None + } + }; + + let mut git = self.config.git(); + if let Some(branch) = current_branch { + // If there is a tag named after the current branch, git will try to disambiguate by prepending `heads/` to the branch name. + // This syntax isn't accepted by `branch.{branch}`. Strip it. + let branch = branch.strip_prefix("heads/").unwrap_or(&branch); + git.arg("-c").arg(format!("branch.{branch}.remote=origin")); + } + git.args(&["submodule", "update", "--init", "--recursive", "--depth=1"]); + if progress { + git.arg("--progress"); + } + git.arg(relative_path); + git + }; + // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails. + if !update(true).status().map_or(false, |status| status.success()) { + self.run(&mut update(false)); + } + + // Save any local changes, but avoid running `git stash pop` if there are none (since it will exit with an error). + // diff-index reports the modifications through the exit status + let has_local_modifications = !self.run_cmd( + BootstrapCommand::from( + Command::new("git") + .args(&["diff-index", "--quiet", "HEAD"]) + .current_dir(&absolute_path), + ) + .allow_failure() + .output_mode(match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }), + ); + if has_local_modifications { + self.run(Command::new("git").args(&["stash", "push"]).current_dir(&absolute_path)); + } + + self.run(Command::new("git").args(&["reset", "-q", "--hard"]).current_dir(&absolute_path)); + self.run(Command::new("git").args(&["clean", "-qdfx"]).current_dir(&absolute_path)); + + if has_local_modifications { + self.run(Command::new("git").args(&["stash", "pop"]).current_dir(absolute_path)); + } + } + + /// If any submodule has been initialized already, sync it unconditionally. + /// This avoids contributors checking in a submodule change by accident. + pub fn update_existing_submodules(&self) { + // Avoid running git when there isn't a git checkout. + if !self.config.submodules(&self.rust_info()) { + return; + } + let output = output( + self.config + .git() + .args(&["config", "--file"]) + .arg(&self.config.src.join(".gitmodules")) + .args(&["--get-regexp", "path"]), + ); + for line in output.lines() { + // Look for `submodule.$name.path = $path` + // Sample output: `submodule.src/rust-installer.path src/tools/rust-installer` + let submodule = Path::new(line.splitn(2, ' ').nth(1).unwrap()); + // Don't update the submodule unless it's already been cloned. + if GitInfo::new(false, submodule).is_managed_git_subrepository() { + self.update_submodule(submodule); + } + } + } + + /// Executes the entire build, as configured by the flags and configuration. + pub fn build(&mut self) { + unsafe { + crate::utils::job::setup(self); + } + + // Download rustfmt early so that it can be used in rust-analyzer configs. + let _ = &builder::Builder::new(&self).initial_rustfmt(); + + // hardcoded subcommands + match &self.config.cmd { + Subcommand::Format { check } => { + return core::build_steps::format::format( + &builder::Builder::new(&self), + *check, + &self.config.paths, + ); + } + Subcommand::Suggest { run } => { + return core::build_steps::suggest::suggest(&builder::Builder::new(&self), *run); + } + _ => (), + } + + { + let builder = builder::Builder::new(&self); + if let Some(path) = builder.paths.get(0) { + if path == Path::new("nonexistent/path/to/trigger/cargo/metadata") { + return; + } + } + } + + if !self.config.dry_run() { + { + self.config.dry_run = DryRun::SelfCheck; + let builder = builder::Builder::new(&self); + builder.execute_cli(); + } + self.config.dry_run = DryRun::Disabled; + let builder = builder::Builder::new(&self); + builder.execute_cli(); + } else { + let builder = builder::Builder::new(&self); + builder.execute_cli(); + } + + // Check for postponed failures from `test --no-fail-fast`. + let failures = self.delayed_failures.borrow(); + if failures.len() > 0 { + eprintln!("\n{} command(s) did not execute successfully:\n", failures.len()); + for failure in failures.iter() { + eprintln!(" - {failure}\n"); + } + exit!(1); + } + + #[cfg(feature = "build-metrics")] + self.metrics.persist(self); + } + + /// Clear out `dir` if `input` is newer. + /// + /// After this executes, it will also ensure that `dir` exists. + fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { + let stamp = dir.join(".stamp"); + let mut cleared = false; + if mtime(&stamp) < mtime(input) { + self.verbose(&format!("Dirty - {}", dir.display())); + let _ = fs::remove_dir_all(dir); + cleared = true; + } else if stamp.exists() { + return cleared; + } + t!(fs::create_dir_all(dir)); + t!(File::create(stamp)); + cleared + } + + fn rust_info(&self) -> &GitInfo { + &self.config.rust_info + } + + /// Gets the space-separated set of activated features for the standard + /// library. + fn std_features(&self, target: TargetSelection) -> String { + let mut features = " panic-unwind".to_string(); + + match self.config.llvm_libunwind(target) { + LlvmLibunwind::InTree => features.push_str(" llvm-libunwind"), + LlvmLibunwind::System => features.push_str(" system-llvm-libunwind"), + LlvmLibunwind::No => {} + } + if self.config.backtrace { + features.push_str(" backtrace"); + } + if self.config.profiler_enabled(target) { + features.push_str(" profiler"); + } + features + } + + /// Gets the space-separated set of activated features for the compiler. + fn rustc_features(&self, kind: Kind) -> String { + let mut features = vec![]; + if self.config.jemalloc { + features.push("jemalloc"); + } + if self.config.llvm_enabled() || kind == Kind::Check { + features.push("llvm"); + } + // keep in sync with `bootstrap/compile.rs:rustc_cargo_env` + if self.config.rustc_parallel { + features.push("rustc_use_parallel_compiler"); + } + + // If debug logging is on, then we want the default for tracing: + // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26 + // which is everything (including debug/trace/etc.) + // if its unset, if debug_assertions is on, then debug_logging will also be on + // as well as tracing *ignoring* this feature when debug_assertions is on + if !self.config.rust_debug_logging { + features.push("max_level_info"); + } + + features.join(" ") + } + + /// Component directory that Cargo will produce output into (e.g. + /// release/debug) + fn cargo_dir(&self) -> &'static str { + if self.config.rust_optimize.is_release() { "release" } else { "debug" } + } + + fn tools_dir(&self, compiler: Compiler) -> PathBuf { + let out = self + .out + .join(&*compiler.host.triple) + .join(format!("stage{}-tools-bin", compiler.stage)); + t!(fs::create_dir_all(&out)); + out + } + + /// Returns the root directory for all output generated in a particular + /// stage when running with a particular host compiler. + /// + /// The mode indicates what the root directory is for. + fn stage_out(&self, compiler: Compiler, mode: Mode) -> PathBuf { + let suffix = match mode { + Mode::Std => "-std", + Mode::Rustc => "-rustc", + Mode::Codegen => "-codegen", + Mode::ToolBootstrap => "-bootstrap-tools", + Mode::ToolStd | Mode::ToolRustc => "-tools", + }; + self.out.join(&*compiler.host.triple).join(format!("stage{}{}", compiler.stage, suffix)) + } + + /// Returns the root output directory for all Cargo output in a given stage, + /// running a particular compiler, whether or not we're building the + /// standard library, and targeting the specified architecture. + fn cargo_out(&self, compiler: Compiler, mode: Mode, target: TargetSelection) -> PathBuf { + self.stage_out(compiler, mode).join(&*target.triple).join(self.cargo_dir()) + } + + /// Root output directory for LLVM compiled for `target` + /// + /// Note that if LLVM is configured externally then the directory returned + /// will likely be empty. + fn llvm_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("llvm") + } + + fn lld_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("lld") + } + + /// Output directory for all documentation for a target + fn doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("doc") + } + + /// Output directory for all JSON-formatted documentation for a target + fn json_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("json-doc") + } + + fn test_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("test") + } + + /// Output directory for all documentation for a target + fn compiler_doc_out(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("compiler-doc") + } + + /// Output directory for some generated md crate documentation for a target (temporary) + fn md_doc_out(&self, target: TargetSelection) -> Interned { + INTERNER.intern_path(self.out.join(&*target.triple).join("md-doc")) + } + + /// Returns `true` if no custom `llvm-config` is set for the specified target. + /// + /// If no custom `llvm-config` was specified then Rust's llvm will be used. + fn is_rust_llvm(&self, target: TargetSelection) -> bool { + match self.config.target_config.get(&target) { + Some(Target { llvm_has_rust_patches: Some(patched), .. }) => *patched, + Some(Target { llvm_config, .. }) => { + // If the user set llvm-config we assume Rust is not patched, + // but first check to see if it was configured by llvm-from-ci. + (self.config.llvm_from_ci && target == self.config.build) || llvm_config.is_none() + } + None => true, + } + } + + /// Returns the path to `FileCheck` binary for the specified target + fn llvm_filecheck(&self, target: TargetSelection) -> PathBuf { + let target_config = self.config.target_config.get(&target); + if let Some(s) = target_config.and_then(|c| c.llvm_filecheck.as_ref()) { + s.to_path_buf() + } else if let Some(s) = target_config.and_then(|c| c.llvm_config.as_ref()) { + let llvm_bindir = output(Command::new(s).arg("--bindir")); + let filecheck = Path::new(llvm_bindir.trim()).join(exe("FileCheck", target)); + if filecheck.exists() { + filecheck + } else { + // On Fedora the system LLVM installs FileCheck in the + // llvm subdirectory of the libdir. + let llvm_libdir = output(Command::new(s).arg("--libdir")); + let lib_filecheck = + Path::new(llvm_libdir.trim()).join("llvm").join(exe("FileCheck", target)); + if lib_filecheck.exists() { + lib_filecheck + } else { + // Return the most normal file name, even though + // it doesn't exist, so that any error message + // refers to that. + filecheck + } + } + } else { + let base = self.llvm_out(target).join("build"); + let base = if !self.ninja() && target.contains("msvc") { + if self.config.llvm_optimize { + if self.config.llvm_release_debuginfo { + base.join("RelWithDebInfo") + } else { + base.join("Release") + } + } else { + base.join("Debug") + } + } else { + base + }; + base.join("bin").join(exe("FileCheck", target)) + } + } + + /// Directory for libraries built from C/C++ code and shared between stages. + fn native_dir(&self, target: TargetSelection) -> PathBuf { + self.out.join(&*target.triple).join("native") + } + + /// Root output directory for rust_test_helpers library compiled for + /// `target` + fn test_helpers_out(&self, target: TargetSelection) -> PathBuf { + self.native_dir(target).join("rust-test-helpers") + } + + /// Adds the `RUST_TEST_THREADS` env var if necessary + fn add_rust_test_threads(&self, cmd: &mut Command) { + if env::var_os("RUST_TEST_THREADS").is_none() { + cmd.env("RUST_TEST_THREADS", self.jobs().to_string()); + } + } + + /// Returns the libdir of the snapshot compiler. + fn rustc_snapshot_libdir(&self) -> PathBuf { + self.rustc_snapshot_sysroot().join(libdir(self.config.build)) + } + + /// Returns the sysroot of the snapshot compiler. + fn rustc_snapshot_sysroot(&self) -> &Path { + static SYSROOT_CACHE: OnceCell = once_cell::sync::OnceCell::new(); + SYSROOT_CACHE.get_or_init(|| { + let mut rustc = Command::new(&self.initial_rustc); + rustc.args(&["--print", "sysroot"]); + output(&mut rustc).trim().into() + }) + } + + /// Runs a command, printing out nice contextual information if it fails. + fn run(&self, cmd: &mut Command) { + self.run_cmd(BootstrapCommand::from(cmd).fail_fast().output_mode( + match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }, + )); + } + + /// Runs a command, printing out contextual info if it fails, and delaying errors until the build finishes. + pub(crate) fn run_delaying_failure(&self, cmd: &mut Command) -> bool { + self.run_cmd(BootstrapCommand::from(cmd).delay_failure().output_mode( + match self.is_verbose() { + true => OutputMode::PrintAll, + false => OutputMode::PrintOutput, + }, + )) + } + + /// Runs a command, printing out nice contextual information if it fails. + fn run_quiet(&self, cmd: &mut Command) { + self.run_cmd( + BootstrapCommand::from(cmd).fail_fast().output_mode(OutputMode::SuppressOnSuccess), + ); + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Exits if the command failed to execute at all, otherwise returns its + /// `status.success()`. + fn run_quiet_delaying_failure(&self, cmd: &mut Command) -> bool { + self.run_cmd( + BootstrapCommand::from(cmd).delay_failure().output_mode(OutputMode::SuppressOnSuccess), + ) + } + + /// A centralized function for running commands that do not return output. + pub(crate) fn run_cmd<'a, C: Into>>(&self, cmd: C) -> bool { + if self.config.dry_run() { + return true; + } + + let command = cmd.into(); + self.verbose(&format!("running: {command:?}")); + + let (output, print_error) = match command.output_mode { + mode @ (OutputMode::PrintAll | OutputMode::PrintOutput) => ( + command.command.status().map(|status| Output { + status, + stdout: Vec::new(), + stderr: Vec::new(), + }), + matches!(mode, OutputMode::PrintAll), + ), + OutputMode::SuppressOnSuccess => (command.command.output(), true), + }; + + let output = match output { + Ok(output) => output, + Err(e) => fail(&format!("failed to execute command: {:?}\nerror: {}", command, e)), + }; + let result = if !output.status.success() { + if print_error { + println!( + "\n\ncommand did not execute successfully: {:?}\n\ + expected success, got: {}\n\n\ + stdout ----\n{}\n\ + stderr ----\n{}\n\n", + command.command, + output.status, + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + ); + } + Err(()) + } else { + Ok(()) + }; + + match result { + Ok(_) => true, + Err(_) => { + match command.failure_behavior { + BehaviorOnFailure::DelayFail => { + if self.fail_fast { + exit!(1); + } + + let mut failures = self.delayed_failures.borrow_mut(); + failures.push(format!("{command:?}")); + } + BehaviorOnFailure::Exit => { + exit!(1); + } + BehaviorOnFailure::Ignore => {} + } + false + } + } + } + + pub fn is_verbose_than(&self, level: usize) -> bool { + self.verbosity > level + } + + /// Prints a message if this build is configured in more verbose mode than `level`. + fn verbose_than(&self, level: usize, msg: &str) { + if self.is_verbose_than(level) { + println!("{msg}"); + } + } + + fn info(&self, msg: &str) { + match self.config.dry_run { + DryRun::SelfCheck => (), + DryRun::Disabled | DryRun::UserSelected => { + println!("{msg}"); + } + } + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_check( + &self, + what: impl Display, + target: impl Into>, + ) -> Option { + self.msg(Kind::Check, self.config.stage, what, self.config.build, target) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_doc( + &self, + compiler: Compiler, + what: impl Display, + target: impl Into> + Copy, + ) -> Option { + self.msg(Kind::Doc, compiler.stage, what, compiler.host, target.into()) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_build( + &self, + compiler: Compiler, + what: impl Display, + target: impl Into>, + ) -> Option { + self.msg(Kind::Build, compiler.stage, what, compiler.host, target) + } + + /// Return a `Group` guard for a [`Step`] that is built for each `--stage`. + /// + /// [`Step`]: crate::core::builder::Step + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg( + &self, + action: impl Into, + stage: u32, + what: impl Display, + host: impl Into>, + target: impl Into>, + ) -> Option { + let action = action.into().description(); + let msg = |fmt| format!("{action} stage{stage} {what}{fmt}"); + let msg = if let Some(target) = target.into() { + let host = host.into().unwrap(); + if host == target { + msg(format_args!(" ({target})")) + } else { + msg(format_args!(" ({host} -> {target})")) + } + } else { + msg(format_args!("")) + }; + self.group(&msg) + } + + /// Return a `Group` guard for a [`Step`] that is only built once and isn't affected by `--stage`. + /// + /// [`Step`]: crate::core::builder::Step + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_unstaged( + &self, + action: impl Into, + what: impl Display, + target: TargetSelection, + ) -> Option { + let action = action.into().description(); + let msg = format!("{action} {what} for {target}"); + self.group(&msg) + } + + #[must_use = "Groups should not be dropped until the Step finishes running"] + #[track_caller] + fn msg_sysroot_tool( + &self, + action: impl Into, + stage: u32, + what: impl Display, + host: TargetSelection, + target: TargetSelection, + ) -> Option { + let action = action.into().description(); + let msg = |fmt| format!("{action} {what} {fmt}"); + let msg = if host == target { + msg(format_args!("(stage{stage} -> stage{}, {target})", stage + 1)) + } else { + msg(format_args!("(stage{stage}:{host} -> stage{}:{target})", stage + 1)) + }; + self.group(&msg) + } + + #[track_caller] + fn group(&self, msg: &str) -> Option { + match self.config.dry_run { + DryRun::SelfCheck => None, + DryRun::Disabled | DryRun::UserSelected => Some(gha::group(&msg)), + } + } + + /// Returns the number of parallel jobs that have been configured for this + /// build. + fn jobs(&self) -> u32 { + self.config.jobs.unwrap_or_else(|| { + std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 + }) + } + + fn debuginfo_map_to(&self, which: GitRepo) -> Option { + if !self.config.rust_remap_debuginfo { + return None; + } + + match which { + GitRepo::Rustc => { + let sha = self.rust_sha().unwrap_or(&self.version); + Some(format!("/rustc/{sha}")) + } + GitRepo::Llvm => Some(String::from("/rustc/llvm")), + } + } + + /// Returns the path to the C compiler for the target specified. + fn cc(&self, target: TargetSelection) -> PathBuf { + if self.config.dry_run() { + return PathBuf::new(); + } + self.cc.borrow()[&target].path().into() + } + + /// Returns a list of flags to pass to the C compiler for the target + /// specified. + fn cflags(&self, target: TargetSelection, which: GitRepo, c: CLang) -> Vec { + if self.config.dry_run() { + return Vec::new(); + } + let base = match c { + CLang::C => self.cc.borrow()[&target].clone(), + CLang::Cxx => self.cxx.borrow()[&target].clone(), + }; + + // Filter out -O and /O (the optimization flags) that we picked up from + // cc-rs because the build scripts will determine that for themselves. + let mut base = base + .args() + .iter() + .map(|s| s.to_string_lossy().into_owned()) + .filter(|s| !s.starts_with("-O") && !s.starts_with("/O")) + .collect::>(); + + // If we're compiling C++ on macOS then we add a flag indicating that + // we want libc++ (more filled out than libstdc++), ensuring that + // LLVM/etc are all properly compiled. + if matches!(c, CLang::Cxx) && target.contains("apple-darwin") { + base.push("-stdlib=libc++".into()); + } + + // Work around an apparently bad MinGW / GCC optimization, + // See: https://lists.llvm.org/pipermail/cfe-dev/2016-December/051980.html + // See: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=78936 + if &*target.triple == "i686-pc-windows-gnu" { + base.push("-fno-omit-frame-pointer".into()); + } + + if let Some(map_to) = self.debuginfo_map_to(which) { + let map = format!("{}={}", self.src.display(), map_to); + let cc = self.cc(target); + if cc.ends_with("clang") || cc.ends_with("gcc") { + base.push(format!("-fdebug-prefix-map={map}")); + } else if cc.ends_with("clang-cl.exe") { + base.push("-Xclang".into()); + base.push(format!("-fdebug-prefix-map={map}")); + } + } + base + } + + /// Returns the path to the `ar` archive utility for the target specified. + fn ar(&self, target: TargetSelection) -> Option { + if self.config.dry_run() { + return None; + } + self.ar.borrow().get(&target).cloned() + } + + /// Returns the path to the `ranlib` utility for the target specified. + fn ranlib(&self, target: TargetSelection) -> Option { + if self.config.dry_run() { + return None; + } + self.ranlib.borrow().get(&target).cloned() + } + + /// Returns the path to the C++ compiler for the target specified. + fn cxx(&self, target: TargetSelection) -> Result { + if self.config.dry_run() { + return Ok(PathBuf::new()); + } + match self.cxx.borrow().get(&target) { + Some(p) => Ok(p.path().into()), + None => Err(format!("target `{target}` is not configured as a host, only as a target")), + } + } + + /// Returns the path to the linker for the given target if it needs to be overridden. + fn linker(&self, target: TargetSelection) -> Option { + if self.config.dry_run() { + return Some(PathBuf::new()); + } + if let Some(linker) = self.config.target_config.get(&target).and_then(|c| c.linker.clone()) + { + Some(linker) + } else if target.contains("vxworks") { + // need to use CXX compiler as linker to resolve the exception functions + // that are only existed in CXX libraries + Some(self.cxx.borrow()[&target].path().into()) + } else if target != self.config.build + && helpers::use_host_linker(target) + && !target.contains("msvc") + { + Some(self.cc(target)) + } else if self.config.use_lld && !self.is_fuse_ld_lld(target) && self.build == target { + Some(self.initial_lld.clone()) + } else { + None + } + } + + // LLD is used through `-fuse-ld=lld` rather than directly. + // Only MSVC targets use LLD directly at the moment. + fn is_fuse_ld_lld(&self, target: TargetSelection) -> bool { + self.config.use_lld && !target.contains("msvc") + } + + fn lld_flags(&self, target: TargetSelection) -> impl Iterator { + let mut options = [None, None]; + + if self.config.use_lld { + if self.is_fuse_ld_lld(target) { + options[0] = Some("-Clink-arg=-fuse-ld=lld".to_string()); + } + + let no_threads = helpers::lld_flag_no_threads(target.contains("windows")); + options[1] = Some(format!("-Clink-arg=-Wl,{no_threads}")); + } + + IntoIterator::into_iter(options).flatten() + } + + /// Returns if this target should statically link the C runtime, if specified + fn crt_static(&self, target: TargetSelection) -> Option { + if target.contains("pc-windows-msvc") { + Some(true) + } else { + self.config.target_config.get(&target).and_then(|t| t.crt_static) + } + } + + /// Returns the "musl root" for this `target`, if defined + fn musl_root(&self, target: TargetSelection) -> Option<&Path> { + self.config + .target_config + .get(&target) + .and_then(|t| t.musl_root.as_ref()) + .or_else(|| self.config.musl_root.as_ref()) + .map(|p| &**p) + } + + /// Returns the "musl libdir" for this `target`. + fn musl_libdir(&self, target: TargetSelection) -> Option { + let t = self.config.target_config.get(&target)?; + if let libdir @ Some(_) = &t.musl_libdir { + return libdir.clone(); + } + self.musl_root(target).map(|root| root.join("lib")) + } + + /// Returns the sysroot for the wasi target, if defined + fn wasi_root(&self, target: TargetSelection) -> Option<&Path> { + self.config.target_config.get(&target).and_then(|t| t.wasi_root.as_ref()).map(|p| &**p) + } + + /// Returns `true` if this is a no-std `target`, if defined + fn no_std(&self, target: TargetSelection) -> Option { + self.config.target_config.get(&target).map(|t| t.no_std) + } + + /// Returns `true` if the target will be tested using the `remote-test-client` + /// and `remote-test-server` binaries. + fn remote_tested(&self, target: TargetSelection) -> bool { + self.qemu_rootfs(target).is_some() + || target.contains("android") + || env::var_os("TEST_DEVICE_ADDR").is_some() + } + + /// Returns the root of the "rootfs" image that this target will be using, + /// if one was configured. + /// + /// If `Some` is returned then that means that tests for this target are + /// emulated with QEMU and binaries will need to be shipped to the emulator. + fn qemu_rootfs(&self, target: TargetSelection) -> Option<&Path> { + self.config.target_config.get(&target).and_then(|t| t.qemu_rootfs.as_ref()).map(|p| &**p) + } + + /// Path to the python interpreter to use + fn python(&self) -> &Path { + if self.config.build.ends_with("apple-darwin") { + // Force /usr/bin/python3 on macOS for LLDB tests because we're loading the + // LLDB plugin's compiled module which only works with the system python + // (namely not Homebrew-installed python) + Path::new("/usr/bin/python3") + } else { + self.config + .python + .as_ref() + .expect("python is required for running LLDB or rustdoc tests") + } + } + + /// Temporary directory that extended error information is emitted to. + fn extended_error_dir(&self) -> PathBuf { + self.out.join("tmp/extended-error-metadata") + } + + /// Tests whether the `compiler` compiling for `target` should be forced to + /// use a stage1 compiler instead. + /// + /// Currently, by default, the build system does not perform a "full + /// bootstrap" by default where we compile the compiler three times. + /// Instead, we compile the compiler two times. The final stage (stage2) + /// just copies the libraries from the previous stage, which is what this + /// method detects. + /// + /// Here we return `true` if: + /// + /// * The build isn't performing a full bootstrap + /// * The `compiler` is in the final stage, 2 + /// * We're not cross-compiling, so the artifacts are already available in + /// stage1 + /// + /// When all of these conditions are met the build will lift artifacts from + /// the previous stage forward. + fn force_use_stage1(&self, stage: u32, target: TargetSelection) -> bool { + !self.config.full_bootstrap + && !self.config.download_rustc() + && stage >= 2 + && (self.hosts.iter().any(|h| *h == target) || target == self.build) + } + + /// Checks whether the `compiler` compiling for `target` should be forced to + /// use a stage2 compiler instead. + /// + /// When we download the pre-compiled version of rustc and compiler stage is >= 2, + /// it should be forced to use a stage2 compiler. + fn force_use_stage2(&self, stage: u32) -> bool { + self.config.download_rustc() && stage >= 2 + } + + /// Given `num` in the form "a.b.c" return a "release string" which + /// describes the release version number. + /// + /// For example on nightly this returns "a.b.c-nightly", on beta it returns + /// "a.b.c-beta.1" and on stable it just returns "a.b.c". + fn release(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => { + if !self.config.omit_git_hash { + format!("{}-beta.{}", num, self.beta_prerelease_version()) + } else { + format!("{num}-beta") + } + } + "nightly" => format!("{num}-nightly"), + _ => format!("{num}-dev"), + } + } + + fn beta_prerelease_version(&self) -> u32 { + fn extract_beta_rev_from_file>(version_file: P) -> Option { + let version = fs::read_to_string(version_file).ok()?; + + helpers::extract_beta_rev(&version) + } + + if let Some(s) = self.prerelease_version.get() { + return s; + } + + // First check if there is a version file available. + // If available, we read the beta revision from that file. + // This only happens when building from a source tarball when Git should not be used. + let count = extract_beta_rev_from_file(self.src.join("version")).unwrap_or_else(|| { + // Figure out how many merge commits happened since we branched off master. + // That's our beta number! + // (Note that we use a `..` range, not the `...` symmetric difference.) + output(self.config.git().arg("rev-list").arg("--count").arg("--merges").arg(format!( + "refs/remotes/origin/{}..HEAD", + self.config.stage0_metadata.config.nightly_branch + ))) + }); + let n = count.trim().parse().unwrap(); + self.prerelease_version.set(Some(n)); + n + } + + /// Returns the value of `release` above for Rust itself. + fn rust_release(&self) -> String { + self.release(&self.version) + } + + /// Returns the "package version" for a component given the `num` release + /// number. + /// + /// The package version is typically what shows up in the names of tarballs. + /// For channels like beta/nightly it's just the channel name, otherwise + /// it's the `num` provided. + fn package_vers(&self, num: &str) -> String { + match &self.config.channel[..] { + "stable" => num.to_string(), + "beta" => "beta".to_string(), + "nightly" => "nightly".to_string(), + _ => format!("{num}-dev"), + } + } + + /// Returns the value of `package_vers` above for Rust itself. + fn rust_package_vers(&self) -> String { + self.package_vers(&self.version) + } + + /// Returns the `version` string associated with this compiler for Rust + /// itself. + /// + /// Note that this is a descriptive string which includes the commit date, + /// sha, version, etc. + fn rust_version(&self) -> String { + let mut version = self.rust_info().version(self, &self.version); + if let Some(ref s) = self.config.description { + version.push_str(" ("); + version.push_str(s); + version.push(')'); + } + version + } + + /// Returns the full commit hash. + fn rust_sha(&self) -> Option<&str> { + self.rust_info().sha() + } + + /// Returns the `a.b.c` version that the given package is at. + fn release_num(&self, package: &str) -> String { + let toml_file_name = self.src.join(&format!("src/tools/{package}/Cargo.toml")); + let toml = t!(fs::read_to_string(&toml_file_name)); + for line in toml.lines() { + if let Some(stripped) = + line.strip_prefix("version = \"").and_then(|s| s.strip_suffix("\"")) + { + return stripped.to_owned(); + } + } + + panic!("failed to find version in {package}'s Cargo.toml") + } + + /// Returns `true` if unstable features should be enabled for the compiler + /// we're building. + fn unstable_features(&self) -> bool { + match &self.config.channel[..] { + "stable" | "beta" => false, + "nightly" | _ => true, + } + } + + /// Returns a Vec of all the dependencies of the given root crate, + /// including transitive dependencies and the root itself. Only includes + /// "local" crates (those in the local source tree, not from a registry). + fn in_tree_crates(&self, root: &str, target: Option) -> Vec<&Crate> { + let mut ret = Vec::new(); + let mut list = vec![INTERNER.intern_str(root)]; + let mut visited = HashSet::new(); + while let Some(krate) = list.pop() { + let krate = self + .crates + .get(&krate) + .unwrap_or_else(|| panic!("metadata missing for {krate}: {:?}", self.crates)); + ret.push(krate); + for dep in &krate.deps { + if !self.crates.contains_key(dep) { + // Ignore non-workspace members. + continue; + } + // Don't include optional deps if their features are not + // enabled. Ideally this would be computed from `cargo + // metadata --features …`, but that is somewhat slow. In + // the future, we may want to consider just filtering all + // build and dev dependencies in metadata::build. + if visited.insert(dep) + && (dep != "profiler_builtins" + || target + .map(|t| self.config.profiler_enabled(t)) + .unwrap_or_else(|| self.config.any_profiler_enabled())) + && (dep != "rustc_codegen_llvm" || self.config.llvm_enabled()) + { + list.push(*dep); + } + } + } + ret.sort_unstable_by_key(|krate| krate.name); // reproducible order needed for tests + ret + } + + fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> { + if self.config.dry_run() { + return Vec::new(); + } + + if !stamp.exists() { + eprintln!( + "ERROR: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", + stamp.display() + ); + crate::exit!(1); + } + + let mut paths = Vec::new(); + let contents = t!(fs::read(stamp), &stamp); + // This is the method we use for extracting paths from the stamp file passed to us. See + // run_cargo for more information (in compile.rs). + for part in contents.split(|b| *b == 0) { + if part.is_empty() { + continue; + } + let dependency_type = match part[0] as char { + 'h' => DependencyType::Host, + 's' => DependencyType::TargetSelfContained, + 't' => DependencyType::Target, + _ => unreachable!(), + }; + let path = PathBuf::from(t!(str::from_utf8(&part[1..]))); + paths.push((path, dependency_type)); + } + paths + } + + /// Copies a file from `src` to `dst` + pub fn copy(&self, src: &Path, dst: &Path) { + self.copy_internal(src, dst, false); + } + + fn copy_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { + if self.config.dry_run() { + return; + } + self.verbose_than(1, &format!("Copy {src:?} to {dst:?}")); + if src == dst { + return; + } + let _ = fs::remove_file(&dst); + let metadata = t!(src.symlink_metadata()); + let mut src = src.to_path_buf(); + if metadata.file_type().is_symlink() { + if dereference_symlinks { + src = t!(fs::canonicalize(src)); + } else { + let link = t!(fs::read_link(src)); + t!(self.symlink_file(link, dst)); + return; + } + } + if let Ok(()) = fs::hard_link(&src, dst) { + // Attempt to "easy copy" by creating a hard link + // (symlinks don't work on windows), but if that fails + // just fall back to a slow `copy` operation. + } else { + if let Err(e) = fs::copy(&src, dst) { + panic!("failed to copy `{}` to `{}`: {}", src.display(), dst.display(), e) + } + t!(fs::set_permissions(dst, metadata.permissions())); + let atime = FileTime::from_last_access_time(&metadata); + let mtime = FileTime::from_last_modification_time(&metadata); + t!(filetime::set_file_times(dst, atime, mtime)); + } + } + + /// Copies the `src` directory recursively to `dst`. Both are assumed to exist + /// when this function is called. + pub fn cp_r(&self, src: &Path, dst: &Path) { + if self.config.dry_run() { + return; + } + for f in self.read_dir(src) { + let path = f.path(); + let name = path.file_name().unwrap(); + let dst = dst.join(name); + if t!(f.file_type()).is_dir() { + t!(fs::create_dir_all(&dst)); + self.cp_r(&path, &dst); + } else { + let _ = fs::remove_file(&dst); + self.copy(&path, &dst); + } + } + } + + /// Copies the `src` directory recursively to `dst`. Both are assumed to exist + /// when this function is called. Unwanted files or directories can be skipped + /// by returning `false` from the filter function. + pub fn cp_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { + // Immediately recurse with an empty relative path + self.recurse_(src, dst, Path::new(""), filter) + } + + // Inner function does the actual work + fn recurse_(&self, src: &Path, dst: &Path, relative: &Path, filter: &dyn Fn(&Path) -> bool) { + for f in self.read_dir(src) { + let path = f.path(); + let name = path.file_name().unwrap(); + let dst = dst.join(name); + let relative = relative.join(name); + // Only copy file or directory if the filter function returns true + if filter(&relative) { + if t!(f.file_type()).is_dir() { + let _ = fs::remove_dir_all(&dst); + self.create_dir(&dst); + self.recurse_(&path, &dst, &relative, filter); + } else { + let _ = fs::remove_file(&dst); + self.copy(&path, &dst); + } + } + } + } + + fn copy_to_folder(&self, src: &Path, dest_folder: &Path) { + let file_name = src.file_name().unwrap(); + let dest = dest_folder.join(file_name); + self.copy(src, &dest); + } + + fn install(&self, src: &Path, dstdir: &Path, perms: u32) { + if self.config.dry_run() { + return; + } + let dst = dstdir.join(src.file_name().unwrap()); + self.verbose_than(1, &format!("Install {src:?} to {dst:?}")); + t!(fs::create_dir_all(dstdir)); + if !src.exists() { + panic!("ERROR: File \"{}\" not found!", src.display()); + } + self.copy_internal(src, &dst, true); + chmod(&dst, perms); + } + + fn read(&self, path: &Path) -> String { + if self.config.dry_run() { + return String::new(); + } + t!(fs::read_to_string(path)) + } + + fn create_dir(&self, dir: &Path) { + if self.config.dry_run() { + return; + } + t!(fs::create_dir_all(dir)) + } + + fn remove_dir(&self, dir: &Path) { + if self.config.dry_run() { + return; + } + t!(fs::remove_dir_all(dir)) + } + + fn read_dir(&self, dir: &Path) -> impl Iterator { + let iter = match fs::read_dir(dir) { + Ok(v) => v, + Err(_) if self.config.dry_run() => return vec![].into_iter(), + Err(err) => panic!("could not read dir {dir:?}: {err:?}"), + }; + iter.map(|e| t!(e)).collect::>().into_iter() + } + + fn symlink_file, Q: AsRef>(&self, src: P, link: Q) -> io::Result<()> { + #[cfg(unix)] + use std::os::unix::fs::symlink as symlink_file; + #[cfg(windows)] + use std::os::windows::fs::symlink_file; + if !self.config.dry_run() { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) } + } + + /// Returns if config.ninja is enabled, and checks for ninja existence, + /// exiting with a nicer error message if not. + fn ninja(&self) -> bool { + let mut cmd_finder = crate::core::sanity::Finder::new(); + + if self.config.ninja_in_file { + // Some Linux distros rename `ninja` to `ninja-build`. + // CMake can work with either binary name. + if cmd_finder.maybe_have("ninja-build").is_none() + && cmd_finder.maybe_have("ninja").is_none() + { + eprintln!( + " +Couldn't find required command: ninja (or ninja-build) + +You should install ninja as described at +, +or set `ninja = false` in the `[llvm]` section of `config.toml`. +Alternatively, set `download-ci-llvm = true` in that `[llvm]` section +to download LLVM rather than building it. +" + ); + exit!(1); + } + } + + // If ninja isn't enabled but we're building for MSVC then we try + // doubly hard to enable it. It was realized in #43767 that the msbuild + // CMake generator for MSVC doesn't respect configuration options like + // disabling LLVM assertions, which can often be quite important! + // + // In these cases we automatically enable Ninja if we find it in the + // environment. + if !self.config.ninja_in_file + && self.config.build.contains("msvc") + && cmd_finder.maybe_have("ninja").is_some() + { + return true; + } + + self.config.ninja_in_file + } + + pub fn colored_stdout R>(&self, f: F) -> R { + self.colored_stream_inner(StandardStream::stdout, self.config.stdout_is_tty, f) + } + + pub fn colored_stderr R>(&self, f: F) -> R { + self.colored_stream_inner(StandardStream::stderr, self.config.stderr_is_tty, f) + } + + fn colored_stream_inner(&self, constructor: C, is_tty: bool, f: F) -> R + where + C: Fn(ColorChoice) -> StandardStream, + F: FnOnce(&mut dyn WriteColor) -> R, + { + let choice = match self.config.color { + flags::Color::Always => ColorChoice::Always, + flags::Color::Never => ColorChoice::Never, + flags::Color::Auto if !is_tty => ColorChoice::Never, + flags::Color::Auto => ColorChoice::Auto, + }; + let mut stream = constructor(choice); + let result = f(&mut stream); + stream.reset().unwrap(); + result + } +} + +#[cfg(unix)] +fn chmod(path: &Path, perms: u32) { + use std::os::unix::fs::*; + t!(fs::set_permissions(path, fs::Permissions::from_mode(perms))); +} +#[cfg(windows)] +fn chmod(_path: &Path, _perms: u32) {} + +impl Compiler { + pub fn with_stage(mut self, stage: u32) -> Compiler { + self.stage = stage; + self + } + + /// Returns `true` if this is a snapshot compiler for `build`'s configuration + pub fn is_snapshot(&self, build: &Build) -> bool { + self.stage == 0 && self.host == build.build + } + + /// Returns if this compiler should be treated as a final stage one in the + /// current build session. + /// This takes into account whether we're performing a full bootstrap or + /// not; don't directly compare the stage with `2`! + pub fn is_final_stage(&self, build: &Build) -> bool { + let final_stage = if build.config.full_bootstrap { 2 } else { 1 }; + self.stage >= final_stage + } +} + +fn envify(s: &str) -> String { + s.chars() + .map(|c| match c { + '-' => '_', + c => c, + }) + .flat_map(|c| c.to_uppercase()) + .collect() +} + +pub fn find_recent_config_change_ids(current_id: usize) -> Vec { + if !CONFIG_CHANGE_HISTORY.contains(¤t_id) { + // If the current change-id is greater than the most recent one, return + // an empty list (it may be due to switching from a recent branch to an + // older one); otherwise, return the full list (assuming the user provided + // the incorrect change-id by accident). + if let Some(max_id) = CONFIG_CHANGE_HISTORY.iter().max() { + if ¤t_id > max_id { + return Vec::new(); + } + } + + return CONFIG_CHANGE_HISTORY.to_vec(); + } + + let index = CONFIG_CHANGE_HISTORY.iter().position(|&id| id == current_id).unwrap(); + + CONFIG_CHANGE_HISTORY + .iter() + .skip(index + 1) // Skip the current_id and IDs before it + .cloned() + .collect() +} diff --git a/src/bootstrap/src/tests/builder.rs b/src/bootstrap/src/tests/builder.rs new file mode 100644 index 000000000..96139f7b0 --- /dev/null +++ b/src/bootstrap/src/tests/builder.rs @@ -0,0 +1,701 @@ +use super::*; +use crate::core::config::{Config, DryRun, TargetSelection}; +use crate::core::build_steps::doc::DocumentationFormat; +use std::thread; + +fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config { + configure_with_args(&[cmd.to_owned()], host, target) +} + +fn configure_with_args(cmd: &[String], host: &[&str], target: &[&str]) -> Config { + let mut config = Config::parse(cmd); + // don't save toolstates + config.save_toolstates = None; + config.dry_run = DryRun::SelfCheck; + + // Ignore most submodules, since we don't need them for a dry run. + // But make sure to check out the `doc` and `rust-analyzer` submodules, since some steps need them + // just to know which commands to run. + let submodule_build = Build::new(Config { + // don't include LLVM, so CI doesn't require ninja/cmake to be installed + rust_codegen_backends: vec![], + ..Config::parse(&["check".to_owned()]) + }); + submodule_build.update_submodule(Path::new("src/doc/book")); + config.submodules = Some(false); + + config.ninja_in_file = false; + // try to avoid spurious failures in dist where we create/delete each others file + // HACK: rather than pull in `tempdir`, use the one that cargo has conveniently created for us + let dir = Path::new(env!("OUT_DIR")) + .join("tmp-rustbuild-tests") + .join(&thread::current().name().unwrap_or("unknown").replace(":", "-")); + t!(fs::create_dir_all(&dir)); + config.out = dir; + config.build = TargetSelection::from_user("A"); + config.hosts = host.iter().map(|s| TargetSelection::from_user(s)).collect(); + config.targets = target.iter().map(|s| TargetSelection::from_user(s)).collect(); + config +} + +fn first(v: Vec<(A, B)>) -> Vec { + v.into_iter().map(|(a, _)| a).collect::>() +} + +fn run_build(paths: &[PathBuf], config: Config) -> Cache { + let kind = config.cmd.kind(); + let build = Build::new(config); + let builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(kind), paths); + builder.cache +} + +fn check_cli(paths: [&str; N]) { + run_build( + &paths.map(PathBuf::from), + configure_with_args(&paths.map(String::from), &["A"], &["A"]), + ); +} + +macro_rules! std { + ($host:ident => $target:ident, stage = $stage:literal) => { + compile::Std::new( + Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, + TargetSelection::from_user(stringify!($target)), + ) + }; +} + +macro_rules! doc_std { + ($host:ident => $target:ident, stage = $stage:literal) => {{ + let config = configure("doc", &["A"], &["A"]); + let build = Build::new(config); + let builder = Builder::new(&build); + doc::Std::new( + $stage, + TargetSelection::from_user(stringify!($target)), + &builder, + DocumentationFormat::HTML, + ) + }}; +} + +macro_rules! rustc { + ($host:ident => $target:ident, stage = $stage:literal) => { + compile::Rustc::new( + Compiler { host: TargetSelection::from_user(stringify!($host)), stage: $stage }, + TargetSelection::from_user(stringify!($target)), + ) + }; +} + +#[test] +fn test_valid() { + // make sure multi suite paths are accepted + check_cli(["test", "tests/ui/attr-start.rs", "tests/ui/attr-shebang.rs"]); +} + +#[test] +#[should_panic] +fn test_invalid() { + // make sure that invalid paths are caught, even when combined with valid paths + check_cli(["test", "library/std", "x"]); +} + +#[test] +fn test_intersection() { + let set = |paths: &[&str]| { + PathSet::Set(paths.into_iter().map(|p| TaskPath { path: p.into(), kind: None }).collect()) + }; + let library_set = set(&["library/core", "library/alloc", "library/std"]); + let mut command_paths = + vec![Path::new("library/core"), Path::new("library/alloc"), Path::new("library/stdarch")]; + let subset = library_set.intersection_removing_matches(&mut command_paths, Kind::Build); + assert_eq!(subset, set(&["library/core", "library/alloc"]),); + assert_eq!(command_paths, vec![Path::new("library/stdarch")]); +} + +#[test] +fn test_exclude() { + let mut config = configure("test", &["A"], &["A"]); + config.skip = vec!["src/tools/tidy".into()]; + let cache = run_build(&[], config); + + // Ensure we have really excluded tidy + assert!(!cache.contains::()); + + // Ensure other tests are not affected. + assert!(cache.contains::()); +} + +#[test] +fn test_exclude_kind() { + let path = PathBuf::from("compiler/rustc_data_structures"); + + let mut config = configure("test", &["A"], &["A"]); + // Ensure our test is valid, and `test::Rustc` would be run without the exclude. + assert!(run_build(&[], config.clone()).contains::()); + // Ensure tests for rustc are not skipped. + config.skip = vec![path.clone()]; + assert!(run_build(&[], config.clone()).contains::()); + // Ensure builds for rustc are not skipped. + assert!(run_build(&[], config).contains::()); +} + +/// Ensure that if someone passes both a single crate and `library`, all library crates get built. +#[test] +fn alias_and_path_for_library() { + let mut cache = + run_build(&["library".into(), "core".into()], configure("build", &["A"], &["A"])); + assert_eq!( + first(cache.all::()), + &[std!(A => A, stage = 0), std!(A => A, stage = 1)] + ); + + let mut cache = run_build(&["library".into(), "core".into()], configure("doc", &["A"], &["A"])); + assert_eq!(first(cache.all::()), &[doc_std!(A => A, stage = 0)]); +} + +#[test] +fn test_beta_rev_parsing() { + use crate::utils::helpers::extract_beta_rev; + + // single digit revision + assert_eq!(extract_beta_rev("1.99.9-beta.7 (xxxxxx)"), Some("7".to_string())); + // multiple digits + assert_eq!(extract_beta_rev("1.99.9-beta.777 (xxxxxx)"), Some("777".to_string())); + // nightly channel (no beta revision) + assert_eq!(extract_beta_rev("1.99.9-nightly (xxxxxx)"), None); + // stable channel (no beta revision) + assert_eq!(extract_beta_rev("1.99.9 (xxxxxxx)"), None); + // invalid string + assert_eq!(extract_beta_rev("invalid"), None); +} + +mod defaults { + use super::{configure, first, run_build}; + use crate::core::builder::*; + use crate::Config; + use pretty_assertions::assert_eq; + + #[test] + fn build_default() { + let mut cache = run_build(&[], configure("build", &["A"], &["A"])); + + let a = TargetSelection::from_user("A"); + assert_eq!( + first(cache.all::()), + &[std!(A => A, stage = 0), std!(A => A, stage = 1),] + ); + assert!(!cache.all::().is_empty()); + // Make sure rustdoc is only built once. + assert_eq!( + first(cache.all::()), + // Recall that rustdoc stages are off-by-one + // - this is the compiler it's _linked_ to, not built with. + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }], + ); + assert_eq!(first(cache.all::()), &[rustc!(A => A, stage = 0)],); + } + + #[test] + fn build_stage_0() { + let config = Config { stage: 0, ..configure("build", &["A"], &["A"]) }; + let mut cache = run_build(&[], config); + + let a = TargetSelection::from_user("A"); + assert_eq!(first(cache.all::()), &[std!(A => A, stage = 0)]); + assert!(!cache.all::().is_empty()); + assert_eq!( + first(cache.all::()), + // This is the beta rustdoc. + // Add an assert here to make sure this is the only rustdoc built. + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }], + ); + assert!(cache.all::().is_empty()); + } + + #[test] + fn build_cross_compile() { + let config = Config { stage: 1, ..configure("build", &["A", "B"], &["A", "B"]) }; + let mut cache = run_build(&[], config); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + // Ideally, this build wouldn't actually have `target: a` + // rustdoc/rustcc/std here (the user only requested a host=B build, so + // there's not really a need for us to build for target A in this case + // (since we're producing stage 1 libraries/binaries). But currently + // rustbuild is just a bit buggy here; this should be fixed though. + assert_eq!( + first(cache.all::()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => B, stage = 0), + std!(A => B, stage = 1), + ] + ); + assert_eq!( + first(cache.all::()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: b, stage: 1 } }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: b, stage: 1 } }, + ], + ); + assert_eq!( + first(cache.all::()), + &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 0),] + ); + } + + #[test] + fn doc_default() { + let mut config = configure("doc", &["A"], &["A"]); + config.compiler_docs = true; + config.cmd = Subcommand::Doc { open: false, json: false }; + let mut cache = run_build(&[], config); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 0 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!(first(cache.all::()), &[doc::ErrorIndex { target: a },]); + assert_eq!( + first(cache.all::()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 0 } }] + ); + // docs should be built with the beta compiler, not with the stage0 artifacts. + // recall that rustdoc is off-by-one: `stage` is the compiler rustdoc is _linked_ to, + // not the one it was built by. + assert_eq!( + first(cache.all::()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } },] + ); + } +} + +mod dist { + use super::{first, run_build, Config}; + use crate::core::builder::*; + use pretty_assertions::assert_eq; + + fn configure(host: &[&str], target: &[&str]) -> Config { + Config { stage: 2, ..super::configure("dist", host, target) } + } + + #[test] + fn dist_baseline() { + let mut cache = run_build(&[], configure(&["A"], &["A"])); + + let a = TargetSelection::from_user("A"); + + assert_eq!(first(cache.all::()), &[dist::Docs { host: a },]); + assert_eq!(first(cache.all::()), &[dist::Mingw { host: a },]); + assert_eq!( + first(cache.all::()), + &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },] + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + // Make sure rustdoc is only built once. + assert_eq!( + first(cache.all::()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + ); + } + + #[test] + fn dist_with_targets() { + let mut cache = run_build(&[], configure(&["A"], &["A", "B"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Rustc { compiler: Compiler { host: a, stage: 2 } },] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 2 }, target: b }, + ] + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + } + + #[test] + fn dist_with_hosts() { + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => A, stage = 2), + std!(A => B, stage = 1), + std!(A => B, stage = 2), + ], + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + } + + #[test] + fn dist_only_cross_host() { + let b = TargetSelection::from_user("B"); + let mut config = configure(&["A", "B"], &["A", "B"]); + config.docs = false; + config.extended = true; + config.hosts = vec![b]; + let mut cache = run_build(&[], config); + + assert_eq!( + first(cache.all::()), + &[dist::Rustc { compiler: Compiler { host: b, stage: 2 } },] + ); + assert_eq!( + first(cache.all::()), + &[rustc!(A => A, stage = 0), rustc!(A => B, stage = 1),] + ); + } + + #[test] + fn dist_with_targets_and_hosts() { + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B", "C"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + let c = TargetSelection::from_user("C"); + + assert_eq!( + first(cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c }, + ] + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + } + + #[test] + fn dist_with_empty_host() { + let config = configure(&[], &["C"]); + let mut cache = run_build(&[], config); + + let a = TargetSelection::from_user("A"); + let c = TargetSelection::from_user("C"); + + assert_eq!(first(cache.all::()), &[dist::Docs { host: c },]); + assert_eq!(first(cache.all::()), &[dist::Mingw { host: c },]); + assert_eq!( + first(cache.all::()), + &[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },] + ); + } + + #[test] + fn dist_with_same_targets_and_hosts() { + let mut cache = run_build(&[], configure(&["A", "B"], &["A", "B"])); + + let a = TargetSelection::from_user("A"); + let b = TargetSelection::from_user("B"); + + assert_eq!( + first(cache.all::()), + &[dist::Docs { host: a }, dist::Docs { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[dist::Mingw { host: a }, dist::Mingw { host: b },] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Rustc { compiler: Compiler { host: a, stage: 2 } }, + dist::Rustc { compiler: Compiler { host: b, stage: 2 } }, + ] + ); + assert_eq!( + first(cache.all::()), + &[ + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a }, + dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b }, + ] + ); + assert_eq!(first(cache.all::()), &[dist::Src]); + assert_eq!( + first(cache.all::()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => A, stage = 2), + std!(A => B, stage = 1), + std!(A => B, stage = 2), + ] + ); + assert_eq!( + first(cache.all::()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, + compile::Assemble { target_compiler: Compiler { host: b, stage: 2 } }, + ] + ); + } + + #[test] + fn build_all() { + let build = Build::new(configure(&["A", "B"], &["A", "B", "C"])); + let mut builder = Builder::new(&build); + builder.run_step_descriptions( + &Builder::get_step_descriptions(Kind::Build), + &["compiler/rustc".into(), "library".into()], + ); + + assert_eq!( + first(builder.cache.all::()), + &[ + std!(A => A, stage = 0), + std!(A => A, stage = 1), + std!(A => A, stage = 2), + std!(A => B, stage = 1), + std!(A => B, stage = 2), + std!(A => C, stage = 2), + ] + ); + assert_eq!(builder.cache.all::().len(), 5); + assert_eq!( + first(builder.cache.all::()), + &[ + rustc!(A => A, stage = 0), + rustc!(A => A, stage = 1), + rustc!(A => A, stage = 2), + rustc!(A => B, stage = 1), + rustc!(A => B, stage = 2), + ] + ); + } + + #[test] + fn build_with_empty_host() { + let config = configure(&[], &["C"]); + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]); + + let a = TargetSelection::from_user("A"); + + assert_eq!( + first(builder.cache.all::()), + &[std!(A => A, stage = 0), std!(A => A, stage = 1), std!(A => C, stage = 2),] + ); + assert_eq!( + first(builder.cache.all::()), + &[ + compile::Assemble { target_compiler: Compiler { host: a, stage: 0 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 1 } }, + compile::Assemble { target_compiler: Compiler { host: a, stage: 2 } }, + ] + ); + assert_eq!( + first(builder.cache.all::()), + &[rustc!(A => A, stage = 0), rustc!(A => A, stage = 1),] + ); + } + + #[test] + fn test_with_no_doc_stage0() { + let mut config = configure(&["A"], &["A"]); + config.stage = 0; + config.paths = vec!["library/std".into()]; + config.cmd = Subcommand::Test { + test_args: vec![], + rustc_args: vec![], + no_fail_fast: false, + no_doc: true, + doc: false, + bless: false, + force_rerun: false, + compare_mode: None, + rustfix_coverage: false, + pass: None, + run: None, + only_modified: false, + skip: vec![], + extra_checks: None, + }; + + let build = Build::new(config); + let mut builder = Builder::new(&build); + + let host = TargetSelection::from_user("A"); + + builder.run_step_descriptions( + &[StepDescription::from::(Kind::Test)], + &["library/std".into()], + ); + + // Ensure we don't build any compiler artifacts. + assert!(!builder.cache.contains::()); + assert_eq!( + first(builder.cache.all::()), + &[test::Crate { + compiler: Compiler { host, stage: 0 }, + target: host, + mode: Mode::Std, + crates: vec![INTERNER.intern_str("std")], + },] + ); + } + + #[test] + fn doc_ci() { + let mut config = configure(&["A"], &["A"]); + config.compiler_docs = true; + config.cmd = Subcommand::Doc { open: false, json: false }; + let build = Build::new(config); + let mut builder = Builder::new(&build); + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Doc), &[]); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 1 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!( + first(builder.cache.all::()), + &[doc::ErrorIndex { target: a },] + ); + assert_eq!( + first(builder.cache.all::()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] + ); + // This is actually stage 1, but Rustdoc::run swaps out the compiler with + // stage minus 1 if --stage is not 0. Very confusing! + assert_eq!( + first(builder.cache.all::()), + &[tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } },] + ); + } + + #[test] + fn test_docs() { + // Behavior of `x.py test` doing various documentation tests. + let mut config = configure(&["A"], &["A"]); + config.cmd = Subcommand::Test { + test_args: vec![], + rustc_args: vec![], + no_fail_fast: false, + doc: true, + no_doc: false, + skip: vec![], + bless: false, + force_rerun: false, + compare_mode: None, + rustfix_coverage: false, + pass: None, + run: None, + only_modified: false, + extra_checks: None, + }; + // Make sure rustfmt binary not being found isn't an error. + config.channel = "beta".to_string(); + let build = Build::new(config); + let mut builder = Builder::new(&build); + + builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Test), &[]); + let a = TargetSelection::from_user("A"); + + // error_index_generator uses stage 1 to share rustdoc artifacts with the + // rustdoc tool. + assert_eq!( + first(builder.cache.all::()), + &[doc::ErrorIndex { target: a },] + ); + assert_eq!( + first(builder.cache.all::()), + &[tool::ErrorIndex { compiler: Compiler { host: a, stage: 1 } }] + ); + // Unfortunately rustdoc is built twice. Once from stage1 for compiletest + // (and other things), and once from stage0 for std crates. Ideally it + // would only be built once. If someone wants to fix this, it might be + // worth investigating if it would be possible to test std from stage1. + // Note that the stages here are +1 than what they actually are because + // Rustdoc::run swaps out the compiler with stage minus 1 if --stage is + // not 0. + // + // The stage 0 copy is the one downloaded for bootstrapping. It is + // (currently) needed to run "cargo test" on the linkchecker, and + // should be relatively "free". + assert_eq!( + first(builder.cache.all::()), + &[ + tool::Rustdoc { compiler: Compiler { host: a, stage: 0 } }, + tool::Rustdoc { compiler: Compiler { host: a, stage: 1 } }, + tool::Rustdoc { compiler: Compiler { host: a, stage: 2 } }, + ] + ); + } +} diff --git a/src/bootstrap/src/tests/config.rs b/src/bootstrap/src/tests/config.rs new file mode 100644 index 000000000..59bd52a94 --- /dev/null +++ b/src/bootstrap/src/tests/config.rs @@ -0,0 +1,219 @@ +use crate::core::config::TomlConfig; +use super::{Config, Flags}; + +use clap::CommandFactory; +use serde::Deserialize; +use std::{ + env, + fs::{remove_file, File}, + io::Write, + path::Path, +}; + +fn parse(config: &str) -> Config { + Config::parse_inner(&["check".to_owned(), "--config=/does/not/exist".to_owned()], |&_| { + toml::from_str(config).unwrap() + }) +} + +#[test] +fn download_ci_llvm() { + if crate::core::build_steps::llvm::is_ci_llvm_modified(&parse("")) { + eprintln!("Detected LLVM as non-available: running in CI and modified LLVM in this change"); + return; + } + + let parse_llvm = |s| parse(s).llvm_from_ci; + let if_available = parse_llvm("llvm.download-ci-llvm = \"if-available\""); + + assert!(parse_llvm("llvm.download-ci-llvm = true")); + assert!(!parse_llvm("llvm.download-ci-llvm = false")); + assert_eq!(parse_llvm(""), if_available); + assert_eq!(parse_llvm("rust.channel = \"dev\""), if_available); + assert!(!parse_llvm("rust.channel = \"stable\"")); + assert!(parse_llvm("build.build = \"x86_64-unknown-linux-gnu\"")); + assert!(parse_llvm( + "llvm.assertions = true \r\n build.build = \"x86_64-unknown-linux-gnu\" \r\n llvm.download-ci-llvm = \"if-available\"" + )); + assert!(!parse_llvm( + "llvm.assertions = true \r\n build.build = \"aarch64-apple-darwin\" \r\n llvm.download-ci-llvm = \"if-available\"" + )); +} + +// FIXME(onur-ozkan): extend scope of the test +// refs: +// - https://github.com/rust-lang/rust/issues/109120 +// - https://github.com/rust-lang/rust/pull/109162#issuecomment-1496782487 +#[test] +fn detect_src_and_out() { + fn test(cfg: Config, build_dir: Option<&str>) { + // This will bring absolute form of `src/bootstrap` path + let current_dir = std::env::current_dir().unwrap(); + + // get `src` by moving into project root path + let expected_src = current_dir.ancestors().nth(2).unwrap(); + assert_eq!(&cfg.src, expected_src); + + // Sanity check for `src` + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let expected_src = manifest_dir.ancestors().nth(2).unwrap(); + assert_eq!(&cfg.src, expected_src); + + // test if build-dir was manually given in config.toml + if let Some(custom_build_dir) = build_dir { + assert_eq!(&cfg.out, Path::new(custom_build_dir)); + } + // test the native bootstrap way + else { + // This should bring output path of bootstrap in absolute form + let cargo_target_dir = env::var_os("CARGO_TARGET_DIR").expect( + "CARGO_TARGET_DIR must been provided for the test environment from bootstrap", + ); + + // Move to `build` from `build/bootstrap` + let expected_out = Path::new(&cargo_target_dir).parent().unwrap(); + assert_eq!(&cfg.out, expected_out); + + let args: Vec = env::args().collect(); + + // Another test for `out` as a sanity check + // + // This will bring something similar to: + // `{build-dir}/bootstrap/debug/deps/bootstrap-c7ee91d5661e2804` + // `{build-dir}` can be anywhere, not just in the rust project directory. + let dep = Path::new(args.first().unwrap()); + let expected_out = dep.ancestors().nth(4).unwrap(); + + assert_eq!(&cfg.out, expected_out); + } + } + + test(parse(""), None); + + { + let build_dir = if cfg!(windows) { Some("C:\\tmp") } else { Some("/tmp") }; + test(parse("build.build-dir = \"/tmp\""), build_dir); + } +} + +#[test] +fn clap_verify() { + Flags::command().debug_assert(); +} + +#[test] +fn override_toml() { + let config = Config::parse_inner( + &[ + "check".to_owned(), + "--config=/does/not/exist".to_owned(), + "--set=change-id=1".to_owned(), + "--set=rust.lto=fat".to_owned(), + "--set=rust.deny-warnings=false".to_owned(), + "--set=build.gdb=\"bar\"".to_owned(), + "--set=build.tools=[\"cargo\"]".to_owned(), + "--set=llvm.build-config={\"foo\" = \"bar\"}".to_owned(), + ], + |&_| { + toml::from_str( + r#" +change-id = 0 +[rust] +lto = "off" +deny-warnings = true + +[build] +gdb = "foo" +tools = [] + +[llvm] +download-ci-llvm = false +build-config = {} + "#, + ) + .unwrap() + }, + ); + assert_eq!(config.change_id, Some(1), "setting top-level value"); + assert_eq!( + config.rust_lto, + crate::core::config::RustcLto::Fat, + "setting string value without quotes" + ); + assert_eq!(config.gdb, Some("bar".into()), "setting string value with quotes"); + assert!(!config.deny_warnings, "setting boolean value"); + assert_eq!( + config.tools, + Some(["cargo".to_string()].into_iter().collect()), + "setting list value" + ); + assert_eq!( + config.llvm_build_config, + [("foo".to_string(), "bar".to_string())].into_iter().collect(), + "setting dictionary value" + ); +} + +#[test] +#[should_panic] +fn override_toml_duplicate() { + Config::parse_inner( + &[ + "check".to_owned(), + "--config=/does/not/exist".to_owned(), + "--set=change-id=1".to_owned(), + "--set=change-id=2".to_owned(), + ], + |&_| toml::from_str("change-id = 0").unwrap(), + ); +} + +#[test] +fn profile_user_dist() { + fn get_toml(file: &Path) -> TomlConfig { + let contents = if file.ends_with("config.toml") { + "profile = \"user\"".to_owned() + } else { + assert!(file.ends_with("config.dist.toml")); + std::fs::read_to_string(file).unwrap() + }; + toml::from_str(&contents) + .and_then(|table: toml::Value| TomlConfig::deserialize(table)) + .unwrap() + } + Config::parse_inner(&["check".to_owned()], get_toml); +} + +#[test] +fn rust_optimize() { + assert!(parse("").rust_optimize.is_release()); + assert!(!parse("rust.optimize = false").rust_optimize.is_release()); + assert!(parse("rust.optimize = true").rust_optimize.is_release()); + assert!(!parse("rust.optimize = 0").rust_optimize.is_release()); + assert!(parse("rust.optimize = 1").rust_optimize.is_release()); + assert!(parse("rust.optimize = \"s\"").rust_optimize.is_release()); + assert_eq!(parse("rust.optimize = 1").rust_optimize.get_opt_level(), Some("1".to_string())); + assert_eq!(parse("rust.optimize = \"s\"").rust_optimize.get_opt_level(), Some("s".to_string())); +} + +#[test] +#[should_panic] +fn invalid_rust_optimize() { + parse("rust.optimize = \"a\""); +} + +#[test] +fn verify_file_integrity() { + let config = parse(""); + + let tempfile = config.tempdir().join(".tmp-test-file"); + File::create(&tempfile).unwrap().write_all(b"dummy value").unwrap(); + assert!(tempfile.exists()); + + assert!( + config + .verify(&tempfile, "7e255dd9542648a8779268a0f268b891a198e9828e860ed23f826440e786eae5") + ); + + remove_file(tempfile).unwrap(); +} diff --git a/src/bootstrap/src/tests/setup.rs b/src/bootstrap/src/tests/setup.rs new file mode 100644 index 000000000..0fe6e4a46 --- /dev/null +++ b/src/bootstrap/src/tests/setup.rs @@ -0,0 +1,14 @@ +use super::{RUST_ANALYZER_SETTINGS, SETTINGS_HASHES}; +use sha2::Digest; + +#[test] +fn check_matching_settings_hash() { + let mut hasher = sha2::Sha256::new(); + hasher.update(&RUST_ANALYZER_SETTINGS); + let hash = hex::encode(hasher.finalize().as_slice()); + assert_eq!( + &hash, + SETTINGS_HASHES.last().unwrap(), + "Update `SETTINGS_HASHES` with the new hash of `src/etc/rust_analyzer_settings.json`" + ); +} diff --git a/src/bootstrap/src/utils/bin_helpers.rs b/src/bootstrap/src/utils/bin_helpers.rs new file mode 100644 index 000000000..c90fd2805 --- /dev/null +++ b/src/bootstrap/src/utils/bin_helpers.rs @@ -0,0 +1,28 @@ +//! This file is meant to be included directly from bootstrap shims to avoid a +//! dependency on the bootstrap library. This reduces the binary size and +//! improves compilation time by reducing the linking time. + +/// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`. +/// If it was not defined, returns 0 by default. +/// +/// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer. +pub(crate) fn parse_rustc_verbose() -> usize { + use std::str::FromStr; + + match std::env::var("RUSTC_VERBOSE") { + Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"), + Err(_) => 0, + } +} + +/// Parses the value of the "RUSTC_STAGE" environment variable and returns it as a `String`. +/// +/// If "RUSTC_STAGE" was not set, the program will be terminated with 101. +pub(crate) fn parse_rustc_stage() -> String { + std::env::var("RUSTC_STAGE").unwrap_or_else(|_| { + // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead. + eprintln!("rustc shim: FATAL: RUSTC_STAGE was not set"); + eprintln!("rustc shim: NOTE: use `x.py build -vvv` to see all environment variables set by bootstrap"); + std::process::exit(101); + }) +} diff --git a/src/bootstrap/src/utils/cache.rs b/src/bootstrap/src/utils/cache.rs new file mode 100644 index 000000000..1b2aa9c23 --- /dev/null +++ b/src/bootstrap/src/utils/cache.rs @@ -0,0 +1,272 @@ +use std::any::{Any, TypeId}; +use std::borrow::Borrow; +use std::cell::RefCell; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; +use std::mem; +use std::ops::Deref; +use std::path::PathBuf; +use std::sync::Mutex; + +// FIXME: replace with std::lazy after it gets stabilized and reaches beta +use once_cell::sync::Lazy; + +use crate::core::builder::Step; + +pub struct Interned(usize, PhantomData<*const T>); + +impl Default for Interned { + fn default() -> Self { + T::default().intern() + } +} + +impl Copy for Interned {} +impl Clone for Interned { + fn clone(&self) -> Interned { + *self + } +} + +impl PartialEq for Interned { + fn eq(&self, other: &Self) -> bool { + self.0 == other.0 + } +} +impl Eq for Interned {} + +impl PartialEq for Interned { + fn eq(&self, other: &str) -> bool { + *self == other + } +} +impl<'a> PartialEq<&'a str> for Interned { + fn eq(&self, other: &&str) -> bool { + **self == **other + } +} +impl<'a, T> PartialEq<&'a Interned> for Interned { + fn eq(&self, other: &&Self) -> bool { + self.0 == other.0 + } +} +impl<'a, T> PartialEq> for &'a Interned { + fn eq(&self, other: &Interned) -> bool { + self.0 == other.0 + } +} + +unsafe impl Send for Interned {} +unsafe impl Sync for Interned {} + +impl fmt::Display for Interned { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: &str = &*self; + f.write_str(s) + } +} + +impl fmt::Debug for Interned +where + Self: Deref, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let s: &U = &*self; + f.write_fmt(format_args!("{s:?}")) + } +} + +impl Hash for Interned { + fn hash(&self, state: &mut H) { + let l = T::intern_cache().lock().unwrap(); + l.get(*self).hash(state) + } +} + +impl Deref for Interned { + type Target = T::Target; + fn deref(&self) -> &Self::Target { + let l = T::intern_cache().lock().unwrap(); + unsafe { mem::transmute::<&Self::Target, &Self::Target>(l.get(*self)) } + } +} + +impl, U: ?Sized> AsRef for Interned { + fn as_ref(&self) -> &U { + let l = T::intern_cache().lock().unwrap(); + unsafe { mem::transmute::<&U, &U>(l.get(*self).as_ref()) } + } +} + +impl PartialOrd for Interned { + fn partial_cmp(&self, other: &Self) -> Option { + let l = T::intern_cache().lock().unwrap(); + l.get(*self).partial_cmp(l.get(*other)) + } +} + +impl Ord for Interned { + fn cmp(&self, other: &Self) -> Ordering { + let l = T::intern_cache().lock().unwrap(); + l.get(*self).cmp(l.get(*other)) + } +} + +struct TyIntern { + items: Vec, + set: HashMap>, +} + +impl Default for TyIntern { + fn default() -> Self { + TyIntern { items: Vec::new(), set: Default::default() } + } +} + +impl TyIntern { + fn intern_borrow(&mut self, item: &B) -> Interned + where + B: Eq + Hash + ToOwned + ?Sized, + T: Borrow, + { + if let Some(i) = self.set.get(&item) { + return *i; + } + let item = item.to_owned(); + let interned = Interned(self.items.len(), PhantomData::<*const T>); + self.set.insert(item.clone(), interned); + self.items.push(item); + interned + } + + fn intern(&mut self, item: T) -> Interned { + if let Some(i) = self.set.get(&item) { + return *i; + } + let interned = Interned(self.items.len(), PhantomData::<*const T>); + self.set.insert(item.clone(), interned); + self.items.push(item); + interned + } + + fn get(&self, i: Interned) -> &T { + &self.items[i.0] + } +} + +#[derive(Default)] +pub struct Interner { + strs: Mutex>, + paths: Mutex>, + lists: Mutex>>, +} + +trait Internable: Clone + Eq + Hash + 'static { + fn intern_cache() -> &'static Mutex>; + + fn intern(self) -> Interned { + Self::intern_cache().lock().unwrap().intern(self) + } +} + +impl Internable for String { + fn intern_cache() -> &'static Mutex> { + &INTERNER.strs + } +} + +impl Internable for PathBuf { + fn intern_cache() -> &'static Mutex> { + &INTERNER.paths + } +} + +impl Internable for Vec { + fn intern_cache() -> &'static Mutex> { + &INTERNER.lists + } +} + +impl Interner { + pub fn intern_str(&self, s: &str) -> Interned { + self.strs.lock().unwrap().intern_borrow(s) + } + pub fn intern_string(&self, s: String) -> Interned { + self.strs.lock().unwrap().intern(s) + } + + pub fn intern_path(&self, s: PathBuf) -> Interned { + self.paths.lock().unwrap().intern(s) + } + + pub fn intern_list(&self, v: Vec) -> Interned> { + self.lists.lock().unwrap().intern(v) + } +} + +pub static INTERNER: Lazy = Lazy::new(Interner::default); + +/// This is essentially a `HashMap` which allows storing any type in its input and +/// any type in its output. It is a write-once cache; values are never evicted, +/// which means that references to the value can safely be returned from the +/// `get()` method. +#[derive(Debug)] +pub struct Cache( + RefCell< + HashMap< + TypeId, + Box, // actually a HashMap> + >, + >, +); + +impl Cache { + pub fn new() -> Cache { + Cache(RefCell::new(HashMap::new())) + } + + pub fn put(&self, step: S, value: S::Output) { + let mut cache = self.0.borrow_mut(); + let type_id = TypeId::of::(); + let stepcache = cache + .entry(type_id) + .or_insert_with(|| Box::new(HashMap::::new())) + .downcast_mut::>() + .expect("invalid type mapped"); + assert!(!stepcache.contains_key(&step), "processing {step:?} a second time"); + stepcache.insert(step, value); + } + + pub fn get(&self, step: &S) -> Option { + let mut cache = self.0.borrow_mut(); + let type_id = TypeId::of::(); + let stepcache = cache + .entry(type_id) + .or_insert_with(|| Box::new(HashMap::::new())) + .downcast_mut::>() + .expect("invalid type mapped"); + stepcache.get(step).cloned() + } +} + +#[cfg(test)] +impl Cache { + pub fn all(&mut self) -> Vec<(S, S::Output)> { + let cache = self.0.get_mut(); + let type_id = TypeId::of::(); + let mut v = cache + .remove(&type_id) + .map(|b| b.downcast::>().expect("correct type")) + .map(|m| m.into_iter().collect::>()) + .unwrap_or_default(); + v.sort_by_key(|(s, _)| s.clone()); + v + } + + pub fn contains(&self) -> bool { + self.0.borrow().contains_key(&TypeId::of::()) + } +} diff --git a/src/bootstrap/src/utils/cc_detect.rs b/src/bootstrap/src/utils/cc_detect.rs new file mode 100644 index 000000000..52b36ce75 --- /dev/null +++ b/src/bootstrap/src/utils/cc_detect.rs @@ -0,0 +1,288 @@ +//! C-compiler probing and detection. +//! +//! This module will fill out the `cc` and `cxx` maps of `Build` by looking for +//! C and C++ compilers for each target configured. A compiler is found through +//! a number of vectors (in order of precedence) +//! +//! 1. Configuration via `target.$target.cc` in `config.toml`. +//! 2. Configuration via `target.$target.android-ndk` in `config.toml`, if +//! applicable +//! 3. Special logic to probe on OpenBSD +//! 4. The `CC_$target` environment variable. +//! 5. The `CC` environment variable. +//! 6. "cc" +//! +//! Some of this logic is implemented here, but much of it is farmed out to the +//! `cc` crate itself, so we end up having the same fallbacks as there. +//! Similar logic is then used to find a C++ compiler, just some s/cc/c++/ is +//! used. +//! +//! It is intended that after this module has run no C/C++ compiler will +//! ever be probed for. Instead the compilers found here will be used for +//! everything. + +use std::collections::HashSet; +use std::path::{Path, PathBuf}; +use std::process::Command; +use std::{env, iter}; + +use crate::core::config::TargetSelection; +use crate::utils::helpers::output; +use crate::{Build, CLang, GitRepo}; + +// The `cc` crate doesn't provide a way to obtain a path to the detected archiver, +// so use some simplified logic here. First we respect the environment variable `AR`, then +// try to infer the archiver path from the C compiler path. +// In the future this logic should be replaced by calling into the `cc` crate. +fn cc2ar(cc: &Path, target: TargetSelection) -> Option { + if let Some(ar) = env::var_os(format!("AR_{}", target.triple.replace("-", "_"))) { + Some(PathBuf::from(ar)) + } else if let Some(ar) = env::var_os("AR") { + Some(PathBuf::from(ar)) + } else if target.contains("msvc") { + None + } else if target.contains("musl") { + Some(PathBuf::from("ar")) + } else if target.contains("openbsd") { + Some(PathBuf::from("ar")) + } else if target.contains("vxworks") { + Some(PathBuf::from("wr-ar")) + } else if target.contains("android") { + Some(cc.parent().unwrap().join(PathBuf::from("llvm-ar"))) + } else { + let parent = cc.parent().unwrap(); + let file = cc.file_name().unwrap().to_str().unwrap(); + for suffix in &["gcc", "cc", "clang"] { + if let Some(idx) = file.rfind(suffix) { + let mut file = file[..idx].to_owned(); + file.push_str("ar"); + return Some(parent.join(&file)); + } + } + Some(parent.join(file)) + } +} + +fn new_cc_build(build: &Build, target: TargetSelection) -> cc::Build { + let mut cfg = cc::Build::new(); + cfg.cargo_metadata(false) + .opt_level(2) + .warnings(false) + .debug(false) + // Compress debuginfo + .flag_if_supported("-gz") + .target(&target.triple) + .host(&build.build.triple); + match build.crt_static(target) { + Some(a) => { + cfg.static_crt(a); + } + None => { + if target.contains("msvc") { + cfg.static_crt(true); + } + if target.contains("musl") { + cfg.static_flag(true); + } + } + } + cfg +} + +pub fn find(build: &Build) { + // For all targets we're going to need a C compiler for building some shims + // and such as well as for being a linker for Rust code. + let targets = build + .targets + .iter() + .chain(&build.hosts) + .cloned() + .chain(iter::once(build.build)) + .collect::>(); + for target in targets.into_iter() { + find_target(build, target); + } +} + +pub fn find_target(build: &Build, target: TargetSelection) { + let mut cfg = new_cc_build(build, target); + let config = build.config.target_config.get(&target); + if let Some(cc) = config + .and_then(|c| c.cc.clone()) + .or_else(|| default_compiler(&mut cfg, Language::C, target, build)) + { + cfg.compiler(cc); + } + + let compiler = cfg.get_compiler(); + let ar = if let ar @ Some(..) = config.and_then(|c| c.ar.clone()) { + ar + } else { + cc2ar(compiler.path(), target) + }; + + build.cc.borrow_mut().insert(target, compiler.clone()); + let cflags = build.cflags(target, GitRepo::Rustc, CLang::C); + + // If we use llvm-libunwind, we will need a C++ compiler as well for all targets + // We'll need one anyways if the target triple is also a host triple + let mut cfg = new_cc_build(build, target); + cfg.cpp(true); + let cxx_configured = if let Some(cxx) = config + .and_then(|c| c.cxx.clone()) + .or_else(|| default_compiler(&mut cfg, Language::CPlusPlus, target, build)) + { + cfg.compiler(cxx); + true + } else { + // Use an auto-detected compiler (or one configured via `CXX_target_triple` env vars). + cfg.try_get_compiler().is_ok() + }; + + // for VxWorks, record CXX compiler which will be used in lib.rs:linker() + if cxx_configured || target.contains("vxworks") { + let compiler = cfg.get_compiler(); + build.cxx.borrow_mut().insert(target, compiler); + } + + build.verbose(&format!("CC_{} = {:?}", &target.triple, build.cc(target))); + build.verbose(&format!("CFLAGS_{} = {:?}", &target.triple, cflags)); + if let Ok(cxx) = build.cxx(target) { + let cxxflags = build.cflags(target, GitRepo::Rustc, CLang::Cxx); + build.verbose(&format!("CXX_{} = {:?}", &target.triple, cxx)); + build.verbose(&format!("CXXFLAGS_{} = {:?}", &target.triple, cxxflags)); + } + if let Some(ar) = ar { + build.verbose(&format!("AR_{} = {:?}", &target.triple, ar)); + build.ar.borrow_mut().insert(target, ar); + } + + if let Some(ranlib) = config.and_then(|c| c.ranlib.clone()) { + build.ranlib.borrow_mut().insert(target, ranlib); + } +} + +fn default_compiler( + cfg: &mut cc::Build, + compiler: Language, + target: TargetSelection, + build: &Build, +) -> Option { + match &*target.triple { + // When compiling for android we may have the NDK configured in the + // config.toml in which case we look there. Otherwise the default + // compiler already takes into account the triple in question. + t if t.contains("android") => build + .config + .android_ndk + .as_ref() + .map(|ndk| ndk_compiler(compiler, &*target.triple, ndk)), + + // The default gcc version from OpenBSD may be too old, try using egcc, + // which is a gcc version from ports, if this is the case. + t if t.contains("openbsd") => { + let c = cfg.get_compiler(); + let gnu_compiler = compiler.gcc(); + if !c.path().ends_with(gnu_compiler) { + return None; + } + + let output = output(c.to_command().arg("--version")); + let i = output.find(" 4.")?; + match output[i + 3..].chars().next().unwrap() { + '0'..='6' => {} + _ => return None, + } + let alternative = format!("e{gnu_compiler}"); + if Command::new(&alternative).output().is_ok() { + Some(PathBuf::from(alternative)) + } else { + None + } + } + + "mips-unknown-linux-musl" if compiler == Language::C => { + if cfg.get_compiler().path().to_str() == Some("gcc") { + Some(PathBuf::from("mips-linux-musl-gcc")) + } else { + None + } + } + "mipsel-unknown-linux-musl" if compiler == Language::C => { + if cfg.get_compiler().path().to_str() == Some("gcc") { + Some(PathBuf::from("mipsel-linux-musl-gcc")) + } else { + None + } + } + + t if t.contains("musl") && compiler == Language::C => { + if let Some(root) = build.musl_root(target) { + let guess = root.join("bin/musl-gcc"); + if guess.exists() { Some(guess) } else { None } + } else { + None + } + } + + _ => None, + } +} + +pub(crate) fn ndk_compiler(compiler: Language, triple: &str, ndk: &Path) -> PathBuf { + let mut triple_iter = triple.split("-"); + let triple_translated = if let Some(arch) = triple_iter.next() { + let arch_new = match arch { + "arm" | "armv7" | "armv7neon" | "thumbv7" | "thumbv7neon" => "armv7a", + other => other, + }; + std::iter::once(arch_new).chain(triple_iter).collect::>().join("-") + } else { + triple.to_string() + }; + + // API 19 is the earliest API level supported by NDK r25b but AArch64 and x86_64 support + // begins at API level 21. + let api_level = + if triple.contains("aarch64") || triple.contains("x86_64") { "21" } else { "19" }; + let compiler = format!("{}{}-{}", triple_translated, api_level, compiler.clang()); + let host_tag = if cfg!(target_os = "macos") { + // The NDK uses universal binaries, so this is correct even on ARM. + "darwin-x86_64" + } else if cfg!(target_os = "windows") { + "windows-x86_64" + } else { + // NDK r25b only has official releases for macOS, Windows and Linux. + // Try the Linux directory everywhere else, on the assumption that the OS has an + // emulation layer that can cope (e.g. BSDs). + "linux-x86_64" + }; + ndk.join("toolchains").join("llvm").join("prebuilt").join(host_tag).join("bin").join(compiler) +} + +/// The target programming language for a native compiler. +#[derive(PartialEq)] +pub(crate) enum Language { + /// The compiler is targeting C. + C, + /// The compiler is targeting C++. + CPlusPlus, +} + +impl Language { + /// Obtains the name of a compiler in the GCC collection. + fn gcc(self) -> &'static str { + match self { + Language::C => "gcc", + Language::CPlusPlus => "g++", + } + } + + /// Obtains the name of a compiler in the clang suite. + fn clang(self) -> &'static str { + match self { + Language::C => "clang", + Language::CPlusPlus => "clang++", + } + } +} diff --git a/src/bootstrap/src/utils/channel.rs b/src/bootstrap/src/utils/channel.rs new file mode 100644 index 000000000..e59d7f22a --- /dev/null +++ b/src/bootstrap/src/utils/channel.rs @@ -0,0 +1,159 @@ +//! Build configuration for Rust's release channels. +//! +//! Implements the stable/beta/nightly channel distinctions by setting various +//! flags like the `unstable_features`, calculating variables like `release` and +//! `package_vers`, and otherwise indicating to the compiler what it should +//! print out as part of its version information. + +use std::fs; +use std::path::Path; +use std::process::Command; + +use crate::utils::helpers::{output, t}; +use crate::Build; + +#[derive(Clone, Default)] +pub enum GitInfo { + /// This is not a git repository. + #[default] + Absent, + /// This is a git repository. + /// If the info should be used (`omit_git_hash` is false), this will be + /// `Some`, otherwise it will be `None`. + Present(Option), + /// This is not a git repository, but the info can be fetched from the + /// `git-commit-info` file. + RecordedForTarball(Info), +} + +#[derive(Clone)] +pub struct Info { + pub commit_date: String, + pub sha: String, + pub short_sha: String, +} + +impl GitInfo { + pub fn new(omit_git_hash: bool, dir: &Path) -> GitInfo { + // See if this even begins to look like a git dir + if !dir.join(".git").exists() { + match read_commit_info_file(dir) { + Some(info) => return GitInfo::RecordedForTarball(info), + None => return GitInfo::Absent, + } + } + + // Make sure git commands work + match Command::new("git").arg("rev-parse").current_dir(dir).output() { + Ok(ref out) if out.status.success() => {} + _ => return GitInfo::Absent, + } + + // If we're ignoring the git info, we don't actually need to collect it, just make sure this + // was a git repo in the first place. + if omit_git_hash { + return GitInfo::Present(None); + } + + // Ok, let's scrape some info + let ver_date = output( + Command::new("git") + .current_dir(dir) + .arg("log") + .arg("-1") + .arg("--date=short") + .arg("--pretty=format:%cd"), + ); + let ver_hash = output(Command::new("git").current_dir(dir).arg("rev-parse").arg("HEAD")); + let short_ver_hash = output( + Command::new("git").current_dir(dir).arg("rev-parse").arg("--short=9").arg("HEAD"), + ); + GitInfo::Present(Some(Info { + commit_date: ver_date.trim().to_string(), + sha: ver_hash.trim().to_string(), + short_sha: short_ver_hash.trim().to_string(), + })) + } + + pub fn info(&self) -> Option<&Info> { + match self { + GitInfo::Absent => None, + GitInfo::Present(info) => info.as_ref(), + GitInfo::RecordedForTarball(info) => Some(info), + } + } + + pub fn sha(&self) -> Option<&str> { + self.info().map(|s| &s.sha[..]) + } + + pub fn sha_short(&self) -> Option<&str> { + self.info().map(|s| &s.short_sha[..]) + } + + pub fn commit_date(&self) -> Option<&str> { + self.info().map(|s| &s.commit_date[..]) + } + + pub fn version(&self, build: &Build, num: &str) -> String { + let mut version = build.release(num); + if let Some(ref inner) = self.info() { + version.push_str(" ("); + version.push_str(&inner.short_sha); + version.push(' '); + version.push_str(&inner.commit_date); + version.push(')'); + } + version + } + + /// Returns whether this directory has a `.git` directory which should be managed by bootstrap. + pub fn is_managed_git_subrepository(&self) -> bool { + match self { + GitInfo::Absent | GitInfo::RecordedForTarball(_) => false, + GitInfo::Present(_) => true, + } + } + + /// Returns whether this is being built from a tarball. + pub fn is_from_tarball(&self) -> bool { + match self { + GitInfo::Absent | GitInfo::Present(_) => false, + GitInfo::RecordedForTarball(_) => true, + } + } +} + +/// Read the commit information from the `git-commit-info` file given the +/// project root. +pub fn read_commit_info_file(root: &Path) -> Option { + if let Ok(contents) = fs::read_to_string(root.join("git-commit-info")) { + let mut lines = contents.lines(); + let sha = lines.next(); + let short_sha = lines.next(); + let commit_date = lines.next(); + let info = match (commit_date, sha, short_sha) { + (Some(commit_date), Some(sha), Some(short_sha)) => Info { + commit_date: commit_date.to_owned(), + sha: sha.to_owned(), + short_sha: short_sha.to_owned(), + }, + _ => panic!("the `git-commit-info` file is malformed"), + }; + Some(info) + } else { + None + } +} + +/// Write the commit information to the `git-commit-info` file given the project +/// root. +pub fn write_commit_info_file(root: &Path, info: &Info) { + let commit_info = format!("{}\n{}\n{}\n", info.sha, info.short_sha, info.commit_date); + t!(fs::write(root.join("git-commit-info"), &commit_info)); +} + +/// Write the commit hash to the `git-commit-hash` file given the project root. +pub fn write_commit_hash_file(root: &Path, sha: &str) { + t!(fs::write(root.join("git-commit-hash"), sha)); +} diff --git a/src/bootstrap/src/utils/dylib.rs b/src/bootstrap/src/utils/dylib.rs new file mode 100644 index 000000000..279a6a010 --- /dev/null +++ b/src/bootstrap/src/utils/dylib.rs @@ -0,0 +1,27 @@ +//! Various utilities for working with dylib paths. + +/// Returns the environment variable which the dynamic library lookup path +/// resides in for this platform. +pub fn dylib_path_var() -> &'static str { + if cfg!(target_os = "windows") { + "PATH" + } else if cfg!(target_os = "macos") { + "DYLD_LIBRARY_PATH" + } else if cfg!(target_os = "haiku") { + "LIBRARY_PATH" + } else if cfg!(target_os = "aix") { + "LIBPATH" + } else { + "LD_LIBRARY_PATH" + } +} + +/// Parses the `dylib_path_var()` environment variable, returning a list of +/// paths that are members of this lookup path. +pub fn dylib_path() -> Vec { + let var = match std::env::var_os(dylib_path_var()) { + Some(v) => v, + None => return vec![], + }; + std::env::split_paths(&var).collect() +} diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs new file mode 100644 index 000000000..0aede2022 --- /dev/null +++ b/src/bootstrap/src/utils/exec.rs @@ -0,0 +1,60 @@ +use std::process::Command; + +/// What should be done when the command fails. +#[derive(Debug, Copy, Clone)] +pub enum BehaviorOnFailure { + /// Immediately stop bootstrap. + Exit, + /// Delay failure until the end of bootstrap invocation. + DelayFail, + /// Ignore the failure, the command can fail in an expected way. + Ignore, +} + +/// How should the output of the command be handled. +#[derive(Debug, Copy, Clone)] +pub enum OutputMode { + /// Print both the output (by inheriting stdout/stderr) and also the command itself, if it + /// fails. + PrintAll, + /// Print the output (by inheriting stdout/stderr). + PrintOutput, + /// Suppress the output if the command succeeds, otherwise print the output. + SuppressOnSuccess, +} + +/// Wrapper around `std::process::Command`. +#[derive(Debug)] +pub struct BootstrapCommand<'a> { + pub command: &'a mut Command, + pub failure_behavior: BehaviorOnFailure, + pub output_mode: OutputMode, +} + +impl<'a> BootstrapCommand<'a> { + pub fn delay_failure(self) -> Self { + Self { failure_behavior: BehaviorOnFailure::DelayFail, ..self } + } + + pub fn fail_fast(self) -> Self { + Self { failure_behavior: BehaviorOnFailure::Exit, ..self } + } + + pub fn allow_failure(self) -> Self { + Self { failure_behavior: BehaviorOnFailure::Ignore, ..self } + } + + pub fn output_mode(self, output_mode: OutputMode) -> Self { + Self { output_mode, ..self } + } +} + +impl<'a> From<&'a mut Command> for BootstrapCommand<'a> { + fn from(command: &'a mut Command) -> Self { + Self { + command, + failure_behavior: BehaviorOnFailure::Exit, + output_mode: OutputMode::PrintAll, + } + } +} diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs new file mode 100644 index 000000000..5bc81f2d9 --- /dev/null +++ b/src/bootstrap/src/utils/helpers.rs @@ -0,0 +1,472 @@ +//! Various utility functions used throughout rustbuild. +//! +//! Simple things like testing the various filesystem operations here and there, +//! not a lot of interesting happenings here unfortunately. + +use build_helper::util::fail; +use std::env; +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::str; +use std::time::{Instant, SystemTime, UNIX_EPOCH}; + +use crate::core::builder::Builder; +use crate::core::config::{Config, TargetSelection}; +use crate::OnceCell; + +pub use crate::utils::dylib::{dylib_path, dylib_path_var}; + +/// A helper macro to `unwrap` a result except also print out details like: +/// +/// * The file/line of the panic +/// * The expression that failed +/// * The error itself +/// +/// This is currently used judiciously throughout the build system rather than +/// using a `Result` with `try!`, but this may change one day... +#[macro_export] +macro_rules! t { + ($e:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + } + }; + // it can show extra info in the second parameter + ($e:expr, $extra:expr) => { + match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra), + } + }; +} +pub use t; + +/// Given an executable called `name`, return the filename for the +/// executable for a particular target. +pub fn exe(name: &str, target: TargetSelection) -> String { + if target.contains("windows") { + format!("{name}.exe") + } else if target.contains("uefi") { + format!("{name}.efi") + } else { + name.to_string() + } +} + +/// Returns `true` if the file name given looks like a dynamic library. +pub fn is_dylib(name: &str) -> bool { + name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll") +} + +/// Returns `true` if the file name given looks like a debug info file +pub fn is_debug_info(name: &str) -> bool { + // FIXME: consider split debug info on other platforms (e.g., Linux, macOS) + name.ends_with(".pdb") +} + +/// Returns the corresponding relative library directory that the compiler's +/// dylibs will be found in. +pub fn libdir(target: TargetSelection) -> &'static str { + if target.contains("windows") { "bin" } else { "lib" } +} + +/// Adds a list of lookup paths to `cmd`'s dynamic library lookup path. +/// If the dylib_path_var is already set for this cmd, the old value will be overwritten! +pub fn add_dylib_path(path: Vec, cmd: &mut Command) { + let mut list = dylib_path(); + for path in path { + list.insert(0, path); + } + cmd.env(dylib_path_var(), t!(env::join_paths(list))); +} + +/// Adds a list of lookup paths to `cmd`'s link library lookup path. +pub fn add_link_lib_path(path: Vec, cmd: &mut Command) { + let mut list = link_lib_path(); + for path in path { + list.insert(0, path); + } + cmd.env(link_lib_path_var(), t!(env::join_paths(list))); +} + +/// Returns the environment variable which the link library lookup path +/// resides in for this platform. +fn link_lib_path_var() -> &'static str { + if cfg!(target_env = "msvc") { "LIB" } else { "LIBRARY_PATH" } +} + +/// Parses the `link_lib_path_var()` environment variable, returning a list of +/// paths that are members of this lookup path. +fn link_lib_path() -> Vec { + let var = match env::var_os(link_lib_path_var()) { + Some(v) => v, + None => return vec![], + }; + env::split_paths(&var).collect() +} + +pub struct TimeIt(bool, Instant); + +/// Returns an RAII structure that prints out how long it took to drop. +pub fn timeit(builder: &Builder<'_>) -> TimeIt { + TimeIt(builder.config.dry_run(), Instant::now()) +} + +impl Drop for TimeIt { + fn drop(&mut self) { + let time = self.1.elapsed(); + if !self.0 { + println!("\tfinished in {}.{:03} seconds", time.as_secs(), time.subsec_millis()); + } + } +} + +/// Used for download caching +pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { + if !stamp.exists() { + return true; + } + t!(fs::read_to_string(stamp)) != key +} + +/// Symlinks two directories, using junctions on Windows and normal symlinks on +/// Unix. +pub fn symlink_dir(config: &Config, original: &Path, link: &Path) -> io::Result<()> { + if config.dry_run() { + return Ok(()); + } + let _ = fs::remove_dir(link); + return symlink_dir_inner(original, link); + + #[cfg(not(windows))] + fn symlink_dir_inner(original: &Path, link: &Path) -> io::Result<()> { + use std::os::unix::fs; + fs::symlink(original, link) + } + + #[cfg(windows)] + fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> { + junction::create(&target, &junction) + } +} + +pub fn forcing_clang_based_tests() -> bool { + if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") { + match &var.to_string_lossy().to_lowercase()[..] { + "1" | "yes" | "on" => true, + "0" | "no" | "off" => false, + other => { + // Let's make sure typos don't go unnoticed + panic!( + "Unrecognized option '{other}' set in \ + RUSTBUILD_FORCE_CLANG_BASED_TESTS" + ) + } + } + } else { + false + } +} + +pub fn use_host_linker(target: TargetSelection) -> bool { + // FIXME: this information should be gotten by checking the linker flavor + // of the rustc target + !(target.contains("emscripten") + || target.contains("wasm32") + || target.contains("nvptx") + || target.contains("fortanix") + || target.contains("fuchsia") + || target.contains("bpf") + || target.contains("switch")) +} + +pub fn target_supports_cranelift_backend(target: TargetSelection) -> bool { + if target.contains("linux") { + target.contains("x86_64") + || target.contains("aarch64") + || target.contains("s390x") + || target.contains("riscv64gc") + } else if target.contains("darwin") || target.contains("windows") { + target.contains("x86_64") + } else { + false + } +} + +pub fn is_valid_test_suite_arg<'a, P: AsRef>( + path: &'a Path, + suite_path: P, + builder: &Builder<'_>, +) -> Option<&'a str> { + let suite_path = suite_path.as_ref(); + let path = match path.strip_prefix(".") { + Ok(p) => p, + Err(_) => path, + }; + if !path.starts_with(suite_path) { + return None; + } + let abs_path = builder.src.join(path); + let exists = abs_path.is_dir() || abs_path.is_file(); + if !exists { + panic!( + "Invalid test suite filter \"{}\": file or directory does not exist", + abs_path.display() + ); + } + // Since test suite paths are themselves directories, if we don't + // specify a directory or file, we'll get an empty string here + // (the result of the test suite directory without its suite prefix). + // Therefore, we need to filter these out, as only the first --test-args + // flag is respected, so providing an empty --test-args conflicts with + // any following it. + match path.strip_prefix(suite_path).ok().and_then(|p| p.to_str()) { + Some(s) if !s.is_empty() => Some(s), + _ => None, + } +} + +pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { + let status = match cmd.status() { + Ok(status) => status, + Err(e) => { + println!("failed to execute command: {cmd:?}\nERROR: {e}"); + return false; + } + }; + if !status.success() && print_cmd_on_fail { + println!( + "\n\ncommand did not execute successfully: {cmd:?}\n\ + expected success, got: {status}\n\n" + ); + } + status.success() +} + +pub fn make(host: &str) -> PathBuf { + if host.contains("dragonfly") + || host.contains("freebsd") + || host.contains("netbsd") + || host.contains("openbsd") + { + PathBuf::from("gmake") + } else { + PathBuf::from("make") + } +} + +#[track_caller] +pub fn output(cmd: &mut Command) -> String { + let output = match cmd.stderr(Stdio::inherit()).output() { + Ok(status) => status, + Err(e) => fail(&format!("failed to execute command: {cmd:?}\nERROR: {e}")), + }; + if !output.status.success() { + panic!( + "command did not execute successfully: {:?}\n\ + expected success, got: {}", + cmd, output.status + ); + } + String::from_utf8(output.stdout).unwrap() +} + +/// Returns the last-modified time for `path`, or zero if it doesn't exist. +pub fn mtime(path: &Path) -> SystemTime { + fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) +} + +/// Returns `true` if `dst` is up to date given that the file or files in `src` +/// are used to generate it. +/// +/// Uses last-modified time checks to verify this. +pub fn up_to_date(src: &Path, dst: &Path) -> bool { + if !dst.exists() { + return false; + } + let threshold = mtime(dst); + let meta = match fs::metadata(src) { + Ok(meta) => meta, + Err(e) => panic!("source {src:?} failed to get metadata: {e}"), + }; + if meta.is_dir() { + dir_up_to_date(src, threshold) + } else { + meta.modified().unwrap_or(UNIX_EPOCH) <= threshold + } +} + +fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool { + t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| { + let meta = t!(e.metadata()); + if meta.is_dir() { + dir_up_to_date(&e.path(), threshold) + } else { + meta.modified().unwrap_or(UNIX_EPOCH) < threshold + } + }) +} + +/// Copied from `std::path::absolute` until it stabilizes. +/// +/// FIXME: this shouldn't exist. +pub(crate) fn absolute(path: &Path) -> PathBuf { + if path.as_os_str().is_empty() { + panic!("can't make empty path absolute"); + } + #[cfg(unix)] + { + t!(absolute_unix(path), format!("could not make path absolute: {}", path.display())) + } + #[cfg(windows)] + { + t!(absolute_windows(path), format!("could not make path absolute: {}", path.display())) + } + #[cfg(not(any(unix, windows)))] + { + println!("WARNING: bootstrap is not supported on non-unix platforms"); + t!(std::fs::canonicalize(t!(std::env::current_dir()))).join(path) + } +} + +#[cfg(unix)] +/// Make a POSIX path absolute without changing its semantics. +fn absolute_unix(path: &Path) -> io::Result { + // This is mostly a wrapper around collecting `Path::components`, with + // exceptions made where this conflicts with the POSIX specification. + // See 4.13 Pathname Resolution, IEEE Std 1003.1-2017 + // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + + use std::os::unix::prelude::OsStrExt; + let mut components = path.components(); + let path_os = path.as_os_str().as_bytes(); + + let mut normalized = if path.is_absolute() { + // "If a pathname begins with two successive characters, the + // first component following the leading characters may be + // interpreted in an implementation-defined manner, although more than + // two leading characters shall be treated as a single + // character." + if path_os.starts_with(b"//") && !path_os.starts_with(b"///") { + components.next(); + PathBuf::from("//") + } else { + PathBuf::new() + } + } else { + env::current_dir()? + }; + normalized.extend(components); + + // "Interfaces using pathname resolution may specify additional constraints + // when a pathname that does not name an existing directory contains at + // least one non- character and contains one or more trailing + // characters". + // A trailing is also meaningful if "a symbolic link is + // encountered during pathname resolution". + + if path_os.ends_with(b"/") { + normalized.push(""); + } + + Ok(normalized) +} + +#[cfg(windows)] +fn absolute_windows(path: &std::path::Path) -> std::io::Result { + use std::ffi::OsString; + use std::io::Error; + use std::os::windows::ffi::{OsStrExt, OsStringExt}; + use std::ptr::null_mut; + #[link(name = "kernel32")] + extern "system" { + fn GetFullPathNameW( + lpFileName: *const u16, + nBufferLength: u32, + lpBuffer: *mut u16, + lpFilePart: *mut *const u16, + ) -> u32; + } + + unsafe { + // encode the path as UTF-16 + let path: Vec = path.as_os_str().encode_wide().chain([0]).collect(); + let mut buffer = Vec::new(); + // Loop until either success or failure. + loop { + // Try to get the absolute path + let len = GetFullPathNameW( + path.as_ptr(), + buffer.len().try_into().unwrap(), + buffer.as_mut_ptr(), + null_mut(), + ); + match len as usize { + // Failure + 0 => return Err(Error::last_os_error()), + // Buffer is too small, resize. + len if len > buffer.len() => buffer.resize(len, 0), + // Success! + len => { + buffer.truncate(len); + return Ok(OsString::from_wide(&buffer).into()); + } + } + } + } +} + +/// Adapted from +/// +/// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource +/// directory to the linker flags, otherwise there will be linker errors about the profiler runtime +/// missing. This function returns the path to that directory. +pub fn get_clang_cl_resource_dir(clang_cl_path: &str) -> PathBuf { + // Similar to how LLVM does it, to find clang's library runtime directory: + // - we ask `clang-cl` to locate the `clang_rt.builtins` lib. + let mut builtins_locator = Command::new(clang_cl_path); + builtins_locator.args(&["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]); + + let clang_rt_builtins = output(&mut builtins_locator); + let clang_rt_builtins = Path::new(clang_rt_builtins.trim()); + assert!( + clang_rt_builtins.exists(), + "`clang-cl` must correctly locate the library runtime directory" + ); + + // - the profiler runtime will be located in the same directory as the builtins lib, like + // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`. + let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist"); + clang_rt_dir.to_path_buf() +} + +pub fn lld_flag_no_threads(is_windows: bool) -> &'static str { + static LLD_NO_THREADS: OnceCell<(&'static str, &'static str)> = OnceCell::new(); + let (windows, other) = LLD_NO_THREADS.get_or_init(|| { + let out = output(Command::new("lld").arg("-flavor").arg("ld").arg("--version")); + let newer = match (out.find(char::is_numeric), out.find('.')) { + (Some(b), Some(e)) => out.as_str()[b..e].parse::().ok().unwrap_or(14) > 10, + _ => true, + }; + if newer { ("/threads:1", "--threads=1") } else { ("/no-threads", "--no-threads") } + }); + if is_windows { windows } else { other } +} + +pub fn dir_is_empty(dir: &Path) -> bool { + t!(std::fs::read_dir(dir)).next().is_none() +} + +/// Extract the beta revision from the full version string. +/// +/// The full version string looks like "a.b.c-beta.y". And we need to extract +/// the "y" part from the string. +pub fn extract_beta_rev(version: &str) -> Option { + let parts = version.splitn(2, "-beta.").collect::>(); + let count = parts.get(1).and_then(|s| s.find(' ').map(|p| (&s[..p]).to_string())); + + count +} diff --git a/src/bootstrap/src/utils/job.rs b/src/bootstrap/src/utils/job.rs new file mode 100644 index 000000000..c5c718a89 --- /dev/null +++ b/src/bootstrap/src/utils/job.rs @@ -0,0 +1,159 @@ +#[cfg(windows)] +pub use for_windows::*; + +#[cfg(any(target_os = "haiku", target_os = "hermit", not(any(unix, windows))))] +pub unsafe fn setup(_build: &mut crate::Build) {} + +#[cfg(all(unix, not(target_os = "haiku")))] +pub unsafe fn setup(build: &mut crate::Build) { + if build.config.low_priority { + libc::setpriority(libc::PRIO_PGRP as _, 0, 10); + } +} + +#[cfg(windows)] +mod for_windows { + //! Job management on Windows for bootstrapping + //! + //! Most of the time when you're running a build system (e.g., make) you expect + //! Ctrl-C or abnormal termination to actually terminate the entire tree of + //! process in play, not just the one at the top. This currently works "by + //! default" on Unix platforms because Ctrl-C actually sends a signal to the + //! *process group* rather than the parent process, so everything will get torn + //! down. On Windows, however, this does not happen and Ctrl-C just kills the + //! parent process. + //! + //! To achieve the same semantics on Windows we use Job Objects to ensure that + //! all processes die at the same time. Job objects have a mode of operation + //! where when all handles to the object are closed it causes all child + //! processes associated with the object to be terminated immediately. + //! Conveniently whenever a process in the job object spawns a new process the + //! child will be associated with the job object as well. This means if we add + //! ourselves to the job object we create then everything will get torn down! + //! + //! Unfortunately most of the time the build system is actually called from a + //! python wrapper (which manages things like building the build system) so this + //! all doesn't quite cut it so far. To go the last mile we duplicate the job + //! object handle into our parent process (a python process probably) and then + //! close our own handle. This means that the only handle to the job object + //! resides in the parent python process, so when python dies the whole build + //! system dies (as one would probably expect!). + //! + //! Note that this module has a #[cfg(windows)] above it as none of this logic + //! is required on Unix. + + use crate::Build; + use std::env; + use std::ffi::c_void; + use std::io; + use std::mem; + + use windows::{ + core::PCWSTR, + Win32::Foundation::{CloseHandle, DuplicateHandle, DUPLICATE_SAME_ACCESS, HANDLE}, + Win32::System::Diagnostics::Debug::{ + SetErrorMode, SEM_NOGPFAULTERRORBOX, THREAD_ERROR_MODE, + }, + Win32::System::JobObjects::{ + AssignProcessToJobObject, CreateJobObjectW, JobObjectExtendedLimitInformation, + SetInformationJobObject, JOBOBJECT_EXTENDED_LIMIT_INFORMATION, + JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE, JOB_OBJECT_LIMIT_PRIORITY_CLASS, + }, + Win32::System::Threading::{ + GetCurrentProcess, OpenProcess, BELOW_NORMAL_PRIORITY_CLASS, PROCESS_DUP_HANDLE, + }, + }; + + pub unsafe fn setup(build: &mut Build) { + // Enable the Windows Error Reporting dialog which msys disables, + // so we can JIT debug rustc + let mode = SetErrorMode(THREAD_ERROR_MODE::default()); + let mode = THREAD_ERROR_MODE(mode); + SetErrorMode(mode & !SEM_NOGPFAULTERRORBOX); + + // Create a new job object for us to use + let job = CreateJobObjectW(None, PCWSTR::null()).unwrap(); + + // Indicate that when all handles to the job object are gone that all + // process in the object should be killed. Note that this includes our + // entire process tree by default because we've added ourselves and our + // children will reside in the job by default. + let mut info = JOBOBJECT_EXTENDED_LIMIT_INFORMATION::default(); + info.BasicLimitInformation.LimitFlags = JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE; + if build.config.low_priority { + info.BasicLimitInformation.LimitFlags |= JOB_OBJECT_LIMIT_PRIORITY_CLASS; + info.BasicLimitInformation.PriorityClass = BELOW_NORMAL_PRIORITY_CLASS.0; + } + let r = SetInformationJobObject( + job, + JobObjectExtendedLimitInformation, + &info as *const _ as *const c_void, + mem::size_of_val(&info) as u32, + ); + assert!(r.is_ok(), "{}", io::Error::last_os_error()); + + // Assign our process to this job object. Note that if this fails, one very + // likely reason is that we are ourselves already in a job object! This can + // happen on the build bots that we've got for Windows, or if just anyone + // else is instrumenting the build. In this case we just bail out + // immediately and assume that they take care of it. + // + // Also note that nested jobs (why this might fail) are supported in recent + // versions of Windows, but the version of Windows that our bots are running + // at least don't support nested job objects. + let r = AssignProcessToJobObject(job, GetCurrentProcess()); + if r.is_err() { + CloseHandle(job).ok(); + return; + } + + // If we've got a parent process (e.g., the python script that called us) + // then move ownership of this job object up to them. That way if the python + // script is killed (e.g., via ctrl-c) then we'll all be torn down. + // + // If we don't have a parent (e.g., this was run directly) then we + // intentionally leak the job object handle. When our process exits + // (normally or abnormally) it will close the handle implicitly, causing all + // processes in the job to be cleaned up. + let pid = match env::var("BOOTSTRAP_PARENT_ID") { + Ok(s) => s, + Err(..) => return, + }; + + let parent = match OpenProcess(PROCESS_DUP_HANDLE, false, pid.parse().unwrap()).ok() { + Some(parent) => parent, + _ => { + // If we get a null parent pointer here, it is possible that either + // we have an invalid pid or the parent process has been closed. + // Since the first case rarely happens + // (only when wrongly setting the environmental variable), + // it might be better to improve the experience of the second case + // when users have interrupted the parent process and we haven't finish + // duplicating the handle yet. We just need close the job object if that occurs. + CloseHandle(job).ok(); + return; + } + }; + + let mut parent_handle = HANDLE::default(); + let r = DuplicateHandle( + GetCurrentProcess(), + job, + parent, + &mut parent_handle, + 0, + false, + DUPLICATE_SAME_ACCESS, + ); + + // If this failed, well at least we tried! An example of DuplicateHandle + // failing in the past has been when the wrong python2 package spawned this + // build system (e.g., the `python2` package in MSYS instead of + // `mingw-w64-x86_64-python2`). Not sure why it failed, but the "failure + // mode" here is that we only clean everything up when the build system + // dies, not when the python parent does, so not too bad. + if r.is_err() { + CloseHandle(job).ok(); + } + } +} diff --git a/src/bootstrap/src/utils/metrics.rs b/src/bootstrap/src/utils/metrics.rs new file mode 100644 index 000000000..174f37422 --- /dev/null +++ b/src/bootstrap/src/utils/metrics.rs @@ -0,0 +1,258 @@ +//! This module is responsible for collecting metrics profiling information for the current build +//! and dumping it to disk as JSON, to aid investigations on build and CI performance. +//! +//! As this module requires additional dependencies not present during local builds, it's cfg'd +//! away whenever the `build.metrics` config option is not set to `true`. + +use crate::core::builder::{Builder, Step}; +use crate::utils::helpers::t; +use crate::Build; +use build_helper::metrics::{ + JsonInvocation, JsonInvocationSystemStats, JsonNode, JsonRoot, JsonStepSystemStats, Test, + TestOutcome, TestSuite, TestSuiteMetadata, +}; +use std::cell::RefCell; +use std::fs::File; +use std::io::BufWriter; +use std::time::{Duration, Instant, SystemTime}; +use sysinfo::{CpuExt, System, SystemExt}; + +// Update this number whenever a breaking change is made to the build metrics. +// +// The output format is versioned for two reasons: +// +// - The metadata is intended to be consumed by external tooling, and exposing a format version +// helps the tools determine whether they're compatible with a metrics file. +// +// - If a developer enables build metrics in their local checkout, making a breaking change to the +// metrics format would result in a hard-to-diagnose error message when an existing metrics file +// is not compatible with the new changes. With a format version number, bootstrap can discard +// incompatible metrics files instead of appending metrics to them. +// +// Version changelog: +// +// - v0: initial version +// - v1: replaced JsonNode::Test with JsonNode::TestSuite +// +const CURRENT_FORMAT_VERSION: usize = 1; + +pub(crate) struct BuildMetrics { + state: RefCell, +} + +/// NOTE: this isn't really cloning anything, but `x suggest` doesn't need metrics so this is probably ok. +impl Clone for BuildMetrics { + fn clone(&self) -> Self { + Self::init() + } +} + +impl BuildMetrics { + pub(crate) fn init() -> Self { + let state = RefCell::new(MetricsState { + finished_steps: Vec::new(), + running_steps: Vec::new(), + + system_info: System::new(), + timer_start: None, + invocation_timer_start: Instant::now(), + invocation_start: SystemTime::now(), + }); + + BuildMetrics { state } + } + + pub(crate) fn enter_step(&self, step: &S, builder: &Builder<'_>) { + // Do not record dry runs, as they'd be duplicates of the actual steps. + if builder.config.dry_run() { + return; + } + + let mut state = self.state.borrow_mut(); + + // Consider all the stats gathered so far as the parent's. + if !state.running_steps.is_empty() { + self.collect_stats(&mut *state); + } + + state.system_info.refresh_cpu(); + state.timer_start = Some(Instant::now()); + + state.running_steps.push(StepMetrics { + type_: std::any::type_name::().into(), + debug_repr: format!("{step:?}"), + + cpu_usage_time_sec: 0.0, + duration_excluding_children_sec: Duration::ZERO, + + children: Vec::new(), + test_suites: Vec::new(), + }); + } + + pub(crate) fn exit_step(&self, builder: &Builder<'_>) { + // Do not record dry runs, as they'd be duplicates of the actual steps. + if builder.config.dry_run() { + return; + } + + let mut state = self.state.borrow_mut(); + + self.collect_stats(&mut *state); + + let step = state.running_steps.pop().unwrap(); + if state.running_steps.is_empty() { + state.finished_steps.push(step); + state.timer_start = None; + } else { + state.running_steps.last_mut().unwrap().children.push(step); + + // Start collecting again for the parent step. + state.system_info.refresh_cpu(); + state.timer_start = Some(Instant::now()); + } + } + + pub(crate) fn begin_test_suite(&self, metadata: TestSuiteMetadata, builder: &Builder<'_>) { + // Do not record dry runs, as they'd be duplicates of the actual steps. + if builder.config.dry_run() { + return; + } + + let mut state = self.state.borrow_mut(); + let step = state.running_steps.last_mut().unwrap(); + step.test_suites.push(TestSuite { metadata, tests: Vec::new() }); + } + + pub(crate) fn record_test(&self, name: &str, outcome: TestOutcome, builder: &Builder<'_>) { + // Do not record dry runs, as they'd be duplicates of the actual steps. + if builder.config.dry_run() { + return; + } + + let mut state = self.state.borrow_mut(); + let step = state.running_steps.last_mut().unwrap(); + + if let Some(test_suite) = step.test_suites.last_mut() { + test_suite.tests.push(Test { name: name.to_string(), outcome }); + } else { + panic!("metrics.record_test() called without calling metrics.begin_test_suite() first"); + } + } + + fn collect_stats(&self, state: &mut MetricsState) { + let step = state.running_steps.last_mut().unwrap(); + + let elapsed = state.timer_start.unwrap().elapsed(); + step.duration_excluding_children_sec += elapsed; + + state.system_info.refresh_cpu(); + let cpu = state.system_info.cpus().iter().map(|p| p.cpu_usage()).sum::(); + step.cpu_usage_time_sec += cpu as f64 / 100.0 * elapsed.as_secs_f64(); + } + + pub(crate) fn persist(&self, build: &Build) { + let mut state = self.state.borrow_mut(); + assert!(state.running_steps.is_empty(), "steps are still executing"); + + let dest = build.out.join("metrics.json"); + + let mut system = System::new(); + system.refresh_cpu(); + system.refresh_memory(); + + let system_stats = JsonInvocationSystemStats { + cpu_threads_count: system.cpus().len(), + cpu_model: system.cpus()[0].brand().into(), + + memory_total_bytes: system.total_memory(), + }; + let steps = std::mem::take(&mut state.finished_steps); + + // Some of our CI builds consist of multiple independent CI invocations. Ensure all the + // previous invocations are still present in the resulting file. + let mut invocations = match std::fs::read(&dest) { + Ok(contents) => { + // We first parse just the format_version field to have the check succeed even if + // the rest of the contents are not valid anymore. + let version: OnlyFormatVersion = t!(serde_json::from_slice(&contents)); + if version.format_version == CURRENT_FORMAT_VERSION { + t!(serde_json::from_slice::(&contents)).invocations + } else { + println!( + "WARNING: overriding existing build/metrics.json, as it's not \ + compatible with build metrics format version {CURRENT_FORMAT_VERSION}." + ); + Vec::new() + } + } + Err(err) => { + if err.kind() != std::io::ErrorKind::NotFound { + panic!("failed to open existing metrics file at {}: {err}", dest.display()); + } + Vec::new() + } + }; + invocations.push(JsonInvocation { + start_time: state + .invocation_start + .duration_since(SystemTime::UNIX_EPOCH) + .unwrap() + .as_secs(), + duration_including_children_sec: state.invocation_timer_start.elapsed().as_secs_f64(), + children: steps.into_iter().map(|step| self.prepare_json_step(step)).collect(), + }); + + let json = JsonRoot { format_version: CURRENT_FORMAT_VERSION, system_stats, invocations }; + + t!(std::fs::create_dir_all(dest.parent().unwrap())); + let mut file = BufWriter::new(t!(File::create(&dest))); + t!(serde_json::to_writer(&mut file, &json)); + } + + fn prepare_json_step(&self, step: StepMetrics) -> JsonNode { + let mut children = Vec::new(); + children.extend(step.children.into_iter().map(|child| self.prepare_json_step(child))); + children.extend(step.test_suites.into_iter().map(JsonNode::TestSuite)); + + JsonNode::RustbuildStep { + type_: step.type_, + debug_repr: step.debug_repr, + + duration_excluding_children_sec: step.duration_excluding_children_sec.as_secs_f64(), + system_stats: JsonStepSystemStats { + cpu_utilization_percent: step.cpu_usage_time_sec * 100.0 + / step.duration_excluding_children_sec.as_secs_f64(), + }, + + children, + } + } +} + +struct MetricsState { + finished_steps: Vec, + running_steps: Vec, + + system_info: System, + timer_start: Option, + invocation_timer_start: Instant, + invocation_start: SystemTime, +} + +struct StepMetrics { + type_: String, + debug_repr: String, + + cpu_usage_time_sec: f64, + duration_excluding_children_sec: Duration, + + children: Vec, + test_suites: Vec, +} + +#[derive(serde_derive::Deserialize)] +struct OnlyFormatVersion { + #[serde(default)] // For version 0 the field was not present. + format_version: usize, +} diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs new file mode 100644 index 000000000..8ca22d008 --- /dev/null +++ b/src/bootstrap/src/utils/mod.rs @@ -0,0 +1,15 @@ +//! This module contains integral components of the build and configuration process, providing +//! support for a wide range of tasks and operations such as caching, tarballs, release +//! channels, job management, etc. + +pub(crate) mod cache; +pub(crate) mod cc_detect; +pub(crate) mod channel; +pub(crate) mod dylib; +pub(crate) mod exec; +pub(crate) mod helpers; +pub(crate) mod job; +#[cfg(feature = "build-metrics")] +pub(crate) mod metrics; +pub(crate) mod render_tests; +pub(crate) mod tarball; diff --git a/src/bootstrap/src/utils/render_tests.rs b/src/bootstrap/src/utils/render_tests.rs new file mode 100644 index 000000000..bff47f65c --- /dev/null +++ b/src/bootstrap/src/utils/render_tests.rs @@ -0,0 +1,400 @@ +//! This module renders the JSON output of libtest into a human-readable form, trying to be as +//! similar to libtest's native output as possible. +//! +//! This is needed because we need to use libtest in JSON mode to extract granular information +//! about the executed tests. Doing so suppresses the human-readable output, and (compared to Cargo +//! and rustc) libtest doesn't include the rendered human-readable output as a JSON field. We had +//! to reimplement all the rendering logic in this module because of that. + +use crate::core::builder::Builder; +use std::io::{BufRead, BufReader, Read, Write}; +use std::process::{ChildStdout, Command, Stdio}; +use std::time::Duration; +use termcolor::{Color, ColorSpec, WriteColor}; + +const TERSE_TESTS_PER_LINE: usize = 88; + +pub(crate) fn add_flags_and_try_run_tests(builder: &Builder<'_>, cmd: &mut Command) -> bool { + if cmd.get_args().position(|arg| arg == "--").is_none() { + cmd.arg("--"); + } + cmd.args(&["-Z", "unstable-options", "--format", "json"]); + + try_run_tests(builder, cmd, false) +} + +pub(crate) fn try_run_tests(builder: &Builder<'_>, cmd: &mut Command, stream: bool) -> bool { + if builder.config.dry_run() { + return true; + } + + if !run_tests(builder, cmd, stream) { + if builder.fail_fast { + crate::exit!(1); + } else { + let mut failures = builder.delayed_failures.borrow_mut(); + failures.push(format!("{cmd:?}")); + false + } + } else { + true + } +} + +fn run_tests(builder: &Builder<'_>, cmd: &mut Command, stream: bool) -> bool { + cmd.stdout(Stdio::piped()); + + builder.verbose(&format!("running: {cmd:?}")); + + let mut process = cmd.spawn().unwrap(); + + // This runs until the stdout of the child is closed, which means the child exited. We don't + // run this on another thread since the builder is not Sync. + let renderer = Renderer::new(process.stdout.take().unwrap(), builder); + if stream { + renderer.stream_all(); + } else { + renderer.render_all(); + } + + let result = process.wait_with_output().unwrap(); + if !result.status.success() && builder.is_verbose() { + println!( + "\n\ncommand did not execute successfully: {cmd:?}\n\ + expected success, got: {}", + result.status + ); + } + + result.status.success() +} + +struct Renderer<'a> { + stdout: BufReader, + failures: Vec, + benches: Vec, + builder: &'a Builder<'a>, + tests_count: Option, + executed_tests: usize, + terse_tests_in_line: usize, +} + +impl<'a> Renderer<'a> { + fn new(stdout: ChildStdout, builder: &'a Builder<'a>) -> Self { + Self { + stdout: BufReader::new(stdout), + benches: Vec::new(), + failures: Vec::new(), + builder, + tests_count: None, + executed_tests: 0, + terse_tests_in_line: 0, + } + } + + fn render_all(mut self) { + let mut line = Vec::new(); + loop { + line.clear(); + match self.stdout.read_until(b'\n', &mut line) { + Ok(_) => {} + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(err) => panic!("failed to read output of test runner: {err}"), + } + if line.is_empty() { + break; + } + + match serde_json::from_slice(&line) { + Ok(parsed) => self.render_message(parsed), + Err(_err) => { + // Handle non-JSON output, for example when --nocapture is passed. + let mut stdout = std::io::stdout(); + stdout.write_all(&line).unwrap(); + let _ = stdout.flush(); + } + } + } + } + + /// Renders the stdout characters one by one + fn stream_all(mut self) { + let mut buffer = [0; 1]; + loop { + match self.stdout.read(&mut buffer) { + Ok(0) => break, + Ok(_) => { + let mut stdout = std::io::stdout(); + stdout.write_all(&buffer).unwrap(); + let _ = stdout.flush(); + } + Err(err) if err.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(err) => panic!("failed to read output of test runner: {err}"), + } + } + } + + fn render_test_outcome(&mut self, outcome: Outcome<'_>, test: &TestOutcome) { + self.executed_tests += 1; + + #[cfg(feature = "build-metrics")] + self.builder.metrics.record_test( + &test.name, + match outcome { + Outcome::Ok | Outcome::BenchOk => build_helper::metrics::TestOutcome::Passed, + Outcome::Failed => build_helper::metrics::TestOutcome::Failed, + Outcome::Ignored { reason } => build_helper::metrics::TestOutcome::Ignored { + ignore_reason: reason.map(|s| s.to_string()), + }, + }, + self.builder, + ); + + if self.builder.config.verbose_tests { + self.render_test_outcome_verbose(outcome, test); + } else { + self.render_test_outcome_terse(outcome, test); + } + } + + fn render_test_outcome_verbose(&self, outcome: Outcome<'_>, test: &TestOutcome) { + print!("test {} ... ", test.name); + self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); + if let Some(exec_time) = test.exec_time { + print!(" ({exec_time:.2?})"); + } + println!(); + } + + fn render_test_outcome_terse(&mut self, outcome: Outcome<'_>, _: &TestOutcome) { + if self.terse_tests_in_line != 0 && self.terse_tests_in_line % TERSE_TESTS_PER_LINE == 0 { + if let Some(total) = self.tests_count { + let total = total.to_string(); + let executed = format!("{:>width$}", self.executed_tests - 1, width = total.len()); + print!(" {executed}/{total}"); + } + println!(); + self.terse_tests_in_line = 0; + } + + self.terse_tests_in_line += 1; + self.builder.colored_stdout(|stdout| outcome.write_short(stdout)).unwrap(); + let _ = std::io::stdout().flush(); + } + + fn render_suite_outcome(&self, outcome: Outcome<'_>, suite: &SuiteOutcome) { + // The terse output doesn't end with a newline, so we need to add it ourselves. + if !self.builder.config.verbose_tests { + println!(); + } + + if !self.failures.is_empty() { + println!("\nfailures:\n"); + for failure in &self.failures { + if failure.stdout.is_some() || failure.message.is_some() { + println!("---- {} stdout ----", failure.name); + if let Some(stdout) = &failure.stdout { + println!("{stdout}"); + } + if let Some(message) = &failure.message { + println!("NOTE: {message}"); + } + } + } + + println!("\nfailures:"); + for failure in &self.failures { + println!(" {}", failure.name); + } + } + + if !self.benches.is_empty() { + println!("\nbenchmarks:"); + + let mut rows = Vec::new(); + for bench in &self.benches { + rows.push(( + &bench.name, + format!("{:.2?}/iter", Duration::from_nanos(bench.median)), + format!("+/- {:.2?}", Duration::from_nanos(bench.deviation)), + )); + } + + let max_0 = rows.iter().map(|r| r.0.len()).max().unwrap_or(0); + let max_1 = rows.iter().map(|r| r.1.len()).max().unwrap_or(0); + let max_2 = rows.iter().map(|r| r.2.len()).max().unwrap_or(0); + for row in &rows { + println!(" {:max_1$} {:>max_2$}", row.0, row.1, row.2); + } + } + + print!("\ntest result: "); + self.builder.colored_stdout(|stdout| outcome.write_long(stdout)).unwrap(); + println!( + ". {} passed; {} failed; {} ignored; {} measured; {} filtered out; \ + finished in {:.2?}\n", + suite.passed, + suite.failed, + suite.ignored, + suite.measured, + suite.filtered_out, + Duration::from_secs_f64(suite.exec_time) + ); + } + + fn render_message(&mut self, message: Message) { + match message { + Message::Suite(SuiteMessage::Started { test_count }) => { + println!("\nrunning {test_count} tests"); + self.executed_tests = 0; + self.terse_tests_in_line = 0; + self.tests_count = Some(test_count); + } + Message::Suite(SuiteMessage::Ok(outcome)) => { + self.render_suite_outcome(Outcome::Ok, &outcome); + } + Message::Suite(SuiteMessage::Failed(outcome)) => { + self.render_suite_outcome(Outcome::Failed, &outcome); + } + Message::Bench(outcome) => { + // The formatting for benchmarks doesn't replicate 1:1 the formatting libtest + // outputs, mostly because libtest's formatting is broken in terse mode, which is + // the default used by our monorepo. We use a different formatting instead: + // successful benchmarks are just showed as "benchmarked"/"b", and the details are + // outputted at the bottom like failures. + let fake_test_outcome = TestOutcome { + name: outcome.name.clone(), + exec_time: None, + stdout: None, + message: None, + }; + self.render_test_outcome(Outcome::BenchOk, &fake_test_outcome); + self.benches.push(outcome); + } + Message::Test(TestMessage::Ok(outcome)) => { + self.render_test_outcome(Outcome::Ok, &outcome); + } + Message::Test(TestMessage::Ignored(outcome)) => { + self.render_test_outcome( + Outcome::Ignored { reason: outcome.message.as_deref() }, + &outcome, + ); + } + Message::Test(TestMessage::Failed(outcome)) => { + self.render_test_outcome(Outcome::Failed, &outcome); + self.failures.push(outcome); + } + Message::Test(TestMessage::Timeout { name }) => { + println!("test {name} has been running for a long time"); + } + Message::Test(TestMessage::Started) => {} // Not useful + } + } +} + +enum Outcome<'a> { + Ok, + BenchOk, + Failed, + Ignored { reason: Option<&'a str> }, +} + +impl Outcome<'_> { + fn write_short(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { + match self { + Outcome::Ok => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; + write!(writer, ".")?; + } + Outcome::BenchOk => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; + write!(writer, "b")?; + } + Outcome::Failed => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; + write!(writer, "F")?; + } + Outcome::Ignored { .. } => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; + write!(writer, "i")?; + } + } + writer.reset() + } + + fn write_long(&self, writer: &mut dyn WriteColor) -> Result<(), std::io::Error> { + match self { + Outcome::Ok => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Green)))?; + write!(writer, "ok")?; + } + Outcome::BenchOk => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Cyan)))?; + write!(writer, "benchmarked")?; + } + Outcome::Failed => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Red)))?; + write!(writer, "FAILED")?; + } + Outcome::Ignored { reason } => { + writer.set_color(&ColorSpec::new().set_fg(Some(Color::Yellow)))?; + write!(writer, "ignored")?; + if let Some(reason) = reason { + write!(writer, ", {reason}")?; + } + } + } + writer.reset() + } +} + +#[derive(serde_derive::Deserialize)] +#[serde(tag = "type", rename_all = "snake_case")] +enum Message { + Suite(SuiteMessage), + Test(TestMessage), + Bench(BenchOutcome), +} + +#[derive(serde_derive::Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum SuiteMessage { + Ok(SuiteOutcome), + Failed(SuiteOutcome), + Started { test_count: usize }, +} + +#[derive(serde_derive::Deserialize)] +struct SuiteOutcome { + passed: usize, + failed: usize, + ignored: usize, + measured: usize, + filtered_out: usize, + exec_time: f64, +} + +#[derive(serde_derive::Deserialize)] +#[serde(tag = "event", rename_all = "snake_case")] +enum TestMessage { + Ok(TestOutcome), + Failed(TestOutcome), + Ignored(TestOutcome), + Timeout { name: String }, + Started, +} + +#[derive(serde_derive::Deserialize)] +struct BenchOutcome { + name: String, + median: u64, + deviation: u64, +} + +#[derive(serde_derive::Deserialize)] +struct TestOutcome { + name: String, + exec_time: Option, + stdout: Option, + message: Option, +} diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs new file mode 100644 index 000000000..a8393f88f --- /dev/null +++ b/src/bootstrap/src/utils/tarball.rs @@ -0,0 +1,381 @@ +use std::{ + path::{Path, PathBuf}, + process::Command, +}; + +use crate::core::build_steps::dist::distdir; +use crate::core::builder::Builder; +use crate::utils::channel; +use crate::utils::helpers::t; + +#[derive(Copy, Clone)] +pub(crate) enum OverlayKind { + Rust, + LLVM, + Cargo, + Clippy, + Miri, + Rustfmt, + RustDemangler, + RLS, + RustAnalyzer, + RustcCodegenCranelift, +} + +impl OverlayKind { + fn legal_and_readme(&self) -> &[&str] { + match self { + OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], + OverlayKind::LLVM => { + &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] + } + OverlayKind::Cargo => &[ + "src/tools/cargo/README.md", + "src/tools/cargo/LICENSE-MIT", + "src/tools/cargo/LICENSE-APACHE", + "src/tools/cargo/LICENSE-THIRD-PARTY", + ], + OverlayKind::Clippy => &[ + "src/tools/clippy/README.md", + "src/tools/clippy/LICENSE-APACHE", + "src/tools/clippy/LICENSE-MIT", + ], + OverlayKind::Miri => &[ + "src/tools/miri/README.md", + "src/tools/miri/LICENSE-APACHE", + "src/tools/miri/LICENSE-MIT", + ], + OverlayKind::Rustfmt => &[ + "src/tools/rustfmt/README.md", + "src/tools/rustfmt/LICENSE-APACHE", + "src/tools/rustfmt/LICENSE-MIT", + ], + OverlayKind::RustDemangler => { + &["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"] + } + OverlayKind::RLS => &["src/tools/rls/README.md", "LICENSE-APACHE", "LICENSE-MIT"], + OverlayKind::RustAnalyzer => &[ + "src/tools/rust-analyzer/README.md", + "src/tools/rust-analyzer/LICENSE-APACHE", + "src/tools/rust-analyzer/LICENSE-MIT", + ], + OverlayKind::RustcCodegenCranelift => &[ + "compiler/rustc_codegen_cranelift/Readme.md", + "compiler/rustc_codegen_cranelift/LICENSE-APACHE", + "compiler/rustc_codegen_cranelift/LICENSE-MIT", + ], + } + } + + fn version(&self, builder: &Builder<'_>) -> String { + match self { + OverlayKind::Rust => builder.rust_version(), + OverlayKind::LLVM => builder.rust_version(), + OverlayKind::RustDemangler => builder.release_num("rust-demangler"), + OverlayKind::Cargo => { + builder.cargo_info.version(builder, &builder.release_num("cargo")) + } + OverlayKind::Clippy => { + builder.clippy_info.version(builder, &builder.release_num("clippy")) + } + OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), + OverlayKind::Rustfmt => { + builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) + } + OverlayKind::RLS => builder.release(&builder.release_num("rls")), + OverlayKind::RustAnalyzer => builder + .rust_analyzer_info + .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), + OverlayKind::RustcCodegenCranelift => builder.rust_version(), + } + } +} + +pub(crate) struct Tarball<'a> { + builder: &'a Builder<'a>, + + pkgname: String, + component: String, + target: Option, + product_name: String, + overlay: OverlayKind, + + temp_dir: PathBuf, + image_dir: PathBuf, + overlay_dir: PathBuf, + bulk_dirs: Vec, + + include_target_in_component_name: bool, + is_preview: bool, + permit_symlinks: bool, +} + +impl<'a> Tarball<'a> { + pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { + Self::new_inner(builder, component, Some(target.into())) + } + + pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self { + Self::new_inner(builder, component, None) + } + + fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option) -> Self { + let pkgname = crate::core::build_steps::dist::pkgname(builder, component); + + let mut temp_dir = builder.out.join("tmp").join("tarball").join(component); + if let Some(target) = &target { + temp_dir = temp_dir.join(target); + } + let _ = std::fs::remove_dir_all(&temp_dir); + + let image_dir = temp_dir.join("image"); + let overlay_dir = temp_dir.join("overlay"); + + Self { + builder, + + pkgname, + component: component.into(), + target, + product_name: "Rust".into(), + overlay: OverlayKind::Rust, + + temp_dir, + image_dir, + overlay_dir, + bulk_dirs: Vec::new(), + + include_target_in_component_name: false, + is_preview: false, + permit_symlinks: false, + } + } + + pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) { + self.overlay = overlay; + } + + pub(crate) fn set_product_name(&mut self, name: &str) { + self.product_name = name.into(); + } + + pub(crate) fn include_target_in_component_name(&mut self, include: bool) { + self.include_target_in_component_name = include; + } + + pub(crate) fn is_preview(&mut self, is: bool) { + self.is_preview = is; + } + + pub(crate) fn permit_symlinks(&mut self, flag: bool) { + self.permit_symlinks = flag; + } + + pub(crate) fn image_dir(&self) -> &Path { + t!(std::fs::create_dir_all(&self.image_dir)); + &self.image_dir + } + + pub(crate) fn add_file(&self, src: impl AsRef, destdir: impl AsRef, perms: u32) { + // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply + // uses the base directory as the destination directory. + let destdir = if destdir.as_ref() == Path::new(".") { + self.image_dir.clone() + } else { + self.image_dir.join(destdir.as_ref()) + }; + + t!(std::fs::create_dir_all(&destdir)); + self.builder.install(src.as_ref(), &destdir, perms); + } + + pub(crate) fn add_renamed_file( + &self, + src: impl AsRef, + destdir: impl AsRef, + new_name: &str, + ) { + let destdir = self.image_dir.join(destdir.as_ref()); + t!(std::fs::create_dir_all(&destdir)); + self.builder.copy(src.as_ref(), &destdir.join(new_name)); + } + + pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef) { + for file in self.overlay.legal_and_readme() { + self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644); + } + } + + pub(crate) fn add_dir(&self, src: impl AsRef, dest: impl AsRef) { + let dest = self.image_dir.join(dest.as_ref()); + + t!(std::fs::create_dir_all(&dest)); + self.builder.cp_r(src.as_ref(), &dest); + } + + pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef, dest: impl AsRef) { + self.bulk_dirs.push(dest.as_ref().to_path_buf()); + self.add_dir(src, dest); + } + + pub(crate) fn generate(self) -> GeneratedTarball { + let mut component_name = self.component.clone(); + if self.is_preview { + component_name.push_str("-preview"); + } + if self.include_target_in_component_name { + component_name.push('-'); + component_name.push_str( + &self + .target + .as_ref() + .expect("include_target_in_component_name used in a targetless tarball"), + ); + } + + self.run(|this, cmd| { + cmd.arg("generate") + .arg("--image-dir") + .arg(&this.image_dir) + .arg(format!("--component-name={}", &component_name)); + + if let Some((dir, dirs)) = this.bulk_dirs.split_first() { + let mut arg = dir.as_os_str().to_os_string(); + for dir in dirs { + arg.push(","); + arg.push(dir); + } + cmd.arg("--bulk-dirs").arg(&arg); + } + + this.non_bare_args(cmd); + }) + } + + pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball { + let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string(); + for tarball in &tarballs[1..] { + input_tarballs.push(","); + input_tarballs.push(&tarball.path); + } + + self.run(|this, cmd| { + cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs); + this.non_bare_args(cmd); + }) + } + + pub(crate) fn bare(self) -> GeneratedTarball { + // Bare tarballs should have the top level directory match the package + // name, not "image". We rename the image directory just before passing + // into rust-installer. + let dest = self.temp_dir.join(self.package_name()); + t!(std::fs::rename(&self.image_dir, &dest)); + + self.run(|this, cmd| { + let distdir = distdir(this.builder); + t!(std::fs::create_dir_all(&distdir)); + cmd.arg("tarball") + .arg("--input") + .arg(&dest) + .arg("--output") + .arg(distdir.join(this.package_name())); + }) + } + + fn package_name(&self) -> String { + if let Some(target) = &self.target { + format!("{}-{}", self.pkgname, target) + } else { + self.pkgname.clone() + } + } + + fn non_bare_args(&self, cmd: &mut Command) { + cmd.arg("--rel-manifest-dir=rustlib") + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg(format!("--product-name={}", self.product_name)) + .arg(format!("--success-message={} installed.", self.component)) + .arg(format!("--package-name={}", self.package_name())) + .arg("--non-installed-overlay") + .arg(&self.overlay_dir) + .arg("--output-dir") + .arg(distdir(self.builder)); + } + + fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball { + t!(std::fs::create_dir_all(&self.overlay_dir)); + self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); + if let Some(info) = self.builder.rust_info().info() { + channel::write_commit_hash_file(&self.overlay_dir, &info.sha); + channel::write_commit_info_file(&self.overlay_dir, info); + } + for file in self.overlay.legal_and_readme() { + self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); + } + + let mut cmd = self.builder.tool_cmd(crate::core::build_steps::tool::Tool::RustInstaller); + + let package_name = self.package_name(); + self.builder.info(&format!("Dist {package_name}")); + let _time = crate::utils::helpers::timeit(self.builder); + + build_cli(&self, &mut cmd); + cmd.arg("--work-dir").arg(&self.temp_dir); + if let Some(formats) = &self.builder.config.dist_compression_formats { + assert!(!formats.is_empty(), "dist.compression-formats can't be empty"); + cmd.arg("--compression-formats").arg(formats.join(",")); + } + cmd.args(&["--compression-profile", &self.builder.config.dist_compression_profile]); + self.builder.run(&mut cmd); + + // Ensure there are no symbolic links in the tarball. In particular, + // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links. + let decompressed_output = self.temp_dir.join(&package_name); + if !self.builder.config.dry_run() && !self.permit_symlinks { + for entry in walkdir::WalkDir::new(&decompressed_output) { + let entry = t!(entry); + if entry.path_is_symlink() { + panic!("generated a symlink in a tarball: {}", entry.path().display()); + } + } + } + + // Use either the first compression format defined, or "gz" as the default. + let ext = self + .builder + .config + .dist_compression_formats + .as_ref() + .and_then(|formats| formats.get(0)) + .map(|s| s.as_str()) + .unwrap_or("gz"); + + GeneratedTarball { + path: distdir(self.builder).join(format!("{package_name}.tar.{ext}")), + decompressed_output, + work: self.temp_dir, + } + } +} + +#[derive(Debug, Clone)] +pub struct GeneratedTarball { + path: PathBuf, + decompressed_output: PathBuf, + work: PathBuf, +} + +impl GeneratedTarball { + pub(crate) fn tarball(&self) -> &Path { + &self.path + } + + pub(crate) fn decompressed_output(&self) -> &Path { + &self.decompressed_output + } + + pub(crate) fn work_dir(&self) -> &Path { + &self.work + } +} diff --git a/src/bootstrap/suggest.rs b/src/bootstrap/suggest.rs deleted file mode 100644 index f225104bd..000000000 --- a/src/bootstrap/suggest.rs +++ /dev/null @@ -1,74 +0,0 @@ -#![cfg_attr(feature = "build-metrics", allow(unused))] - -use std::str::FromStr; - -use std::path::PathBuf; - -use clap::Parser; - -use crate::{builder::Builder, tool::Tool}; - -/// Suggests a list of possible `x.py` commands to run based on modified files in branch. -pub fn suggest(builder: &Builder<'_>, run: bool) { - let suggestions = - builder.tool_cmd(Tool::SuggestTests).output().expect("failed to run `suggest-tests` tool"); - - if !suggestions.status.success() { - println!("failed to run `suggest-tests` tool ({})", suggestions.status); - println!( - "`suggest_tests` stdout:\n{}`suggest_tests` stderr:\n{}", - String::from_utf8(suggestions.stdout).unwrap(), - String::from_utf8(suggestions.stderr).unwrap() - ); - panic!("failed to run `suggest-tests`"); - } - - let suggestions = String::from_utf8(suggestions.stdout).unwrap(); - let suggestions = suggestions - .lines() - .map(|line| { - let mut sections = line.split_ascii_whitespace(); - - // this code expects one suggestion per line in the following format: - // {some number of flags} [optional stage number] - let cmd = sections.next().unwrap(); - let stage = sections.next_back().map(|s| str::parse(s).ok()).flatten(); - let paths: Vec = sections.map(|p| PathBuf::from_str(p).unwrap()).collect(); - - (cmd, stage, paths) - }) - .collect::>(); - - if !suggestions.is_empty() { - println!("==== SUGGESTIONS ===="); - for sug in &suggestions { - print!("x {} ", sug.0); - if let Some(stage) = sug.1 { - print!("--stage {stage} "); - } - - for path in &sug.2 { - print!("{} ", path.display()); - } - println!(); - } - println!("====================="); - } else { - println!("No suggestions found!"); - return; - } - - if run { - for sug in suggestions { - let mut build: crate::Build = builder.build.clone(); - build.config.paths = sug.2; - build.config.cmd = crate::flags::Flags::parse_from(["x.py", sug.0]).cmd; - if let Some(stage) = sug.1 { - build.config.stage = stage; - } - build.build(); - } - } else { - println!("help: consider using the `--run` flag to automatically run suggested tests"); - } -} diff --git a/src/bootstrap/synthetic_targets.rs b/src/bootstrap/synthetic_targets.rs deleted file mode 100644 index 7eeac9025..000000000 --- a/src/bootstrap/synthetic_targets.rs +++ /dev/null @@ -1,82 +0,0 @@ -//! In some cases, parts of bootstrap need to change part of a target spec just for one or a few -//! steps. Adding these targets to rustc proper would "leak" this implementation detail of -//! bootstrap, and would make it more complex to apply additional changes if the need arises. -//! -//! To address that problem, this module implements support for "synthetic targets". Synthetic -//! targets are custom target specs generated using builtin target specs as their base. You can use -//! one of the target specs already defined in this module, or create new ones by adding a new step -//! that calls create_synthetic_target. - -use crate::builder::{Builder, ShouldRun, Step}; -use crate::config::TargetSelection; -use crate::Compiler; -use std::process::{Command, Stdio}; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub(crate) struct MirOptPanicAbortSyntheticTarget { - pub(crate) compiler: Compiler, - pub(crate) base: TargetSelection, -} - -impl Step for MirOptPanicAbortSyntheticTarget { - type Output = TargetSelection; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> Self::Output { - create_synthetic_target(builder, self.compiler, "miropt-abort", self.base, |spec| { - spec.insert("panic-strategy".into(), "abort".into()); - }) - } -} - -fn create_synthetic_target( - builder: &Builder<'_>, - compiler: Compiler, - suffix: &str, - base: TargetSelection, - customize: impl FnOnce(&mut serde_json::Map), -) -> TargetSelection { - if base.contains("synthetic") { - // This check is not strictly needed, but nothing currently needs recursive synthetic - // targets. If the need arises, removing this in the future *SHOULD* be safe. - panic!("cannot create synthetic targets with other synthetic targets as their base"); - } - - let name = format!("{base}-synthetic-{suffix}"); - let path = builder.out.join("synthetic-target-specs").join(format!("{name}.json")); - std::fs::create_dir_all(path.parent().unwrap()).unwrap(); - - if builder.config.dry_run() { - std::fs::write(&path, b"dry run\n").unwrap(); - return TargetSelection::create_synthetic(&name, path.to_str().unwrap()); - } - - let mut cmd = Command::new(builder.rustc(compiler)); - cmd.arg("--target").arg(base.rustc_target_arg()); - cmd.args(["-Zunstable-options", "--print", "target-spec-json"]); - cmd.stdout(Stdio::piped()); - - let output = cmd.spawn().unwrap().wait_with_output().unwrap(); - if !output.status.success() { - panic!("failed to gather the target spec for {base}"); - } - - let mut spec: serde_json::Value = serde_json::from_slice(&output.stdout).unwrap(); - let spec_map = spec.as_object_mut().unwrap(); - - // The `is-builtin` attribute of a spec needs to be removed, otherwise rustc will complain. - spec_map.remove("is-builtin"); - - customize(spec_map); - - std::fs::write(&path, &serde_json::to_vec_pretty(&spec).unwrap()).unwrap(); - let target = TargetSelection::create_synthetic(&name, path.to_str().unwrap()); - crate::cc_detect::find_target(builder, target); - - target -} diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs deleted file mode 100644 index 95d909c57..000000000 --- a/src/bootstrap/tarball.rs +++ /dev/null @@ -1,373 +0,0 @@ -use std::{ - path::{Path, PathBuf}, - process::Command, -}; - -use crate::builder::Builder; -use crate::channel; -use crate::util::t; - -#[derive(Copy, Clone)] -pub(crate) enum OverlayKind { - Rust, - LLVM, - Cargo, - Clippy, - Miri, - Rustfmt, - RustDemangler, - RLS, - RustAnalyzer, -} - -impl OverlayKind { - fn legal_and_readme(&self) -> &[&str] { - match self { - OverlayKind::Rust => &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"], - OverlayKind::LLVM => { - &["src/llvm-project/llvm/LICENSE.TXT", "src/llvm-project/llvm/README.txt"] - } - OverlayKind::Cargo => &[ - "src/tools/cargo/README.md", - "src/tools/cargo/LICENSE-MIT", - "src/tools/cargo/LICENSE-APACHE", - "src/tools/cargo/LICENSE-THIRD-PARTY", - ], - OverlayKind::Clippy => &[ - "src/tools/clippy/README.md", - "src/tools/clippy/LICENSE-APACHE", - "src/tools/clippy/LICENSE-MIT", - ], - OverlayKind::Miri => &[ - "src/tools/miri/README.md", - "src/tools/miri/LICENSE-APACHE", - "src/tools/miri/LICENSE-MIT", - ], - OverlayKind::Rustfmt => &[ - "src/tools/rustfmt/README.md", - "src/tools/rustfmt/LICENSE-APACHE", - "src/tools/rustfmt/LICENSE-MIT", - ], - OverlayKind::RustDemangler => { - &["src/tools/rust-demangler/README.md", "LICENSE-APACHE", "LICENSE-MIT"] - } - OverlayKind::RLS => &["src/tools/rls/README.md", "LICENSE-APACHE", "LICENSE-MIT"], - OverlayKind::RustAnalyzer => &[ - "src/tools/rust-analyzer/README.md", - "src/tools/rust-analyzer/LICENSE-APACHE", - "src/tools/rust-analyzer/LICENSE-MIT", - ], - } - } - - fn version(&self, builder: &Builder<'_>) -> String { - match self { - OverlayKind::Rust => builder.rust_version(), - OverlayKind::LLVM => builder.rust_version(), - OverlayKind::RustDemangler => builder.release_num("rust-demangler"), - OverlayKind::Cargo => { - builder.cargo_info.version(builder, &builder.release_num("cargo")) - } - OverlayKind::Clippy => { - builder.clippy_info.version(builder, &builder.release_num("clippy")) - } - OverlayKind::Miri => builder.miri_info.version(builder, &builder.release_num("miri")), - OverlayKind::Rustfmt => { - builder.rustfmt_info.version(builder, &builder.release_num("rustfmt")) - } - OverlayKind::RLS => builder.release(&builder.release_num("rls")), - OverlayKind::RustAnalyzer => builder - .rust_analyzer_info - .version(builder, &builder.release_num("rust-analyzer/crates/rust-analyzer")), - } - } -} - -pub(crate) struct Tarball<'a> { - builder: &'a Builder<'a>, - - pkgname: String, - component: String, - target: Option, - product_name: String, - overlay: OverlayKind, - - temp_dir: PathBuf, - image_dir: PathBuf, - overlay_dir: PathBuf, - bulk_dirs: Vec, - - include_target_in_component_name: bool, - is_preview: bool, - permit_symlinks: bool, -} - -impl<'a> Tarball<'a> { - pub(crate) fn new(builder: &'a Builder<'a>, component: &str, target: &str) -> Self { - Self::new_inner(builder, component, Some(target.into())) - } - - pub(crate) fn new_targetless(builder: &'a Builder<'a>, component: &str) -> Self { - Self::new_inner(builder, component, None) - } - - fn new_inner(builder: &'a Builder<'a>, component: &str, target: Option) -> Self { - let pkgname = crate::dist::pkgname(builder, component); - - let mut temp_dir = builder.out.join("tmp").join("tarball").join(component); - if let Some(target) = &target { - temp_dir = temp_dir.join(target); - } - let _ = std::fs::remove_dir_all(&temp_dir); - - let image_dir = temp_dir.join("image"); - let overlay_dir = temp_dir.join("overlay"); - - Self { - builder, - - pkgname, - component: component.into(), - target, - product_name: "Rust".into(), - overlay: OverlayKind::Rust, - - temp_dir, - image_dir, - overlay_dir, - bulk_dirs: Vec::new(), - - include_target_in_component_name: false, - is_preview: false, - permit_symlinks: false, - } - } - - pub(crate) fn set_overlay(&mut self, overlay: OverlayKind) { - self.overlay = overlay; - } - - pub(crate) fn set_product_name(&mut self, name: &str) { - self.product_name = name.into(); - } - - pub(crate) fn include_target_in_component_name(&mut self, include: bool) { - self.include_target_in_component_name = include; - } - - pub(crate) fn is_preview(&mut self, is: bool) { - self.is_preview = is; - } - - pub(crate) fn permit_symlinks(&mut self, flag: bool) { - self.permit_symlinks = flag; - } - - pub(crate) fn image_dir(&self) -> &Path { - t!(std::fs::create_dir_all(&self.image_dir)); - &self.image_dir - } - - pub(crate) fn add_file(&self, src: impl AsRef, destdir: impl AsRef, perms: u32) { - // create_dir_all fails to create `foo/bar/.`, so when the destination is "." this simply - // uses the base directory as the destination directory. - let destdir = if destdir.as_ref() == Path::new(".") { - self.image_dir.clone() - } else { - self.image_dir.join(destdir.as_ref()) - }; - - t!(std::fs::create_dir_all(&destdir)); - self.builder.install(src.as_ref(), &destdir, perms); - } - - pub(crate) fn add_renamed_file( - &self, - src: impl AsRef, - destdir: impl AsRef, - new_name: &str, - ) { - let destdir = self.image_dir.join(destdir.as_ref()); - t!(std::fs::create_dir_all(&destdir)); - self.builder.copy(src.as_ref(), &destdir.join(new_name)); - } - - pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef) { - for file in self.overlay.legal_and_readme() { - self.add_file(self.builder.src.join(file), destdir.as_ref(), 0o644); - } - } - - pub(crate) fn add_dir(&self, src: impl AsRef, dest: impl AsRef) { - let dest = self.image_dir.join(dest.as_ref()); - - t!(std::fs::create_dir_all(&dest)); - self.builder.cp_r(src.as_ref(), &dest); - } - - pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef, dest: impl AsRef) { - self.bulk_dirs.push(dest.as_ref().to_path_buf()); - self.add_dir(src, dest); - } - - pub(crate) fn generate(self) -> GeneratedTarball { - let mut component_name = self.component.clone(); - if self.is_preview { - component_name.push_str("-preview"); - } - if self.include_target_in_component_name { - component_name.push('-'); - component_name.push_str( - &self - .target - .as_ref() - .expect("include_target_in_component_name used in a targetless tarball"), - ); - } - - self.run(|this, cmd| { - cmd.arg("generate") - .arg("--image-dir") - .arg(&this.image_dir) - .arg(format!("--component-name={}", &component_name)); - - if let Some((dir, dirs)) = this.bulk_dirs.split_first() { - let mut arg = dir.as_os_str().to_os_string(); - for dir in dirs { - arg.push(","); - arg.push(dir); - } - cmd.arg("--bulk-dirs").arg(&arg); - } - - this.non_bare_args(cmd); - }) - } - - pub(crate) fn combine(self, tarballs: &[GeneratedTarball]) -> GeneratedTarball { - let mut input_tarballs = tarballs[0].path.as_os_str().to_os_string(); - for tarball in &tarballs[1..] { - input_tarballs.push(","); - input_tarballs.push(&tarball.path); - } - - self.run(|this, cmd| { - cmd.arg("combine").arg("--input-tarballs").arg(input_tarballs); - this.non_bare_args(cmd); - }) - } - - pub(crate) fn bare(self) -> GeneratedTarball { - // Bare tarballs should have the top level directory match the package - // name, not "image". We rename the image directory just before passing - // into rust-installer. - let dest = self.temp_dir.join(self.package_name()); - t!(std::fs::rename(&self.image_dir, &dest)); - - self.run(|this, cmd| { - let distdir = crate::dist::distdir(this.builder); - t!(std::fs::create_dir_all(&distdir)); - cmd.arg("tarball") - .arg("--input") - .arg(&dest) - .arg("--output") - .arg(distdir.join(this.package_name())); - }) - } - - fn package_name(&self) -> String { - if let Some(target) = &self.target { - format!("{}-{}", self.pkgname, target) - } else { - self.pkgname.clone() - } - } - - fn non_bare_args(&self, cmd: &mut Command) { - cmd.arg("--rel-manifest-dir=rustlib") - .arg("--legacy-manifest-dirs=rustlib,cargo") - .arg(format!("--product-name={}", self.product_name)) - .arg(format!("--success-message={} installed.", self.component)) - .arg(format!("--package-name={}", self.package_name())) - .arg("--non-installed-overlay") - .arg(&self.overlay_dir) - .arg("--output-dir") - .arg(crate::dist::distdir(self.builder)); - } - - fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball { - t!(std::fs::create_dir_all(&self.overlay_dir)); - self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); - if let Some(info) = self.builder.rust_info().info() { - channel::write_commit_hash_file(&self.overlay_dir, &info.sha); - channel::write_commit_info_file(&self.overlay_dir, info); - } - for file in self.overlay.legal_and_readme() { - self.builder.install(&self.builder.src.join(file), &self.overlay_dir, 0o644); - } - - let mut cmd = self.builder.tool_cmd(crate::tool::Tool::RustInstaller); - - let package_name = self.package_name(); - self.builder.info(&format!("Dist {package_name}")); - let _time = crate::util::timeit(self.builder); - - build_cli(&self, &mut cmd); - cmd.arg("--work-dir").arg(&self.temp_dir); - if let Some(formats) = &self.builder.config.dist_compression_formats { - assert!(!formats.is_empty(), "dist.compression-formats can't be empty"); - cmd.arg("--compression-formats").arg(formats.join(",")); - } - cmd.args(&["--compression-profile", &self.builder.config.dist_compression_profile]); - self.builder.run(&mut cmd); - - // Ensure there are no symbolic links in the tarball. In particular, - // rustup-toolchain-install-master and most versions of Windows can't handle symbolic links. - let decompressed_output = self.temp_dir.join(&package_name); - if !self.builder.config.dry_run() && !self.permit_symlinks { - for entry in walkdir::WalkDir::new(&decompressed_output) { - let entry = t!(entry); - if entry.path_is_symlink() { - panic!("generated a symlink in a tarball: {}", entry.path().display()); - } - } - } - - // Use either the first compression format defined, or "gz" as the default. - let ext = self - .builder - .config - .dist_compression_formats - .as_ref() - .and_then(|formats| formats.get(0)) - .map(|s| s.as_str()) - .unwrap_or("gz"); - - GeneratedTarball { - path: crate::dist::distdir(self.builder).join(format!("{package_name}.tar.{ext}")), - decompressed_output, - work: self.temp_dir, - } - } -} - -#[derive(Debug, Clone)] -pub struct GeneratedTarball { - path: PathBuf, - decompressed_output: PathBuf, - work: PathBuf, -} - -impl GeneratedTarball { - pub(crate) fn tarball(&self) -> &Path { - &self.path - } - - pub(crate) fn decompressed_output(&self) -> &Path { - &self.decompressed_output - } - - pub(crate) fn work_dir(&self) -> &Path { - &self.work - } -} diff --git a/src/bootstrap/test.rs b/src/bootstrap/test.rs deleted file mode 100644 index ba030f0f5..000000000 --- a/src/bootstrap/test.rs +++ /dev/null @@ -1,3093 +0,0 @@ -//! Implementation of the test-related targets of the build system. -//! -//! This file implements the various regression test suites that we execute on -//! our CI. - -use std::env; -use std::ffi::OsStr; -use std::ffi::OsString; -use std::fs; -use std::iter; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; - -use clap_complete::shells; - -use crate::builder::crate_description; -use crate::builder::{Builder, Compiler, Kind, RunConfig, ShouldRun, Step}; -use crate::cache::Interned; -use crate::cache::INTERNER; -use crate::compile; -use crate::config::TargetSelection; -use crate::dist; -use crate::doc::DocumentationFormat; -use crate::flags::Subcommand; -use crate::llvm; -use crate::render_tests::add_flags_and_try_run_tests; -use crate::synthetic_targets::MirOptPanicAbortSyntheticTarget; -use crate::tool::{self, SourceType, Tool}; -use crate::toolstate::ToolState; -use crate::util::{self, add_link_lib_path, dylib_path, dylib_path_var, output, t, up_to_date}; -use crate::{envify, CLang, DocTests, GitRepo, Mode}; - -const ADB_TEST_DIR: &str = "/data/local/tmp/work"; - -// mir-opt tests have different variants depending on whether a target is 32bit or 64bit, and -// blessing them requires blessing with each target. To aid developers, when blessing the mir-opt -// test suite the corresponding target of the opposite pointer size is also blessed. -// -// This array serves as the known mappings between 32bit and 64bit targets. If you're developing on -// a target where a target with the opposite pointer size exists, feel free to add it here. -const MIR_OPT_BLESS_TARGET_MAPPING: &[(&str, &str)] = &[ - // (32bit, 64bit) - ("i686-unknown-linux-gnu", "x86_64-unknown-linux-gnu"), - ("i686-unknown-linux-musl", "x86_64-unknown-linux-musl"), - ("i686-pc-windows-msvc", "x86_64-pc-windows-msvc"), - ("i686-pc-windows-gnu", "x86_64-pc-windows-gnu"), - ("i686-apple-darwin", "x86_64-apple-darwin"), - // ARM Macs don't have a corresponding 32-bit target that they can (easily) - // build for, so there is no entry for "aarch64-apple-darwin" here. -]; - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrateBootstrap { - path: Interned, - host: TargetSelection, -} - -impl Step for CrateBootstrap { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/jsondoclint") - .path("src/tools/suggest-tests") - .path("src/tools/replace-version-placeholder") - .alias("tidyselftest") - } - - fn make_run(run: RunConfig<'_>) { - for path in run.paths { - let path = INTERNER.intern_path(path.assert_single_path().path.clone()); - run.builder.ensure(CrateBootstrap { host: run.target, path }); - } - } - - fn run(self, builder: &Builder<'_>) { - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(0, bootstrap_host); - let mut path = self.path.to_str().unwrap(); - if path == "tidyselftest" { - path = "src/tools/tidy"; - } - - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - bootstrap_host, - "test", - path, - SourceType::InTree, - &[], - ); - let crate_name = path.rsplit_once('/').unwrap().1; - run_cargo_test(cargo, &[], &[], crate_name, crate_name, compiler, bootstrap_host, builder); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Linkcheck { - host: TargetSelection, -} - -impl Step for Linkcheck { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler. - /// - /// This tool in `src/tools` will verify the validity of all our links in the - /// documentation to ensure we don't have a bunch of dead ones. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let hosts = &builder.hosts; - let targets = &builder.targets; - - // if we have different hosts and targets, some things may be built for - // the host (e.g. rustc) and others for the target (e.g. std). The - // documentation built for each will contain broken links to - // docs built for the other platform (e.g. rustc linking to cargo) - if (hosts != targets) && !hosts.is_empty() && !targets.is_empty() { - panic!( - "Linkcheck currently does not support builds with different hosts and targets. -You can skip linkcheck with --skip src/tools/linkchecker" - ); - } - - builder.info(&format!("Linkcheck ({host})")); - - // Test the linkchecker itself. - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(0, bootstrap_host); - - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - bootstrap_host, - "test", - "src/tools/linkchecker", - SourceType::InTree, - &[], - ); - run_cargo_test( - cargo, - &[], - &[], - "linkchecker", - "linkchecker self tests", - compiler, - bootstrap_host, - builder, - ); - - if builder.doc_tests == DocTests::No { - return; - } - - // Build all the default documentation. - builder.default_doc(&[]); - - // Build the linkchecker before calling `msg`, since GHA doesn't support nested groups. - let mut linkchecker = builder.tool_cmd(Tool::Linkchecker); - - // Run the linkchecker. - let _guard = - builder.msg(Kind::Test, compiler.stage, "Linkcheck", bootstrap_host, bootstrap_host); - let _time = util::timeit(&builder); - builder.run_delaying_failure(linkchecker.arg(builder.out.join(host.triple).join("doc"))); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - let run = run.path("src/tools/linkchecker"); - run.default_condition(builder.config.docs) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Linkcheck { host: run.target }); - } -} - -fn check_if_tidy_is_installed() -> bool { - Command::new("tidy") - .arg("--version") - .stdout(Stdio::null()) - .status() - .map_or(false, |status| status.success()) -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct HtmlCheck { - target: TargetSelection, -} - -impl Step for HtmlCheck { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let run = run.path("src/tools/html-checker"); - run.lazy_default_condition(Box::new(check_if_tidy_is_installed)) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(HtmlCheck { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - if !check_if_tidy_is_installed() { - eprintln!("not running HTML-check tool because `tidy` is missing"); - eprintln!( - "Note that `tidy` is not the in-tree `src/tools/tidy` but needs to be installed" - ); - panic!("Cannot run html-check tests"); - } - // Ensure that a few different kinds of documentation are available. - builder.default_doc(&[]); - builder.ensure(crate::doc::Rustc::new(builder.top_stage, self.target, builder)); - - builder.run_delaying_failure( - builder.tool_cmd(Tool::HtmlChecker).arg(builder.doc_out(self.target)), - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Cargotest { - stage: u32, - host: TargetSelection, -} - -impl Step for Cargotest { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/cargotest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargotest { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs the `cargotest` tool as compiled in `stage` by the `host` compiler. - /// - /// This tool in `src/tools` will check out a few Rust projects and run `cargo - /// test` to ensure that we don't regress the test suites there. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(self.stage, self.host); - builder.ensure(compile::Rustc::new(compiler, compiler.host)); - let cargo = builder.ensure(tool::Cargo { compiler, target: compiler.host }); - - // Note that this is a short, cryptic, and not scoped directory name. This - // is currently to minimize the length of path on Windows where we otherwise - // quickly run into path name limit constraints. - let out_dir = builder.out.join("ct"); - t!(fs::create_dir_all(&out_dir)); - - let _time = util::timeit(&builder); - let mut cmd = builder.tool_cmd(Tool::CargoTest); - builder.run_delaying_failure( - cmd.arg(&cargo) - .arg(&out_dir) - .args(builder.config.test_args()) - .env("RUSTC", builder.rustc(compiler)) - .env("RUSTDOC", builder.rustdoc(compiler)), - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Cargo { - stage: u32, - host: TargetSelection, -} - -impl Step for Cargo { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/cargo") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for `cargo` packaged with Rust. - fn run(self, builder: &Builder<'_>) { - let compiler = builder.compiler(self.stage, self.host); - - builder.ensure(tool::Cargo { compiler, target: self.host }); - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - self.host, - "test", - "src/tools/cargo", - SourceType::Submodule, - &[], - ); - - // NOTE: can't use `run_cargo_test` because we need to overwrite `PATH` - let mut cargo = prepare_cargo_test(cargo, &[], &[], "cargo", compiler, self.host, builder); - - // Don't run cross-compile tests, we may not have cross-compiled libstd libs - // available. - cargo.env("CFG_DISABLE_CROSS_TESTS", "1"); - // Forcibly disable tests using nightly features since any changes to - // those features won't be able to land. - cargo.env("CARGO_TEST_DISABLE_NIGHTLY", "1"); - cargo.env("PATH", &path_for_cargo(builder, compiler)); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::CargoPackage { - crates: vec!["cargo".into()], - target: self.host.triple.to_string(), - host: self.host.triple.to_string(), - stage: self.stage, - }, - builder, - ); - - let _time = util::timeit(&builder); - add_flags_and_try_run_tests(builder, &mut cargo); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustAnalyzer { - stage: u32, - host: TargetSelection, -} - -impl Step for RustAnalyzer { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rust-analyzer") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for rust-analyzer - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - // We don't need to build the whole Rust Analyzer for the proc-macro-srv test suite, - // but we do need the standard library to be present. - builder.ensure(compile::Std::new(compiler, host)); - - let workspace_path = "src/tools/rust-analyzer"; - // until the whole RA test suite runs on `i686`, we only run - // `proc-macro-srv` tests - let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv"; - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolStd, - host, - "test", - crate_path, - SourceType::InTree, - &["sysroot-abi".to_owned()], - ); - cargo.allow_features(tool::RustAnalyzer::ALLOW_FEATURES); - - let dir = builder.src.join(workspace_path); - // needed by rust-analyzer to find its own text fixtures, cf. - // https://github.com/rust-analyzer/expect-test/issues/33 - cargo.env("CARGO_WORKSPACE_DIR", &dir); - - // RA's test suite tries to write to the source directory, that can't - // work in Rust CI - cargo.env("SKIP_SLOW_TESTS", "1"); - - cargo.add_rustc_lib_path(builder, compiler); - run_cargo_test(cargo, &[], &[], "rust-analyzer", "rust-analyzer", compiler, host, builder); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Rustfmt { - stage: u32, - host: TargetSelection, -} - -impl Step for Rustfmt { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustfmt") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustfmt { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for rustfmt. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - builder - .ensure(tool::Rustfmt { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/rustfmt", - SourceType::InTree, - &[], - ); - - let dir = testdir(builder, compiler.host); - t!(fs::create_dir_all(&dir)); - cargo.env("RUSTFMT_TEST_DIR", dir); - - cargo.add_rustc_lib_path(builder, compiler); - - run_cargo_test(cargo, &[], &[], "rustfmt", "rustfmt", compiler, host, builder); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustDemangler { - stage: u32, - host: TargetSelection, -} - -impl Step for RustDemangler { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rust-demangler") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustDemangler { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for rust-demangler. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - let rust_demangler = builder - .ensure(tool::RustDemangler { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/rust-demangler", - SourceType::InTree, - &[], - ); - - let dir = testdir(builder, compiler.host); - t!(fs::create_dir_all(&dir)); - - cargo.env("RUST_DEMANGLER_DRIVER_PATH", rust_demangler); - cargo.add_rustc_lib_path(builder, compiler); - - run_cargo_test( - cargo, - &[], - &[], - "rust-demangler", - "rust-demangler", - compiler, - host, - builder, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Miri { - stage: u32, - host: TargetSelection, - target: TargetSelection, -} - -impl Miri { - /// Run `cargo miri setup` for the given target, return where the Miri sysroot was put. - pub fn build_miri_sysroot( - builder: &Builder<'_>, - compiler: Compiler, - miri: &Path, - target: TargetSelection, - ) -> String { - let miri_sysroot = builder.out.join(compiler.host.triple).join("miri-sysroot"); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - compiler.host, - "run", - "src/tools/miri/cargo-miri", - SourceType::InTree, - &[], - ); - cargo.add_rustc_lib_path(builder, compiler); - cargo.arg("--").arg("miri").arg("setup"); - cargo.arg("--target").arg(target.rustc_target_arg()); - - // Tell `cargo miri setup` where to find the sources. - cargo.env("MIRI_LIB_SRC", builder.src.join("library")); - // Tell it where to find Miri. - cargo.env("MIRI", &miri); - // Tell it where to put the sysroot. - cargo.env("MIRI_SYSROOT", &miri_sysroot); - // Debug things. - cargo.env("RUST_BACKTRACE", "1"); - - let mut cargo = Command::from(cargo); - let _guard = builder.msg( - Kind::Build, - compiler.stage + 1, - "miri sysroot", - compiler.host, - compiler.host, - ); - builder.run(&mut cargo); - - // # Determine where Miri put its sysroot. - // To this end, we run `cargo miri setup --print-sysroot` and capture the output. - // (We do this separately from the above so that when the setup actually - // happens we get some output.) - // We re-use the `cargo` from above. - cargo.arg("--print-sysroot"); - - // FIXME: Is there a way in which we can re-use the usual `run` helpers? - if builder.config.dry_run() { - String::new() - } else { - builder.verbose(&format!("running: {cargo:?}")); - let out = - cargo.output().expect("We already ran `cargo miri setup` before and that worked"); - assert!(out.status.success(), "`cargo miri setup` returned with non-0 exit code"); - // Output is "\n". - let stdout = String::from_utf8(out.stdout) - .expect("`cargo miri setup` stdout is not valid UTF-8"); - let sysroot = stdout.trim_end(); - builder.verbose(&format!("`cargo miri setup --print-sysroot` said: {sysroot:?}")); - sysroot.to_owned() - } - } -} - -impl Step for Miri { - type Output = (); - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/miri") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Miri { - stage: run.builder.top_stage, - host: run.build_triple(), - target: run.target, - }); - } - - /// Runs `cargo test` for miri. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let target = self.target; - let compiler = builder.compiler(stage, host); - // We need the stdlib for the *next* stage, as it was built with this compiler that also built Miri. - // Except if we are at stage 2, the bootstrap loop is complete and we can stick with our current stage. - let compiler_std = builder.compiler(if stage < 2 { stage + 1 } else { stage }, host); - - let miri = builder - .ensure(tool::Miri { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - let _cargo_miri = builder - .ensure(tool::CargoMiri { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - // The stdlib we need might be at a different stage. And just asking for the - // sysroot does not seem to populate it, so we do that first. - builder.ensure(compile::Std::new(compiler_std, host)); - let sysroot = builder.sysroot(compiler_std); - // We also need a Miri sysroot. - let miri_sysroot = Miri::build_miri_sysroot(builder, compiler, &miri, target); - - // # Run `cargo test`. - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/miri", - SourceType::InTree, - &[], - ); - let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "miri", host, host); - - cargo.add_rustc_lib_path(builder, compiler); - - // miri tests need to know about the stage sysroot - cargo.env("MIRI_SYSROOT", &miri_sysroot); - cargo.env("MIRI_HOST_SYSROOT", sysroot); - cargo.env("MIRI", &miri); - if builder.config.locked_deps { - // enforce lockfiles - cargo.env("CARGO_EXTRA_FLAGS", "--locked"); - } - - // Set the target. - cargo.env("MIRI_TEST_TARGET", target.rustc_target_arg()); - - // This can NOT be `run_cargo_test` since the Miri test runner - // does not understand the flags added by `add_flags_and_try_run_test`. - let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); - { - let _time = util::timeit(&builder); - builder.run(&mut cargo); - } - - // Run it again for mir-opt-level 4 to catch some miscompilations. - if builder.config.test_args().is_empty() { - cargo.env("MIRIFLAGS", "-O -Zmir-opt-level=4 -Cdebug-assertions=yes"); - // Optimizations can change backtraces - cargo.env("MIRI_SKIP_UI_CHECKS", "1"); - // `MIRI_SKIP_UI_CHECKS` and `RUSTC_BLESS` are incompatible - cargo.env_remove("RUSTC_BLESS"); - // Optimizations can change error locations and remove UB so don't run `fail` tests. - cargo.args(&["tests/pass", "tests/panic"]); - - let mut cargo = prepare_cargo_test(cargo, &[], &[], "miri", compiler, target, builder); - { - let _time = util::timeit(&builder); - builder.run(&mut cargo); - } - } - - // # Run `cargo miri test`. - // This is just a smoke test (Miri's own CI invokes this in a bunch of different ways and ensures - // that we get the desired output), but that is sufficient to make sure that the libtest harness - // itself executes properly under Miri. - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "run", - "src/tools/miri/cargo-miri", - SourceType::Submodule, - &[], - ); - cargo.add_rustc_lib_path(builder, compiler); - cargo.arg("--").arg("miri").arg("test"); - if builder.config.locked_deps { - cargo.arg("--locked"); - } - cargo - .arg("--manifest-path") - .arg(builder.src.join("src/tools/miri/test-cargo-miri/Cargo.toml")); - cargo.arg("--target").arg(target.rustc_target_arg()); - cargo.arg("--tests"); // don't run doctests, they are too confused by the staging - cargo.arg("--").args(builder.config.test_args()); - - // Tell `cargo miri` where to find things. - cargo.env("MIRI_SYSROOT", &miri_sysroot); - cargo.env("MIRI_HOST_SYSROOT", sysroot); - cargo.env("MIRI", &miri); - // Debug things. - cargo.env("RUST_BACKTRACE", "1"); - - let mut cargo = Command::from(cargo); - { - let _time = util::timeit(&builder); - builder.run(&mut cargo); - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CompiletestTest { - host: TargetSelection, -} - -impl Step for CompiletestTest { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/compiletest") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(CompiletestTest { host: run.target }); - } - - /// Runs `cargo test` for compiletest. - fn run(self, builder: &Builder<'_>) { - let host = self.host; - let compiler = builder.compiler(builder.top_stage, host); - - // We need `ToolStd` for the locally-built sysroot because - // compiletest uses unstable features of the `test` crate. - builder.ensure(compile::Std::new(compiler, host)); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolStd, - host, - "test", - "src/tools/compiletest", - SourceType::InTree, - &[], - ); - cargo.allow_features("test"); - run_cargo_test( - cargo, - &[], - &[], - "compiletest", - "compiletest self test", - compiler, - host, - builder, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Clippy { - stage: u32, - host: TargetSelection, -} - -impl Step for Clippy { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/clippy") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Clippy { stage: run.builder.top_stage, host: run.target }); - } - - /// Runs `cargo test` for clippy. - fn run(self, builder: &Builder<'_>) { - let stage = self.stage; - let host = self.host; - let compiler = builder.compiler(stage, host); - - builder - .ensure(tool::Clippy { compiler, target: self.host, extra_features: Vec::new() }) - .expect("in-tree tool"); - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - host, - "test", - "src/tools/clippy", - SourceType::InTree, - &[], - ); - - cargo.env("RUSTC_TEST_SUITE", builder.rustc(compiler)); - cargo.env("RUSTC_LIB_PATH", builder.rustc_libdir(compiler)); - let host_libs = builder.stage_out(compiler, Mode::ToolRustc).join(builder.cargo_dir()); - cargo.env("HOST_LIBS", host_libs); - - cargo.add_rustc_lib_path(builder, compiler); - let mut cargo = prepare_cargo_test(cargo, &[], &[], "clippy", compiler, host, builder); - - let _guard = builder.msg_sysroot_tool(Kind::Test, compiler.stage, "clippy", host, host); - - #[allow(deprecated)] // Clippy reports errors if it blessed the outputs - if builder.config.try_run(&mut cargo).is_ok() { - // The tests succeeded; nothing to do. - return; - } - - if !builder.config.cmd.bless() { - crate::exit!(1); - } - } -} - -fn path_for_cargo(builder: &Builder<'_>, compiler: Compiler) -> OsString { - // Configure PATH to find the right rustc. NB. we have to use PATH - // and not RUSTC because the Cargo test suite has tests that will - // fail if rustc is not spelled `rustc`. - let path = builder.sysroot(compiler).join("bin"); - let old_path = env::var_os("PATH").unwrap_or_default(); - env::join_paths(iter::once(path).chain(env::split_paths(&old_path))).expect("") -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustdocTheme { - pub compiler: Compiler, -} - -impl Step for RustdocTheme { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustdoc-themes") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.target); - - run.builder.ensure(RustdocTheme { compiler }); - } - - fn run(self, builder: &Builder<'_>) { - let rustdoc = builder.bootstrap_out.join("rustdoc"); - let mut cmd = builder.tool_cmd(Tool::RustdocTheme); - cmd.arg(rustdoc.to_str().unwrap()) - .arg(builder.src.join("src/librustdoc/html/static/css/rustdoc.css").to_str().unwrap()) - .env("RUSTC_STAGE", self.compiler.stage.to_string()) - .env("RUSTC_SYSROOT", builder.sysroot(self.compiler)) - .env("RUSTDOC_LIBDIR", builder.sysroot_libdir(self.compiler, self.compiler.host)) - .env("CFG_RELEASE_CHANNEL", &builder.config.channel) - .env("RUSTDOC_REAL", builder.rustdoc(self.compiler)) - .env("RUSTC_BOOTSTRAP", "1"); - if let Some(linker) = builder.linker(self.compiler.host) { - cmd.env("RUSTDOC_LINKER", linker); - } - if builder.is_fuse_ld_lld(self.compiler.host) { - cmd.env( - "RUSTDOC_LLD_NO_THREADS", - util::lld_flag_no_threads(self.compiler.host.contains("windows")), - ); - } - builder.run_delaying_failure(&mut cmd); - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustdocJSStd { - pub target: TargetSelection, -} - -impl Step for RustdocJSStd { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.nodejs.is_some(); - run.suite_path("tests/rustdoc-js-std").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustdocJSStd { target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let nodejs = - builder.config.nodejs.as_ref().expect("need nodejs to run rustdoc-js-std tests"); - let mut command = Command::new(nodejs); - command - .arg(builder.src.join("src/tools/rustdoc-js/tester.js")) - .arg("--crate-name") - .arg("std") - .arg("--resource-suffix") - .arg(&builder.version) - .arg("--doc-folder") - .arg(builder.doc_out(self.target)) - .arg("--test-folder") - .arg(builder.src.join("tests/rustdoc-js-std")); - for path in &builder.paths { - if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-js-std", builder) { - if !p.ends_with(".js") { - eprintln!("A non-js file was given: `{}`", path.display()); - panic!("Cannot run rustdoc-js-std tests"); - } - command.arg("--test-file").arg(path); - } - } - builder.ensure(crate::doc::Std::new( - builder.top_stage, - self.target, - builder, - DocumentationFormat::HTML, - )); - let _guard = builder.msg( - Kind::Test, - builder.top_stage, - "rustdoc-js-std", - builder.config.build, - self.target, - ); - builder.run(&mut command); - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustdocJSNotStd { - pub target: TargetSelection, - pub compiler: Compiler, -} - -impl Step for RustdocJSNotStd { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let default = run.builder.config.nodejs.is_some(); - run.suite_path("tests/rustdoc-js").default_condition(default) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RustdocJSNotStd { target: run.target, compiler }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: "js-doc-test", - suite: "rustdoc-js", - path: "tests/rustdoc-js", - compare_mode: None, - }); - } -} - -fn get_browser_ui_test_version_inner(npm: &Path, global: bool) -> Option { - let mut command = Command::new(&npm); - command.arg("list").arg("--parseable").arg("--long").arg("--depth=0"); - if global { - command.arg("--global"); - } - let lines = command - .output() - .map(|output| String::from_utf8_lossy(&output.stdout).into_owned()) - .unwrap_or(String::new()); - lines - .lines() - .find_map(|l| l.split(':').nth(1)?.strip_prefix("browser-ui-test@")) - .map(|v| v.to_owned()) -} - -fn get_browser_ui_test_version(npm: &Path) -> Option { - get_browser_ui_test_version_inner(npm, false) - .or_else(|| get_browser_ui_test_version_inner(npm, true)) -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustdocGUI { - pub target: TargetSelection, - pub compiler: Compiler, -} - -impl Step for RustdocGUI { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - let run = run.suite_path("tests/rustdoc-gui"); - run.lazy_default_condition(Box::new(move || { - builder.config.nodejs.is_some() - && builder - .config - .npm - .as_ref() - .map(|p| get_browser_ui_test_version(p).is_some()) - .unwrap_or(false) - })) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(RustdocGUI { target: run.target, compiler }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.target)); - - let mut cmd = builder.tool_cmd(Tool::RustdocGUITest); - - let out_dir = builder.test_out(self.target).join("rustdoc-gui"); - builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler)); - - if let Some(src) = builder.config.src.to_str() { - cmd.arg("--rust-src").arg(src); - } - - if let Some(out_dir) = out_dir.to_str() { - cmd.arg("--out-dir").arg(out_dir); - } - - if let Some(initial_cargo) = builder.config.initial_cargo.to_str() { - cmd.arg("--initial-cargo").arg(initial_cargo); - } - - cmd.arg("--jobs").arg(builder.jobs().to_string()); - - cmd.env("RUSTDOC", builder.rustdoc(self.compiler)) - .env("RUSTC", builder.rustc(self.compiler)); - - for path in &builder.paths { - if let Some(p) = util::is_valid_test_suite_arg(path, "tests/rustdoc-gui", builder) { - if !p.ends_with(".goml") { - eprintln!("A non-goml file was given: `{}`", path.display()); - panic!("Cannot run rustdoc-gui tests"); - } - if let Some(name) = path.file_name().and_then(|f| f.to_str()) { - cmd.arg("--goml-file").arg(name); - } - } - } - - for test_arg in builder.config.test_args() { - cmd.arg("--test-arg").arg(test_arg); - } - - if let Some(ref nodejs) = builder.config.nodejs { - cmd.arg("--nodejs").arg(nodejs); - } - - if let Some(ref npm) = builder.config.npm { - cmd.arg("--npm").arg(npm); - } - - let _time = util::timeit(&builder); - let _guard = builder.msg_sysroot_tool( - Kind::Test, - self.compiler.stage, - "rustdoc-gui", - self.compiler.host, - self.target, - ); - crate::render_tests::try_run_tests(builder, &mut cmd, true); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Tidy; - -impl Step for Tidy { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - /// Runs the `tidy` tool. - /// - /// This tool in `src/tools` checks up on various bits and pieces of style and - /// otherwise just implements a few lint-like checks that are specific to the - /// compiler itself. - /// - /// Once tidy passes, this step also runs `fmt --check` if tests are being run - /// for the `dev` or `nightly` channels. - fn run(self, builder: &Builder<'_>) { - let mut cmd = builder.tool_cmd(Tool::Tidy); - cmd.arg(&builder.src); - cmd.arg(&builder.initial_cargo); - cmd.arg(&builder.out); - // Tidy is heavily IO constrained. Still respect `-j`, but use a higher limit if `jobs` hasn't been configured. - let jobs = builder.config.jobs.unwrap_or_else(|| { - 8 * std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32 - }); - cmd.arg(jobs.to_string()); - if builder.is_verbose() { - cmd.arg("--verbose"); - } - if builder.config.cmd.bless() { - cmd.arg("--bless"); - } - if let Some(s) = builder.config.cmd.extra_checks() { - cmd.arg(format!("--extra-checks={s}")); - } - let mut args = std::env::args_os(); - if let Some(_) = args.find(|arg| arg == OsStr::new("--")) { - cmd.arg("--"); - cmd.args(args); - } - - if builder.config.channel == "dev" || builder.config.channel == "nightly" { - builder.info("fmt check"); - if builder.initial_rustfmt().is_none() { - let inferred_rustfmt_dir = builder.initial_rustc.parent().unwrap(); - eprintln!( - "\ -error: no `rustfmt` binary found in {PATH} -info: `rust.channel` is currently set to \"{CHAN}\" -help: if you are testing a beta branch, set `rust.channel` to \"beta\" in the `config.toml` file -help: to skip test's attempt to check tidiness, pass `--skip src/tools/tidy` to `x.py test`", - PATH = inferred_rustfmt_dir.display(), - CHAN = builder.config.channel, - ); - crate::exit!(1); - } - crate::format::format(&builder, !builder.config.cmd.bless(), &[]); - } - - builder.info("tidy check"); - builder.run_delaying_failure(&mut cmd); - - builder.ensure(ExpandYamlAnchors); - - builder.info("x.py completions check"); - let [bash, fish, powershell] = ["x.py.sh", "x.py.fish", "x.py.ps1"] - .map(|filename| builder.src.join("src/etc/completions").join(filename)); - if builder.config.cmd.bless() { - builder.ensure(crate::run::GenerateCompletions); - } else if crate::flags::get_completion(shells::Bash, &bash).is_some() - || crate::flags::get_completion(shells::Fish, &fish).is_some() - || crate::flags::get_completion(shells::PowerShell, &powershell).is_some() - { - eprintln!( - "x.py completions were changed; run `x.py run generate-completions` to update them" - ); - crate::exit!(1); - } - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/tidy") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Tidy); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ExpandYamlAnchors; - -impl Step for ExpandYamlAnchors { - type Output = (); - const ONLY_HOSTS: bool = true; - - /// Ensure the `generate-ci-config` tool was run locally. - /// - /// The tool in `src/tools` reads the CI definition in `src/ci/builders.yml` and generates the - /// appropriate configuration for all our CI providers. This step ensures the tool was called - /// by the user before committing CI changes. - fn run(self, builder: &Builder<'_>) { - // Note: `.github/` is not included in dist-src tarballs - if !builder.src.join(".github/workflows/ci.yml").exists() { - builder.info("Skipping YAML anchors check: GitHub Actions config not found"); - return; - } - builder.info("Ensuring the YAML anchors in the GitHub Actions config were expanded"); - builder.run_delaying_failure( - &mut builder.tool_cmd(Tool::ExpandYamlAnchors).arg("check").arg(&builder.src), - ); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/expand-yaml-anchors") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ExpandYamlAnchors); - } -} - -fn testdir(builder: &Builder<'_>, host: TargetSelection) -> PathBuf { - builder.out.join(host.triple).join("test") -} - -macro_rules! default_test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { - test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: false }); - }; -} - -macro_rules! default_test_with_compare_mode { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, - compare_mode: $compare_mode:expr }) => { - test_with_compare_mode!($name { - path: $path, - mode: $mode, - suite: $suite, - default: true, - host: false, - compare_mode: $compare_mode - }); - }; -} - -macro_rules! host_test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr }) => { - test!($name { path: $path, mode: $mode, suite: $suite, default: true, host: true }); - }; -} - -macro_rules! test { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, - host: $host:expr }) => { - test_definitions!($name { - path: $path, - mode: $mode, - suite: $suite, - default: $default, - host: $host, - compare_mode: None - }); - }; -} - -macro_rules! test_with_compare_mode { - ($name:ident { path: $path:expr, mode: $mode:expr, suite: $suite:expr, default: $default:expr, - host: $host:expr, compare_mode: $compare_mode:expr }) => { - test_definitions!($name { - path: $path, - mode: $mode, - suite: $suite, - default: $default, - host: $host, - compare_mode: Some($compare_mode) - }); - }; -} - -macro_rules! test_definitions { - ($name:ident { - path: $path:expr, - mode: $mode:expr, - suite: $suite:expr, - default: $default:expr, - host: $host:expr, - compare_mode: $compare_mode:expr - }) => { - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = $default; - const ONLY_HOSTS: bool = $host; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path($path) - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - - run.builder.ensure($name { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Compiletest { - compiler: self.compiler, - target: self.target, - mode: $mode, - suite: $suite, - path: $path, - compare_mode: $compare_mode, - }) - } - } - }; -} - -default_test!(Ui { path: "tests/ui", mode: "ui", suite: "ui" }); - -default_test!(RunPassValgrind { - path: "tests/run-pass-valgrind", - mode: "run-pass-valgrind", - suite: "run-pass-valgrind" -}); - -default_test!(Codegen { path: "tests/codegen", mode: "codegen", suite: "codegen" }); - -default_test!(CodegenUnits { - path: "tests/codegen-units", - mode: "codegen-units", - suite: "codegen-units" -}); - -default_test!(Incremental { path: "tests/incremental", mode: "incremental", suite: "incremental" }); - -default_test_with_compare_mode!(Debuginfo { - path: "tests/debuginfo", - mode: "debuginfo", - suite: "debuginfo", - compare_mode: "split-dwarf" -}); - -host_test!(UiFullDeps { path: "tests/ui-fulldeps", mode: "ui", suite: "ui-fulldeps" }); - -host_test!(Rustdoc { path: "tests/rustdoc", mode: "rustdoc", suite: "rustdoc" }); -host_test!(RustdocUi { path: "tests/rustdoc-ui", mode: "ui", suite: "rustdoc-ui" }); - -host_test!(RustdocJson { path: "tests/rustdoc-json", mode: "rustdoc-json", suite: "rustdoc-json" }); - -host_test!(Pretty { path: "tests/pretty", mode: "pretty", suite: "pretty" }); - -default_test!(RunMake { path: "tests/run-make", mode: "run-make", suite: "run-make" }); - -host_test!(RunMakeFullDeps { - path: "tests/run-make-fulldeps", - mode: "run-make", - suite: "run-make-fulldeps" -}); - -default_test!(Assembly { path: "tests/assembly", mode: "assembly", suite: "assembly" }); - -default_test!(CoverageMap { - path: "tests/coverage-map", - mode: "coverage-map", - suite: "coverage-map" -}); - -host_test!(RunCoverage { path: "tests/run-coverage", mode: "run-coverage", suite: "run-coverage" }); -host_test!(RunCoverageRustdoc { - path: "tests/run-coverage-rustdoc", - mode: "run-coverage", - suite: "run-coverage-rustdoc" -}); - -// For the mir-opt suite we do not use macros, as we need custom behavior when blessing. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct MirOpt { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for MirOpt { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = false; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.suite_path("tests/mir-opt") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = run.builder.compiler(run.builder.top_stage, run.build_triple()); - run.builder.ensure(MirOpt { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let run = |target| { - builder.ensure(Compiletest { - compiler: self.compiler, - target, - mode: "mir-opt", - suite: "mir-opt", - path: "tests/mir-opt", - compare_mode: None, - }) - }; - - // We use custom logic to bless the mir-opt suite: mir-opt tests have multiple variants - // (32bit vs 64bit, and panic=abort vs panic=unwind), and all of them needs to be blessed. - // When blessing, we try best-effort to also bless the other variants, to aid developers. - if builder.config.cmd.bless() { - let targets = MIR_OPT_BLESS_TARGET_MAPPING - .iter() - .filter(|(target_32bit, target_64bit)| { - *target_32bit == &*self.target.triple || *target_64bit == &*self.target.triple - }) - .next() - .map(|(target_32bit, target_64bit)| { - let target_32bit = TargetSelection::from_user(target_32bit); - let target_64bit = TargetSelection::from_user(target_64bit); - - // Running compiletest requires a C compiler to be available, but it might not - // have been detected by bootstrap if the target we're testing wasn't in the - // --target flags. - if !builder.cc.borrow().contains_key(&target_32bit) { - crate::cc_detect::find_target(builder, target_32bit); - } - if !builder.cc.borrow().contains_key(&target_64bit) { - crate::cc_detect::find_target(builder, target_64bit); - } - - vec![target_32bit, target_64bit] - }) - .unwrap_or_else(|| { - eprintln!( - "\ -Note that not all variants of mir-opt tests are going to be blessed, as no mapping between -a 32bit and a 64bit target was found for {target}. -You can add that mapping by changing MIR_OPT_BLESS_TARGET_MAPPING in src/bootstrap/test.rs", - target = self.target, - ); - vec![self.target] - }); - - for target in targets { - run(target); - - let panic_abort_target = builder.ensure(MirOptPanicAbortSyntheticTarget { - compiler: self.compiler, - base: target, - }); - run(panic_abort_target); - } - } else { - run(self.target); - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -struct Compiletest { - compiler: Compiler, - target: TargetSelection, - mode: &'static str, - suite: &'static str, - path: &'static str, - compare_mode: Option<&'static str>, -} - -impl Step for Compiletest { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Executes the `compiletest` tool to run a suite of tests. - /// - /// Compiles all tests with `compiler` for `target` with the specified - /// compiletest `mode` and `suite` arguments. For example `mode` can be - /// "run-pass" or `suite` can be something like `debuginfo`. - fn run(self, builder: &Builder<'_>) { - if builder.top_stage == 0 && env::var("COMPILETEST_FORCE_STAGE0").is_err() { - eprintln!("\ -error: `--stage 0` runs compiletest on the beta compiler, not your local changes, and will almost always cause tests to fail -help: to test the compiler, use `--stage 1` instead -help: to test the standard library, use `--stage 0 library/std` instead -note: if you're sure you want to do this, please open an issue as to why. In the meantime, you can override this with `COMPILETEST_FORCE_STAGE0=1`." - ); - crate::exit!(1); - } - - let mut compiler = self.compiler; - let target = self.target; - let mode = self.mode; - let suite = self.suite; - - // Path for test suite - let suite_path = self.path; - - // Skip codegen tests if they aren't enabled in configuration. - if !builder.config.codegen_tests && suite == "codegen" { - return; - } - - // Support stage 1 ui-fulldeps. This is somewhat complicated: ui-fulldeps tests for the most - // part test the *API* of the compiler, not how it compiles a given file. As a result, we - // can run them against the stage 1 sources as long as we build them with the stage 0 - // bootstrap compiler. - // NOTE: Only stage 1 is special cased because we need the rustc_private artifacts to match the - // running compiler in stage 2 when plugins run. - let stage_id = if suite == "ui-fulldeps" && compiler.stage == 1 { - compiler = builder.compiler(compiler.stage - 1, target); - format!("stage{}-{}", compiler.stage + 1, target) - } else { - format!("stage{}-{}", compiler.stage, target) - }; - - if suite.ends_with("fulldeps") { - builder.ensure(compile::Rustc::new(compiler, target)); - } - - if suite == "debuginfo" { - builder - .ensure(dist::DebuggerScripts { sysroot: builder.sysroot(compiler), host: target }); - } - - builder.ensure(compile::Std::new(compiler, target)); - // ensure that `libproc_macro` is available on the host. - builder.ensure(compile::Std::new(compiler, compiler.host)); - - // Also provide `rust_test_helpers` for the host. - builder.ensure(TestHelpers { target: compiler.host }); - - // As well as the target, except for plain wasm32, which can't build it - if !target.contains("wasm") || target.contains("emscripten") { - builder.ensure(TestHelpers { target }); - } - - builder.ensure(RemoteCopyLibs { compiler, target }); - - let mut cmd = builder.tool_cmd(Tool::Compiletest); - - // compiletest currently has... a lot of arguments, so let's just pass all - // of them! - - cmd.arg("--compile-lib-path").arg(builder.rustc_libdir(compiler)); - cmd.arg("--run-lib-path").arg(builder.sysroot_libdir(compiler, target)); - cmd.arg("--rustc-path").arg(builder.rustc(compiler)); - - let is_rustdoc = suite.ends_with("rustdoc-ui") || suite.ends_with("rustdoc-js"); - - // Avoid depending on rustdoc when we don't need it. - if mode == "rustdoc" - || mode == "run-make" - || (mode == "ui" && is_rustdoc) - || mode == "js-doc-test" - || mode == "rustdoc-json" - || suite == "run-coverage-rustdoc" - { - cmd.arg("--rustdoc-path").arg(builder.rustdoc(compiler)); - } - - if mode == "rustdoc-json" { - // Use the beta compiler for jsondocck - let json_compiler = compiler.with_stage(0); - cmd.arg("--jsondocck-path") - .arg(builder.ensure(tool::JsonDocCk { compiler: json_compiler, target })); - cmd.arg("--jsondoclint-path") - .arg(builder.ensure(tool::JsonDocLint { compiler: json_compiler, target })); - } - - if mode == "coverage-map" { - let coverage_dump = builder.ensure(tool::CoverageDump { - compiler: compiler.with_stage(0), - target: compiler.host, - }); - cmd.arg("--coverage-dump-path").arg(coverage_dump); - } - - if mode == "run-make" || mode == "run-coverage" { - let rust_demangler = builder - .ensure(tool::RustDemangler { - compiler, - target: compiler.host, - extra_features: Vec::new(), - }) - .expect("in-tree tool"); - cmd.arg("--rust-demangler-path").arg(rust_demangler); - } - - cmd.arg("--src-base").arg(builder.src.join("tests").join(suite)); - cmd.arg("--build-base").arg(testdir(builder, compiler.host).join(suite)); - - // When top stage is 0, that means that we're testing an externally provided compiler. - // In that case we need to use its specific sysroot for tests to pass. - let sysroot = if builder.top_stage == 0 { - builder.initial_sysroot.clone() - } else { - builder.sysroot(compiler).to_path_buf() - }; - cmd.arg("--sysroot-base").arg(sysroot); - cmd.arg("--stage-id").arg(stage_id); - cmd.arg("--suite").arg(suite); - cmd.arg("--mode").arg(mode); - cmd.arg("--target").arg(target.rustc_target_arg()); - cmd.arg("--host").arg(&*compiler.host.triple); - cmd.arg("--llvm-filecheck").arg(builder.llvm_filecheck(builder.config.build)); - - if builder.config.cmd.bless() { - cmd.arg("--bless"); - } - - if builder.config.cmd.force_rerun() { - cmd.arg("--force-rerun"); - } - - let compare_mode = - builder.config.cmd.compare_mode().or_else(|| { - if builder.config.test_compare_mode { self.compare_mode } else { None } - }); - - if let Some(ref pass) = builder.config.cmd.pass() { - cmd.arg("--pass"); - cmd.arg(pass); - } - - if let Some(ref run) = builder.config.cmd.run() { - cmd.arg("--run"); - cmd.arg(run); - } - - if let Some(ref nodejs) = builder.config.nodejs { - cmd.arg("--nodejs").arg(nodejs); - } else if mode == "js-doc-test" { - panic!("need nodejs to run js-doc-test suite"); - } - if let Some(ref npm) = builder.config.npm { - cmd.arg("--npm").arg(npm); - } - if builder.config.rust_optimize_tests { - cmd.arg("--optimize-tests"); - } - if builder.config.cmd.only_modified() { - cmd.arg("--only-modified"); - } - - let mut flags = if is_rustdoc { Vec::new() } else { vec!["-Crpath".to_string()] }; - flags.push(format!("-Cdebuginfo={}", builder.config.rust_debuginfo_level_tests)); - flags.extend(builder.config.cmd.rustc_args().iter().map(|s| s.to_string())); - - if let Some(linker) = builder.linker(target) { - cmd.arg("--target-linker").arg(linker); - } - if let Some(linker) = builder.linker(compiler.host) { - cmd.arg("--host-linker").arg(linker); - } - - let mut hostflags = flags.clone(); - hostflags.push(format!("-Lnative={}", builder.test_helpers_out(compiler.host).display())); - hostflags.extend(builder.lld_flags(compiler.host)); - for flag in hostflags { - cmd.arg("--host-rustcflags").arg(flag); - } - - let mut targetflags = flags; - targetflags.push(format!("-Lnative={}", builder.test_helpers_out(target).display())); - targetflags.extend(builder.lld_flags(target)); - for flag in targetflags { - cmd.arg("--target-rustcflags").arg(flag); - } - - cmd.arg("--python").arg(builder.python()); - - if let Some(ref gdb) = builder.config.gdb { - cmd.arg("--gdb").arg(gdb); - } - - let run = |cmd: &mut Command| { - cmd.output().map(|output| { - String::from_utf8_lossy(&output.stdout) - .lines() - .next() - .unwrap_or_else(|| panic!("{:?} failed {:?}", cmd, output)) - .to_string() - }) - }; - let lldb_exe = "lldb"; - let lldb_version = Command::new(lldb_exe) - .arg("--version") - .output() - .map(|output| String::from_utf8_lossy(&output.stdout).to_string()) - .ok(); - if let Some(ref vers) = lldb_version { - cmd.arg("--lldb-version").arg(vers); - let lldb_python_dir = run(Command::new(lldb_exe).arg("-P")).ok(); - if let Some(ref dir) = lldb_python_dir { - cmd.arg("--lldb-python-dir").arg(dir); - } - } - - if util::forcing_clang_based_tests() { - let clang_exe = builder.llvm_out(target).join("bin").join("clang"); - cmd.arg("--run-clang-based-tests-with").arg(clang_exe); - } - - for exclude in &builder.config.skip { - cmd.arg("--skip"); - cmd.arg(&exclude); - } - - // Get paths from cmd args - let paths = match &builder.config.cmd { - Subcommand::Test { .. } => &builder.config.paths[..], - _ => &[], - }; - - // Get test-args by striping suite path - let mut test_args: Vec<&str> = paths - .iter() - .filter_map(|p| util::is_valid_test_suite_arg(p, suite_path, builder)) - .collect(); - - test_args.append(&mut builder.config.test_args()); - - // On Windows, replace forward slashes in test-args by backslashes - // so the correct filters are passed to libtest - if cfg!(windows) { - let test_args_win: Vec = - test_args.iter().map(|s| s.replace("/", "\\")).collect(); - cmd.args(&test_args_win); - } else { - cmd.args(&test_args); - } - - if builder.is_verbose() { - cmd.arg("--verbose"); - } - - cmd.arg("--json"); - - let mut llvm_components_passed = false; - let mut copts_passed = false; - if builder.config.llvm_enabled() { - let llvm::LlvmResult { llvm_config, .. } = - builder.ensure(llvm::Llvm { target: builder.config.build }); - if !builder.config.dry_run() { - let llvm_version = output(Command::new(&llvm_config).arg("--version")); - let llvm_components = output(Command::new(&llvm_config).arg("--components")); - // Remove trailing newline from llvm-config output. - cmd.arg("--llvm-version") - .arg(llvm_version.trim()) - .arg("--llvm-components") - .arg(llvm_components.trim()); - llvm_components_passed = true; - } - if !builder.is_rust_llvm(target) { - cmd.arg("--system-llvm"); - } - - // Tests that use compiler libraries may inherit the `-lLLVM` link - // requirement, but the `-L` library path is not propagated across - // separate compilations. We can add LLVM's library path to the - // platform-specific environment variable as a workaround. - if !builder.config.dry_run() && suite.ends_with("fulldeps") { - let llvm_libdir = output(Command::new(&llvm_config).arg("--libdir")); - add_link_lib_path(vec![llvm_libdir.trim().into()], &mut cmd); - } - - if !builder.config.dry_run() - && (matches!(suite, "run-make" | "run-make-fulldeps") || mode == "run-coverage") - { - // The llvm/bin directory contains many useful cross-platform - // tools. Pass the path to run-make tests so they can use them. - // (The run-coverage tests also need these tools to process - // coverage reports.) - let llvm_bin_path = llvm_config - .parent() - .expect("Expected llvm-config to be contained in directory"); - assert!(llvm_bin_path.is_dir()); - cmd.arg("--llvm-bin-dir").arg(llvm_bin_path); - } - - if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { - // If LLD is available, add it to the PATH - if builder.config.lld_enabled { - let lld_install_root = - builder.ensure(llvm::Lld { target: builder.config.build }); - - let lld_bin_path = lld_install_root.join("bin"); - - let old_path = env::var_os("PATH").unwrap_or_default(); - let new_path = env::join_paths( - std::iter::once(lld_bin_path).chain(env::split_paths(&old_path)), - ) - .expect("Could not add LLD bin path to PATH"); - cmd.env("PATH", new_path); - } - } - } - - // Only pass correct values for these flags for the `run-make` suite as it - // requires that a C++ compiler was configured which isn't always the case. - if !builder.config.dry_run() && matches!(suite, "run-make" | "run-make-fulldeps") { - cmd.arg("--cc") - .arg(builder.cc(target)) - .arg("--cxx") - .arg(builder.cxx(target).unwrap()) - .arg("--cflags") - .arg(builder.cflags(target, GitRepo::Rustc, CLang::C).join(" ")) - .arg("--cxxflags") - .arg(builder.cflags(target, GitRepo::Rustc, CLang::Cxx).join(" ")); - copts_passed = true; - if let Some(ar) = builder.ar(target) { - cmd.arg("--ar").arg(ar); - } - } - - if !llvm_components_passed { - cmd.arg("--llvm-components").arg(""); - } - if !copts_passed { - cmd.arg("--cc") - .arg("") - .arg("--cxx") - .arg("") - .arg("--cflags") - .arg("") - .arg("--cxxflags") - .arg(""); - } - - if builder.remote_tested(target) { - cmd.arg("--remote-test-client").arg(builder.tool_exe(Tool::RemoteTestClient)); - } - - // Running a C compiler on MSVC requires a few env vars to be set, to be - // sure to set them here. - // - // Note that if we encounter `PATH` we make sure to append to our own `PATH` - // rather than stomp over it. - if !builder.config.dry_run() && target.contains("msvc") { - for &(ref k, ref v) in builder.cc.borrow()[&target].env() { - if k != "PATH" { - cmd.env(k, v); - } - } - } - cmd.env("RUSTC_BOOTSTRAP", "1"); - // Override the rustc version used in symbol hashes to reduce the amount of normalization - // needed when diffing test output. - cmd.env("RUSTC_FORCE_RUSTC_VERSION", "compiletest"); - cmd.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); - builder.add_rust_test_threads(&mut cmd); - - if builder.config.sanitizers_enabled(target) { - cmd.env("RUSTC_SANITIZER_SUPPORT", "1"); - } - - if builder.config.profiler_enabled(target) { - cmd.env("RUSTC_PROFILER_SUPPORT", "1"); - } - - cmd.env("RUST_TEST_TMPDIR", builder.tempdir()); - - cmd.arg("--adb-path").arg("adb"); - cmd.arg("--adb-test-dir").arg(ADB_TEST_DIR); - if target.contains("android") && !builder.config.dry_run() { - // Assume that cc for this target comes from the android sysroot - cmd.arg("--android-cross-path") - .arg(builder.cc(target).parent().unwrap().parent().unwrap()); - } else { - cmd.arg("--android-cross-path").arg(""); - } - - if builder.config.cmd.rustfix_coverage() { - cmd.arg("--rustfix-coverage"); - } - - cmd.env("BOOTSTRAP_CARGO", &builder.initial_cargo); - - cmd.arg("--channel").arg(&builder.config.channel); - - if !builder.config.omit_git_hash { - cmd.arg("--git-hash"); - } - - builder.ci_env.force_coloring_in_ci(&mut cmd); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::Compiletest { - suite: suite.into(), - mode: mode.into(), - compare_mode: None, - target: self.target.triple.to_string(), - host: self.compiler.host.triple.to_string(), - stage: self.compiler.stage, - }, - builder, - ); - - let _group = builder.msg( - Kind::Test, - compiler.stage, - &format!("compiletest suite={suite} mode={mode}"), - compiler.host, - target, - ); - crate::render_tests::try_run_tests(builder, &mut cmd, false); - - if let Some(compare_mode) = compare_mode { - cmd.arg("--compare-mode").arg(compare_mode); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::Compiletest { - suite: suite.into(), - mode: mode.into(), - compare_mode: Some(compare_mode.into()), - target: self.target.triple.to_string(), - host: self.compiler.host.triple.to_string(), - stage: self.compiler.stage, - }, - builder, - ); - - builder.info(&format!( - "Check compiletest suite={} mode={} compare_mode={} ({} -> {})", - suite, mode, compare_mode, &compiler.host, target - )); - let _time = util::timeit(&builder); - crate::render_tests::try_run_tests(builder, &mut cmd, false); - } - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -struct BookTest { - compiler: Compiler, - path: PathBuf, - name: &'static str, - is_ext_doc: bool, -} - -impl Step for BookTest { - type Output = (); - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Runs the documentation tests for a book in `src/doc`. - /// - /// This uses the `rustdoc` that sits next to `compiler`. - fn run(self, builder: &Builder<'_>) { - // External docs are different from local because: - // - Some books need pre-processing by mdbook before being tested. - // - They need to save their state to toolstate. - // - They are only tested on the "checktools" builders. - // - // The local docs are tested by default, and we don't want to pay the - // cost of building mdbook, so they use `rustdoc --test` directly. - // Also, the unstable book is special because SUMMARY.md is generated, - // so it is easier to just run `rustdoc` on its files. - if self.is_ext_doc { - self.run_ext_doc(builder); - } else { - self.run_local_doc(builder); - } - } -} - -impl BookTest { - /// This runs the equivalent of `mdbook test` (via the rustbook wrapper) - /// which in turn runs `rustdoc --test` on each file in the book. - fn run_ext_doc(self, builder: &Builder<'_>) { - let compiler = self.compiler; - - builder.ensure(compile::Std::new(compiler, compiler.host)); - - // mdbook just executes a binary named "rustdoc", so we need to update - // PATH so that it points to our rustdoc. - let mut rustdoc_path = builder.rustdoc(compiler); - rustdoc_path.pop(); - let old_path = env::var_os("PATH").unwrap_or_default(); - let new_path = env::join_paths(iter::once(rustdoc_path).chain(env::split_paths(&old_path))) - .expect("could not add rustdoc to PATH"); - - let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - let path = builder.src.join(&self.path); - // Books often have feature-gated example text. - rustbook_cmd.env("RUSTC_BOOTSTRAP", "1"); - rustbook_cmd.env("PATH", new_path).arg("test").arg(path); - builder.add_rust_test_threads(&mut rustbook_cmd); - let _guard = builder.msg( - Kind::Test, - compiler.stage, - format_args!("mdbook {}", self.path.display()), - compiler.host, - compiler.host, - ); - let _time = util::timeit(&builder); - let toolstate = if builder.run_delaying_failure(&mut rustbook_cmd) { - ToolState::TestPass - } else { - ToolState::TestFail - }; - builder.save_toolstate(self.name, toolstate); - } - - /// This runs `rustdoc --test` on all `.md` files in the path. - fn run_local_doc(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let host = self.compiler.host; - - builder.ensure(compile::Std::new(compiler, host)); - - let _guard = - builder.msg(Kind::Test, compiler.stage, &format!("book {}", self.name), host, host); - - // Do a breadth-first traversal of the `src/doc` directory and just run - // tests for all files that end in `*.md` - let mut stack = vec![builder.src.join(self.path)]; - let _time = util::timeit(&builder); - let mut files = Vec::new(); - while let Some(p) = stack.pop() { - if p.is_dir() { - stack.extend(t!(p.read_dir()).map(|p| t!(p).path())); - continue; - } - - if p.extension().and_then(|s| s.to_str()) != Some("md") { - continue; - } - - files.push(p); - } - - files.sort(); - - for file in files { - markdown_test(builder, compiler, &file); - } - } -} - -macro_rules! test_book { - ($($name:ident, $path:expr, $book_name:expr, default=$default:expr;)+) => { - $( - #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] - pub struct $name { - compiler: Compiler, - } - - impl Step for $name { - type Output = (); - const DEFAULT: bool = $default; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path($path) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.target), - }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(BookTest { - compiler: self.compiler, - path: PathBuf::from($path), - name: $book_name, - is_ext_doc: !$default, - }); - } - } - )+ - } -} - -test_book!( - Nomicon, "src/doc/nomicon", "nomicon", default=false; - Reference, "src/doc/reference", "reference", default=false; - RustdocBook, "src/doc/rustdoc", "rustdoc", default=true; - RustcBook, "src/doc/rustc", "rustc", default=true; - RustByExample, "src/doc/rust-by-example", "rust-by-example", default=false; - EmbeddedBook, "src/doc/embedded-book", "embedded-book", default=false; - TheBook, "src/doc/book", "book", default=false; - UnstableBook, "src/doc/unstable-book", "unstable-book", default=true; - EditionGuide, "src/doc/edition-guide", "edition-guide", default=false; -); - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ErrorIndex { - compiler: Compiler, -} - -impl Step for ErrorIndex { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/error_index_generator") - } - - fn make_run(run: RunConfig<'_>) { - // error_index_generator depends on librustdoc. Use the compiler that - // is normally used to build rustdoc for other tests (like compiletest - // tests in tests/rustdoc) so that it shares the same artifacts. - let compiler = run.builder.compiler(run.builder.top_stage, run.builder.config.build); - run.builder.ensure(ErrorIndex { compiler }); - } - - /// Runs the error index generator tool to execute the tests located in the error - /// index. - /// - /// The `error_index_generator` tool lives in `src/tools` and is used to - /// generate a markdown file from the error indexes of the code base which is - /// then passed to `rustdoc --test`. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - - let dir = testdir(builder, compiler.host); - t!(fs::create_dir_all(&dir)); - let output = dir.join("error-index.md"); - - let mut tool = tool::ErrorIndex::command(builder); - tool.arg("markdown").arg(&output); - - let guard = - builder.msg(Kind::Test, compiler.stage, "error-index", compiler.host, compiler.host); - let _time = util::timeit(&builder); - builder.run_quiet(&mut tool); - drop(guard); - // The tests themselves need to link to std, so make sure it is - // available. - builder.ensure(compile::Std::new(compiler, compiler.host)); - markdown_test(builder, compiler, &output); - } -} - -fn markdown_test(builder: &Builder<'_>, compiler: Compiler, markdown: &Path) -> bool { - if let Ok(contents) = fs::read_to_string(markdown) { - if !contents.contains("```") { - return true; - } - } - - builder.verbose(&format!("doc tests for: {}", markdown.display())); - let mut cmd = builder.rustdoc_cmd(compiler); - builder.add_rust_test_threads(&mut cmd); - // allow for unstable options such as new editions - cmd.arg("-Z"); - cmd.arg("unstable-options"); - cmd.arg("--test"); - cmd.arg(markdown); - cmd.env("RUSTC_BOOTSTRAP", "1"); - - let test_args = builder.config.test_args().join(" "); - cmd.arg("--test-args").arg(test_args); - - if builder.config.verbose_tests { - builder.run_delaying_failure(&mut cmd) - } else { - builder.run_quiet_delaying_failure(&mut cmd) - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustcGuide; - -impl Step for RustcGuide { - type Output = (); - const DEFAULT: bool = false; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/doc/rustc-dev-guide") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustcGuide); - } - - fn run(self, builder: &Builder<'_>) { - let relative_path = Path::new("src").join("doc").join("rustc-dev-guide"); - builder.update_submodule(&relative_path); - - let src = builder.src.join(relative_path); - let mut rustbook_cmd = builder.tool_cmd(Tool::Rustbook); - let toolstate = if builder.run_delaying_failure(rustbook_cmd.arg("linkcheck").arg(&src)) { - ToolState::TestPass - } else { - ToolState::TestFail - }; - builder.save_toolstate("rustc-dev-guide", toolstate); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CrateLibrustc { - compiler: Compiler, - target: TargetSelection, - crates: Vec>, -} - -impl Step for CrateLibrustc { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("rustc-main") - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = builder.compiler_for(builder.top_stage, host, host); - let crates = run - .paths - .iter() - .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) - .collect(); - - builder.ensure(CrateLibrustc { compiler, target: run.target, crates }); - } - - fn run(self, builder: &Builder<'_>) { - builder.ensure(Crate { - compiler: self.compiler, - target: self.target, - mode: Mode::Rustc, - crates: self.crates, - }); - } -} - -/// Given a `cargo test` subcommand, add the appropriate flags and run it. -/// -/// Returns whether the test succeeded. -fn run_cargo_test<'a>( - cargo: impl Into, - libtest_args: &[&str], - crates: &[Interned], - primary_crate: &str, - description: impl Into>, - compiler: Compiler, - target: TargetSelection, - builder: &Builder<'_>, -) -> bool { - let mut cargo = - prepare_cargo_test(cargo, libtest_args, crates, primary_crate, compiler, target, builder); - let _time = util::timeit(&builder); - let _group = description.into().and_then(|what| { - builder.msg_sysroot_tool(Kind::Test, compiler.stage, what, compiler.host, target) - }); - - #[cfg(feature = "build-metrics")] - builder.metrics.begin_test_suite( - build_helper::metrics::TestSuiteMetadata::CargoPackage { - crates: crates.iter().map(|c| c.to_string()).collect(), - target: target.triple.to_string(), - host: compiler.host.triple.to_string(), - stage: compiler.stage, - }, - builder, - ); - add_flags_and_try_run_tests(builder, &mut cargo) -} - -/// Given a `cargo test` subcommand, pass it the appropriate test flags given a `builder`. -fn prepare_cargo_test( - cargo: impl Into, - libtest_args: &[&str], - crates: &[Interned], - primary_crate: &str, - compiler: Compiler, - target: TargetSelection, - builder: &Builder<'_>, -) -> Command { - let mut cargo = cargo.into(); - - // Propegate `--bless` if it has not already been set/unset - // Any tools that want to use this should bless if `RUSTC_BLESS` is set to - // anything other than `0`. - if builder.config.cmd.bless() && !cargo.get_envs().any(|v| v.0 == "RUSTC_BLESS") { - cargo.env("RUSTC_BLESS", "Gesundheit"); - } - - // Pass in some standard flags then iterate over the graph we've discovered - // in `cargo metadata` with the maps above and figure out what `-p` - // arguments need to get passed. - if builder.kind == Kind::Test && !builder.fail_fast { - cargo.arg("--no-fail-fast"); - } - match builder.doc_tests { - DocTests::Only => { - cargo.arg("--doc"); - } - DocTests::No => { - let krate = &builder - .crates - .get(&INTERNER.intern_str(primary_crate)) - .unwrap_or_else(|| panic!("missing crate {primary_crate}")); - if krate.has_lib { - cargo.arg("--lib"); - } - cargo.args(&["--bins", "--examples", "--tests", "--benches"]); - } - DocTests::Yes => {} - } - - for &krate in crates { - cargo.arg("-p").arg(krate); - } - - cargo.arg("--").args(&builder.config.test_args()).args(libtest_args); - if !builder.config.verbose_tests { - cargo.arg("--quiet"); - } - - // The tests are going to run with the *target* libraries, so we need to - // ensure that those libraries show up in the LD_LIBRARY_PATH equivalent. - // - // Note that to run the compiler we need to run with the *host* libraries, - // but our wrapper scripts arrange for that to be the case anyway. - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(&*builder.sysroot_libdir(compiler, target))); - cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - if target.contains("emscripten") { - cargo.env( - format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), - builder.config.nodejs.as_ref().expect("nodejs not configured"), - ); - } else if target.starts_with("wasm32") { - let node = builder.config.nodejs.as_ref().expect("nodejs not configured"); - let runner = format!("{} {}/src/etc/wasm32-shim.js", node.display(), builder.src.display()); - cargo.env(format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), &runner); - } else if builder.remote_tested(target) { - cargo.env( - format!("CARGO_TARGET_{}_RUNNER", envify(&target.triple)), - format!("{} run 0", builder.tool_exe(Tool::RemoteTestClient).display()), - ); - } - - cargo -} - -#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord, Hash)] -pub struct Crate { - pub compiler: Compiler, - pub target: TargetSelection, - pub mode: Mode, - pub crates: Vec>, -} - -impl Step for Crate { - type Output = (); - const DEFAULT: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.crate_or_deps("sysroot") - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = builder.compiler_for(builder.top_stage, host, host); - let crates = run - .paths - .iter() - .map(|p| builder.crate_paths[&p.assert_single_path().path].clone()) - .collect(); - - builder.ensure(Crate { compiler, target: run.target, mode: Mode::Std, crates }); - } - - /// Runs all unit tests plus documentation tests for a given crate defined - /// by a `Cargo.toml` (single manifest) - /// - /// This is what runs tests for crates like the standard library, compiler, etc. - /// It essentially is the driver for running `cargo test`. - /// - /// Currently this runs all tests for a DAG by passing a bunch of `-p foo` - /// arguments, and those arguments are discovered from `cargo metadata`. - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - let mode = self.mode; - - // See [field@compile::Std::force_recompile]. - builder.ensure(compile::Std::force_recompile(compiler, target)); - builder.ensure(RemoteCopyLibs { compiler, target }); - - // If we're not doing a full bootstrap but we're testing a stage2 - // version of libstd, then what we're actually testing is the libstd - // produced in stage1. Reflect that here by updating the compiler that - // we're working with automatically. - let compiler = builder.compiler_for(compiler.stage, compiler.host, target); - - let mut cargo = - builder.cargo(compiler, mode, SourceType::InTree, target, builder.kind.as_str()); - match mode { - Mode::Std => { - compile::std_cargo(builder, target, compiler.stage, &mut cargo); - // `std_cargo` actually does the wrong thing: it passes `--sysroot build/host/stage2`, - // but we want to use the force-recompile std we just built in `build/host/stage2-test-sysroot`. - // Override it. - if builder.download_rustc() && compiler.stage > 0 { - let sysroot = builder - .out - .join(compiler.host.triple) - .join(format!("stage{}-test-sysroot", compiler.stage)); - cargo.env("RUSTC_SYSROOT", sysroot); - } - } - Mode::Rustc => { - compile::rustc_cargo(builder, &mut cargo, target, compiler.stage); - } - _ => panic!("can only test libraries"), - }; - - run_cargo_test( - cargo, - &[], - &self.crates, - &self.crates[0], - &*crate_description(&self.crates), - compiler, - target, - builder, - ); - } -} - -/// Rustdoc is special in various ways, which is why this step is different from `Crate`. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrateRustdoc { - host: TargetSelection, -} - -impl Step for CrateRustdoc { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["src/librustdoc", "src/tools/rustdoc"]) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - - builder.ensure(CrateRustdoc { host: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let target = self.host; - - let compiler = if builder.download_rustc() { - builder.compiler(builder.top_stage, target) - } else { - // Use the previous stage compiler to reuse the artifacts that are - // created when running compiletest for tests/rustdoc. If this used - // `compiler`, then it would cause rustdoc to be built *again*, which - // isn't really necessary. - builder.compiler_for(builder.top_stage, target, target) - }; - // NOTE: normally `ensure(Rustc)` automatically runs `ensure(Std)` for us. However, when - // using `download-rustc`, the rustc_private artifacts may be in a *different sysroot* from - // the target rustdoc (`ci-rustc-sysroot` vs `stage2`). In that case, we need to ensure this - // explicitly to make sure it ends up in the stage2 sysroot. - builder.ensure(compile::Std::new(compiler, target)); - builder.ensure(compile::Rustc::new(compiler, target)); - - let mut cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - builder.kind.as_str(), - "src/tools/rustdoc", - SourceType::InTree, - &[], - ); - if self.host.contains("musl") { - cargo.arg("'-Ctarget-feature=-crt-static'"); - } - - // This is needed for running doctests on librustdoc. This is a bit of - // an unfortunate interaction with how bootstrap works and how cargo - // sets up the dylib path, and the fact that the doctest (in - // html/markdown.rs) links to rustc-private libs. For stage1, the - // compiler host dylibs (in stage1/lib) are not the same as the target - // dylibs (in stage1/lib/rustlib/...). This is different from a normal - // rust distribution where they are the same. - // - // On the cargo side, normal tests use `target_process` which handles - // setting up the dylib for a *target* (stage1/lib/rustlib/... in this - // case). However, for doctests it uses `rustdoc_process` which only - // sets up the dylib path for the *host* (stage1/lib), which is the - // wrong directory. - // - // Recall that we special-cased `compiler_for(top_stage)` above, so we always use stage1. - // - // It should be considered to just stop running doctests on - // librustdoc. There is only one test, and it doesn't look too - // important. There might be other ways to avoid this, but it seems - // pretty convoluted. - // - // See also https://github.com/rust-lang/rust/issues/13983 where the - // host vs target dylibs for rustdoc are consistently tricky to deal - // with. - // - // Note that this set the host libdir for `download_rustc`, which uses a normal rust distribution. - let libdir = if builder.download_rustc() { - builder.rustc_libdir(compiler) - } else { - builder.sysroot_libdir(compiler, target).to_path_buf() - }; - let mut dylib_path = dylib_path(); - dylib_path.insert(0, PathBuf::from(&*libdir)); - cargo.env(dylib_path_var(), env::join_paths(&dylib_path).unwrap()); - - run_cargo_test( - cargo, - &[], - &[INTERNER.intern_str("rustdoc:0.0.0")], - "rustdoc", - "rustdoc", - compiler, - target, - builder, - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct CrateRustdocJsonTypes { - host: TargetSelection, -} - -impl Step for CrateRustdocJsonTypes { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/rustdoc-json-types") - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - - builder.ensure(CrateRustdocJsonTypes { host: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let target = self.host; - - // Use the previous stage compiler to reuse the artifacts that are - // created when running compiletest for tests/rustdoc. If this used - // `compiler`, then it would cause rustdoc to be built *again*, which - // isn't really necessary. - let compiler = builder.compiler_for(builder.top_stage, target, target); - builder.ensure(compile::Rustc::new(compiler, target)); - - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolRustc, - target, - builder.kind.as_str(), - "src/rustdoc-json-types", - SourceType::InTree, - &[], - ); - - // FIXME: this looks very wrong, libtest doesn't accept `-C` arguments and the quotes are fishy. - let libtest_args = if self.host.contains("musl") { - ["'-Ctarget-feature=-crt-static'"].as_slice() - } else { - &[] - }; - - run_cargo_test( - cargo, - libtest_args, - &[INTERNER.intern_str("rustdoc-json-types")], - "rustdoc-json-types", - "rustdoc-json-types", - compiler, - target, - builder, - ); - } -} - -/// Some test suites are run inside emulators or on remote devices, and most -/// of our test binaries are linked dynamically which means we need to ship -/// the standard library and such to the emulator ahead of time. This step -/// represents this and is a dependency of all test suites. -/// -/// Most of the time this is a no-op. For some steps such as shipping data to -/// QEMU we have to build our own tools so we've got conditional dependencies -/// on those programs as well. Note that the remote test client is built for -/// the build target (us) and the server is built for the target. -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RemoteCopyLibs { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for RemoteCopyLibs { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - if !builder.remote_tested(target) { - return; - } - - builder.ensure(compile::Std::new(compiler, target)); - - builder.info(&format!("REMOTE copy libs to emulator ({target})")); - - let server = builder.ensure(tool::RemoteTestServer { compiler, target }); - - // Spawn the emulator and wait for it to come online - let tool = builder.tool_exe(Tool::RemoteTestClient); - let mut cmd = Command::new(&tool); - cmd.arg("spawn-emulator").arg(target.triple).arg(&server).arg(builder.tempdir()); - if let Some(rootfs) = builder.qemu_rootfs(target) { - cmd.arg(rootfs); - } - builder.run(&mut cmd); - - // Push all our dylibs to the emulator - for f in t!(builder.sysroot_libdir(compiler, target).read_dir()) { - let f = t!(f); - let name = f.file_name().into_string().unwrap(); - if util::is_dylib(&name) { - builder.run(Command::new(&tool).arg("push").arg(f.path())); - } - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Distcheck; - -impl Step for Distcheck { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("distcheck") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Distcheck); - } - - /// Runs "distcheck", a 'make check' from a tarball - fn run(self, builder: &Builder<'_>) { - builder.info("Distcheck"); - let dir = builder.tempdir().join("distcheck"); - let _ = fs::remove_dir_all(&dir); - t!(fs::create_dir_all(&dir)); - - // Guarantee that these are built before we begin running. - builder.ensure(dist::PlainSourceTarball); - builder.ensure(dist::Src); - - let mut cmd = Command::new("tar"); - cmd.arg("-xf") - .arg(builder.ensure(dist::PlainSourceTarball).tarball()) - .arg("--strip-components=1") - .current_dir(&dir); - builder.run(&mut cmd); - builder.run( - Command::new("./configure") - .args(&builder.config.configure_args) - .arg("--enable-vendor") - .current_dir(&dir), - ); - builder.run( - Command::new(util::make(&builder.config.build.triple)).arg("check").current_dir(&dir), - ); - - // Now make sure that rust-src has all of libstd's dependencies - builder.info("Distcheck rust-src"); - let dir = builder.tempdir().join("distcheck-src"); - let _ = fs::remove_dir_all(&dir); - t!(fs::create_dir_all(&dir)); - - let mut cmd = Command::new("tar"); - cmd.arg("-xf") - .arg(builder.ensure(dist::Src).tarball()) - .arg("--strip-components=1") - .current_dir(&dir); - builder.run(&mut cmd); - - let toml = dir.join("rust-src/lib/rustlib/src/rust/library/std/Cargo.toml"); - builder.run( - Command::new(&builder.initial_cargo) - // Will read the libstd Cargo.toml - // which uses the unstable `public-dependency` feature. - .env("RUSTC_BOOTSTRAP", "1") - .arg("generate-lockfile") - .arg("--manifest-path") - .arg(&toml) - .current_dir(&dir), - ); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct Bootstrap; - -impl Step for Bootstrap { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - /// Tests the build system itself. - fn run(self, builder: &Builder<'_>) { - let host = builder.config.build; - let compiler = builder.compiler(0, host); - let _guard = builder.msg(Kind::Test, 0, "bootstrap", host, host); - - let mut check_bootstrap = Command::new(&builder.python()); - check_bootstrap - .args(["-m", "unittest", "bootstrap_test.py"]) - .env("BUILD_DIR", &builder.out) - .env("BUILD_PLATFORM", &builder.build.build.triple) - .current_dir(builder.src.join("src/bootstrap/")); - // NOTE: we intentionally don't pass test_args here because the args for unittest and cargo test are mutually incompatible. - // Use `python -m unittest` manually if you want to pass arguments. - builder.run_delaying_failure(&mut check_bootstrap); - - let mut cmd = Command::new(&builder.initial_cargo); - cmd.arg("test") - .current_dir(builder.src.join("src/bootstrap")) - .env("RUSTFLAGS", "-Cdebuginfo=2") - .env("CARGO_TARGET_DIR", builder.out.join("bootstrap")) - .env("RUSTC_BOOTSTRAP", "1") - .env("RUSTDOC", builder.rustdoc(compiler)) - .env("RUSTC", &builder.initial_rustc); - if let Some(flags) = option_env!("RUSTFLAGS") { - // Use the same rustc flags for testing as for "normal" compilation, - // so that Cargo doesn’t recompile the entire dependency graph every time: - // https://github.com/rust-lang/rust/issues/49215 - cmd.env("RUSTFLAGS", flags); - } - // rustbuild tests are racy on directory creation so just run them one at a time. - // Since there's not many this shouldn't be a problem. - run_cargo_test(cmd, &["--test-threads=1"], &[], "bootstrap", None, compiler, host, builder); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/bootstrap") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Bootstrap); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct TierCheck { - pub compiler: Compiler, -} - -impl Step for TierCheck { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/tier-check") - } - - fn make_run(run: RunConfig<'_>) { - let compiler = - run.builder.compiler_for(run.builder.top_stage, run.builder.build.build, run.target); - run.builder.ensure(TierCheck { compiler }); - } - - /// Tests the Platform Support page in the rustc book. - fn run(self, builder: &Builder<'_>) { - builder.ensure(compile::Std::new(self.compiler, self.compiler.host)); - let mut cargo = tool::prepare_tool_cargo( - builder, - self.compiler, - Mode::ToolStd, - self.compiler.host, - "run", - "src/tools/tier-check", - SourceType::InTree, - &[], - ); - cargo.arg(builder.src.join("src/doc/rustc/src/platform-support.md")); - cargo.arg(&builder.rustc(self.compiler)); - if builder.is_verbose() { - cargo.arg("--verbose"); - } - - let _guard = builder.msg( - Kind::Test, - self.compiler.stage, - "platform support check", - self.compiler.host, - self.compiler.host, - ); - builder.run_delaying_failure(&mut cargo.into()); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct LintDocs { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for LintDocs { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/lint-docs") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(LintDocs { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - /// Tests that the lint examples in the rustc book generate the correct - /// lints and have the expected format. - fn run(self, builder: &Builder<'_>) { - builder.ensure(crate::doc::RustcBook { - compiler: self.compiler, - target: self.target, - validate: true, - }); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct RustInstaller; - -impl Step for RustInstaller { - type Output = (); - const ONLY_HOSTS: bool = true; - const DEFAULT: bool = true; - - /// Ensure the version placeholder replacement tool builds - fn run(self, builder: &Builder<'_>) { - let bootstrap_host = builder.config.build; - let compiler = builder.compiler(0, bootstrap_host); - let cargo = tool::prepare_tool_cargo( - builder, - compiler, - Mode::ToolBootstrap, - bootstrap_host, - "test", - "src/tools/rust-installer", - SourceType::InTree, - &[], - ); - - let _guard = builder.msg( - Kind::Test, - compiler.stage, - "rust-installer", - bootstrap_host, - bootstrap_host, - ); - run_cargo_test(cargo, &[], &[], "installer", None, compiler, bootstrap_host, builder); - - // We currently don't support running the test.sh script outside linux(?) environments. - // Eventually this should likely migrate to #[test]s in rust-installer proper rather than a - // set of scripts, which will likely allow dropping this if. - if bootstrap_host != "x86_64-unknown-linux-gnu" { - return; - } - - let mut cmd = - std::process::Command::new(builder.src.join("src/tools/rust-installer/test.sh")); - let tmpdir = testdir(builder, compiler.host).join("rust-installer"); - let _ = std::fs::remove_dir_all(&tmpdir); - let _ = std::fs::create_dir_all(&tmpdir); - cmd.current_dir(&tmpdir); - cmd.env("CARGO_TARGET_DIR", tmpdir.join("cargo-target")); - cmd.env("CARGO", &builder.initial_cargo); - cmd.env("RUSTC", &builder.initial_rustc); - cmd.env("TMP_DIR", &tmpdir); - builder.run_delaying_failure(&mut cmd); - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rust-installer") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Self); - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct TestHelpers { - pub target: TargetSelection, -} - -impl Step for TestHelpers { - type Output = (); - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("tests/auxiliary/rust_test_helpers.c") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(TestHelpers { target: run.target }) - } - - /// Compiles the `rust_test_helpers.c` library which we used in various - /// `run-pass` tests for ABI testing. - fn run(self, builder: &Builder<'_>) { - if builder.config.dry_run() { - return; - } - // The x86_64-fortanix-unknown-sgx target doesn't have a working C - // toolchain. However, some x86_64 ELF objects can be linked - // without issues. Use this hack to compile the test helpers. - let target = if self.target == "x86_64-fortanix-unknown-sgx" { - TargetSelection::from_user("x86_64-unknown-linux-gnu") - } else { - self.target - }; - let dst = builder.test_helpers_out(target); - let src = builder.src.join("tests/auxiliary/rust_test_helpers.c"); - if up_to_date(&src, &dst.join("librust_test_helpers.a")) { - return; - } - - let _guard = builder.msg_unstaged(Kind::Build, "test helpers", target); - t!(fs::create_dir_all(&dst)); - let mut cfg = cc::Build::new(); - // FIXME: Workaround for https://github.com/emscripten-core/emscripten/issues/9013 - if target.contains("emscripten") { - cfg.pic(false); - } - - // We may have found various cross-compilers a little differently due to our - // extra configuration, so inform cc of these compilers. Note, though, that - // on MSVC we still need cc's detection of env vars (ugh). - if !target.contains("msvc") { - if let Some(ar) = builder.ar(target) { - cfg.archiver(ar); - } - cfg.compiler(builder.cc(target)); - } - cfg.cargo_metadata(false) - .out_dir(&dst) - .target(&target.triple) - .host(&builder.config.build.triple) - .opt_level(0) - .warnings(false) - .debug(false) - .file(builder.src.join("tests/auxiliary/rust_test_helpers.c")) - .compile("rust_test_helpers"); - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -pub struct CodegenCranelift { - compiler: Compiler, - target: TargetSelection, -} - -impl Step for CodegenCranelift { - type Output = (); - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.paths(&["compiler/rustc_codegen_cranelift"]) - } - - fn make_run(run: RunConfig<'_>) { - let builder = run.builder; - let host = run.build_triple(); - let compiler = run.builder.compiler_for(run.builder.top_stage, host, host); - - if builder.doc_tests == DocTests::Only { - return; - } - - let triple = run.target.triple; - let target_supported = if triple.contains("linux") { - triple.contains("x86_64") || triple.contains("aarch64") || triple.contains("s390x") - } else if triple.contains("darwin") || triple.contains("windows") { - triple.contains("x86_64") - } else { - false - }; - if !target_supported { - builder.info("target not supported by rustc_codegen_cranelift. skipping"); - return; - } - - if builder.remote_tested(run.target) { - builder.info("remote testing is not supported by rustc_codegen_cranelift. skipping"); - return; - } - - if !builder.config.rust_codegen_backends.contains(&INTERNER.intern_str("cranelift")) { - builder.info("cranelift not in rust.codegen-backends. skipping"); - return; - } - - builder.ensure(CodegenCranelift { compiler, target: run.target }); - } - - fn run(self, builder: &Builder<'_>) { - let compiler = self.compiler; - let target = self.target; - - builder.ensure(compile::Std::new(compiler, target)); - - // If we're not doing a full bootstrap but we're testing a stage2 - // version of libstd, then what we're actually testing is the libstd - // produced in stage1. Reflect that here by updating the compiler that - // we're working with automatically. - let compiler = builder.compiler_for(compiler.stage, compiler.host, target); - - let build_cargo = || { - let mut cargo = builder.cargo( - compiler, - Mode::Codegen, // Must be codegen to ensure dlopen on compiled dylibs works - SourceType::InTree, - target, - "run", - ); - cargo.current_dir(&builder.src.join("compiler/rustc_codegen_cranelift")); - cargo - .arg("--manifest-path") - .arg(builder.src.join("compiler/rustc_codegen_cranelift/build_system/Cargo.toml")); - compile::rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - - // Avoid incremental cache issues when changing rustc - cargo.env("CARGO_BUILD_INCREMENTAL", "false"); - - cargo - }; - - builder.info(&format!( - "{} cranelift stage{} ({} -> {})", - Kind::Test.description(), - compiler.stage, - &compiler.host, - target - )); - let _time = util::timeit(&builder); - - // FIXME handle vendoring for source tarballs before removing the --skip-test below - let download_dir = builder.out.join("cg_clif_download"); - - /* - let mut prepare_cargo = build_cargo(); - prepare_cargo.arg("--").arg("prepare").arg("--download-dir").arg(&download_dir); - #[allow(deprecated)] - builder.config.try_run(&mut prepare_cargo.into()).unwrap(); - */ - - let mut cargo = build_cargo(); - cargo - .arg("--") - .arg("test") - .arg("--download-dir") - .arg(&download_dir) - .arg("--out-dir") - .arg(builder.stage_out(compiler, Mode::ToolRustc).join("cg_clif")) - .arg("--no-unstable-features") - .arg("--use-backend") - .arg("cranelift") - // Avoid having to vendor the standard library dependencies - .arg("--sysroot") - .arg("llvm") - // These tests depend on crates that are not yet vendored - // FIXME remove once vendoring is handled - .arg("--skip-test") - .arg("testsuite.extended_sysroot"); - cargo.args(builder.config.test_args()); - - #[allow(deprecated)] - builder.config.try_run(&mut cargo.into()).unwrap(); - } -} diff --git a/src/bootstrap/tool.rs b/src/bootstrap/tool.rs deleted file mode 100644 index f094dd9d7..000000000 --- a/src/bootstrap/tool.rs +++ /dev/null @@ -1,848 +0,0 @@ -use std::env; -use std::fs; -use std::path::PathBuf; -use std::process::Command; - -use crate::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step}; -use crate::channel::GitInfo; -use crate::compile; -use crate::config::TargetSelection; -use crate::toolstate::ToolState; -use crate::util::{add_dylib_path, exe, t}; -use crate::Compiler; -use crate::Mode; -use crate::{gha, Kind}; - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub enum SourceType { - InTree, - Submodule, -} - -#[derive(Debug, Clone, Hash, PartialEq, Eq)] -struct ToolBuild { - compiler: Compiler, - target: TargetSelection, - tool: &'static str, - path: &'static str, - mode: Mode, - is_optional_tool: bool, - source_type: SourceType, - extra_features: Vec, - /// Nightly-only features that are allowed (comma-separated list). - allow_features: &'static str, -} - -impl Builder<'_> { - #[track_caller] - fn msg_tool( - &self, - mode: Mode, - tool: &str, - build_stage: u32, - host: &TargetSelection, - target: &TargetSelection, - ) -> Option { - match mode { - // depends on compiler stage, different to host compiler - Mode::ToolRustc => self.msg_sysroot_tool( - Kind::Build, - build_stage, - format_args!("tool {tool}"), - *host, - *target, - ), - // doesn't depend on compiler, same as host compiler - _ => self.msg(Kind::Build, build_stage, format_args!("tool {tool}"), *host, *target), - } - } -} - -impl Step for ToolBuild { - type Output = Option; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - /// Builds a tool in `src/tools` - /// - /// This will build the specified tool with the specified `host` compiler in - /// `stage` into the normal cargo output directory. - fn run(self, builder: &Builder<'_>) -> Option { - let compiler = self.compiler; - let target = self.target; - let mut tool = self.tool; - let path = self.path; - let is_optional_tool = self.is_optional_tool; - - match self.mode { - Mode::ToolRustc => { - builder.ensure(compile::Std::new(compiler, compiler.host)); - builder.ensure(compile::Rustc::new(compiler, target)); - } - Mode::ToolStd => builder.ensure(compile::Std::new(compiler, target)), - Mode::ToolBootstrap => {} // uses downloaded stage0 compiler libs - _ => panic!("unexpected Mode for tool build"), - } - - let mut cargo = prepare_tool_cargo( - builder, - compiler, - self.mode, - target, - "build", - path, - self.source_type, - &self.extra_features, - ); - if !self.allow_features.is_empty() { - cargo.allow_features(self.allow_features); - } - let _guard = builder.msg_tool( - self.mode, - self.tool, - self.compiler.stage, - &self.compiler.host, - &self.target, - ); - - let mut cargo = Command::from(cargo); - #[allow(deprecated)] // we check this in `is_optional_tool` in a second - let is_expected = builder.config.try_run(&mut cargo).is_ok(); - - builder.save_toolstate( - tool, - if is_expected { ToolState::TestFail } else { ToolState::BuildFail }, - ); - - if !is_expected { - if !is_optional_tool { - crate::exit!(1); - } else { - None - } - } else { - // HACK(#82501): on Windows, the tools directory gets added to PATH when running tests, and - // compiletest confuses HTML tidy with the in-tree tidy. Name the in-tree tidy something - // different so the problem doesn't come up. - if tool == "tidy" { - tool = "rust-tidy"; - } - let cargo_out = builder.cargo_out(compiler, self.mode, target).join(exe(tool, target)); - let bin = builder.tools_dir(compiler).join(exe(tool, target)); - builder.copy(&cargo_out, &bin); - Some(bin) - } - } -} - -pub fn prepare_tool_cargo( - builder: &Builder<'_>, - compiler: Compiler, - mode: Mode, - target: TargetSelection, - command: &'static str, - path: &str, - source_type: SourceType, - extra_features: &[String], -) -> CargoCommand { - let mut cargo = builder.cargo(compiler, mode, source_type, target, command); - let dir = builder.src.join(path); - cargo.arg("--manifest-path").arg(dir.join("Cargo.toml")); - - let mut features = extra_features.to_vec(); - if builder.build.config.cargo_native_static { - if path.ends_with("cargo") - || path.ends_with("rls") - || path.ends_with("clippy") - || path.ends_with("miri") - || path.ends_with("rustfmt") - { - cargo.env("LIBZ_SYS_STATIC", "1"); - } - if path.ends_with("cargo") { - features.push("all-static".to_string()); - } - } - - // clippy tests need to know about the stage sysroot. Set them consistently while building to - // avoid rebuilding when running tests. - cargo.env("SYSROOT", builder.sysroot(compiler)); - - // if tools are using lzma we want to force the build script to build its - // own copy - cargo.env("LZMA_API_STATIC", "1"); - - // CFG_RELEASE is needed by rustfmt (and possibly other tools) which - // import rustc-ap-rustc_attr which requires this to be set for the - // `#[cfg(version(...))]` attribute. - cargo.env("CFG_RELEASE", builder.rust_release()); - cargo.env("CFG_RELEASE_CHANNEL", &builder.config.channel); - cargo.env("CFG_VERSION", builder.rust_version()); - cargo.env("CFG_RELEASE_NUM", &builder.version); - cargo.env("DOC_RUST_LANG_ORG_CHANNEL", builder.doc_rust_lang_org_channel()); - if let Some(ref ver_date) = builder.rust_info().commit_date() { - cargo.env("CFG_VER_DATE", ver_date); - } - if let Some(ref ver_hash) = builder.rust_info().sha() { - cargo.env("CFG_VER_HASH", ver_hash); - } - - let info = GitInfo::new(builder.config.omit_git_hash, &dir); - if let Some(sha) = info.sha() { - cargo.env("CFG_COMMIT_HASH", sha); - } - if let Some(sha_short) = info.sha_short() { - cargo.env("CFG_SHORT_COMMIT_HASH", sha_short); - } - if let Some(date) = info.commit_date() { - cargo.env("CFG_COMMIT_DATE", date); - } - if !features.is_empty() { - cargo.arg("--features").arg(&features.join(", ")); - } - cargo -} - -macro_rules! bootstrap_tool { - ($( - $name:ident, $path:expr, $tool_name:expr - $(,is_external_tool = $external:expr)* - $(,is_unstable_tool = $unstable:expr)* - $(,allow_features = $allow_features:expr)? - ; - )+) => { - #[derive(Copy, PartialEq, Eq, Clone)] - pub enum Tool { - $( - $name, - )+ - } - - impl<'a> Builder<'a> { - pub fn tool_exe(&self, tool: Tool) -> PathBuf { - match tool { - $(Tool::$name => - self.ensure($name { - compiler: self.compiler(0, self.config.build), - target: self.config.build, - }), - )+ - } - } - } - - $( - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - } - - impl Step for $name { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path($path) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - // snapshot compiler - compiler: run.builder.compiler(0, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: $tool_name, - mode: if false $(|| $unstable)* { - // use in-tree libraries for unstable features - Mode::ToolStd - } else { - Mode::ToolBootstrap - }, - path: $path, - is_optional_tool: false, - source_type: if false $(|| $external)* { - SourceType::Submodule - } else { - SourceType::InTree - }, - extra_features: vec![], - allow_features: concat!($($allow_features)*), - }).expect("expected to build -- essential tool") - } - } - )+ - } -} - -bootstrap_tool!( - Rustbook, "src/tools/rustbook", "rustbook"; - UnstableBookGen, "src/tools/unstable-book-gen", "unstable-book-gen"; - Tidy, "src/tools/tidy", "tidy"; - Linkchecker, "src/tools/linkchecker", "linkchecker"; - CargoTest, "src/tools/cargotest", "cargotest"; - Compiletest, "src/tools/compiletest", "compiletest", is_unstable_tool = true, allow_features = "test"; - BuildManifest, "src/tools/build-manifest", "build-manifest"; - RemoteTestClient, "src/tools/remote-test-client", "remote-test-client"; - RustInstaller, "src/tools/rust-installer", "rust-installer"; - RustdocTheme, "src/tools/rustdoc-themes", "rustdoc-themes"; - ExpandYamlAnchors, "src/tools/expand-yaml-anchors", "expand-yaml-anchors"; - LintDocs, "src/tools/lint-docs", "lint-docs"; - JsonDocCk, "src/tools/jsondocck", "jsondocck"; - JsonDocLint, "src/tools/jsondoclint", "jsondoclint"; - HtmlChecker, "src/tools/html-checker", "html-checker"; - BumpStage0, "src/tools/bump-stage0", "bump-stage0"; - ReplaceVersionPlaceholder, "src/tools/replace-version-placeholder", "replace-version-placeholder"; - CollectLicenseMetadata, "src/tools/collect-license-metadata", "collect-license-metadata"; - GenerateCopyright, "src/tools/generate-copyright", "generate-copyright"; - SuggestTests, "src/tools/suggest-tests", "suggest-tests"; - GenerateWindowsSys, "src/tools/generate-windows-sys", "generate-windows-sys"; - RustdocGUITest, "src/tools/rustdoc-gui-test", "rustdoc-gui-test", is_unstable_tool = true, allow_features = "test"; - OptimizedDist, "src/tools/opt-dist", "opt-dist"; - CoverageDump, "src/tools/coverage-dump", "coverage-dump"; -); - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct ErrorIndex { - pub compiler: Compiler, -} - -impl ErrorIndex { - pub fn command(builder: &Builder<'_>) -> Command { - // Error-index-generator links with the rustdoc library, so we need to add `rustc_lib_paths` - // for rustc_private and libLLVM.so, and `sysroot_lib` for libstd, etc. - let host = builder.config.build; - let compiler = builder.compiler_for(builder.top_stage, host, host); - let mut cmd = Command::new(builder.ensure(ErrorIndex { compiler })); - let mut dylib_paths = builder.rustc_lib_paths(compiler); - dylib_paths.push(PathBuf::from(&builder.sysroot_libdir(compiler, compiler.host))); - add_dylib_path(dylib_paths, &mut cmd); - cmd - } -} - -impl Step for ErrorIndex { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/error_index_generator") - } - - fn make_run(run: RunConfig<'_>) { - // Compile the error-index in the same stage as rustdoc to avoid - // recompiling rustdoc twice if we can. - // - // NOTE: This `make_run` isn't used in normal situations, only if you - // manually build the tool with `x.py build - // src/tools/error-index-generator` which almost nobody does. - // Normally, `x.py test` or `x.py doc` will use the - // `ErrorIndex::command` function instead. - let compiler = - run.builder.compiler(run.builder.top_stage.saturating_sub(1), run.builder.config.build); - run.builder.ensure(ErrorIndex { compiler }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.compiler.host, - tool: "error_index_generator", - mode: Mode::ToolRustc, - path: "src/tools/error_index_generator", - is_optional_tool: false, - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool") - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RemoteTestServer { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RemoteTestServer { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/remote-test-server") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RemoteTestServer { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "remote-test-server", - mode: Mode::ToolStd, - path: "src/tools/remote-test-server", - is_optional_tool: false, - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool") - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq, Ord, PartialOrd)] -pub struct Rustdoc { - /// This should only ever be 0 or 2. - /// We sometimes want to reference the "bootstrap" rustdoc, which is why this option is here. - pub compiler: Compiler, -} - -impl Step for Rustdoc { - type Output = PathBuf; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.path("src/tools/rustdoc").path("src/librustdoc") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Rustdoc { - // Note: this is somewhat unique in that we actually want a *target* - // compiler here, because rustdoc *is* a compiler. We won't be using - // this as the compiler to build with, but rather this is "what - // compiler are we producing"? - compiler: run.builder.compiler(run.builder.top_stage, run.target), - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - let target_compiler = self.compiler; - if target_compiler.stage == 0 { - if !target_compiler.is_snapshot(builder) { - panic!("rustdoc in stage 0 must be snapshot rustdoc"); - } - return builder.initial_rustc.with_file_name(exe("rustdoc", target_compiler.host)); - } - let target = target_compiler.host; - // Similar to `compile::Assemble`, build with the previous stage's compiler. Otherwise - // we'd have stageN/bin/rustc and stageN/bin/rustdoc be effectively different stage - // compilers, which isn't what we want. Rustdoc should be linked in the same way as the - // rustc compiler it's paired with, so it must be built with the previous stage compiler. - let build_compiler = builder.compiler(target_compiler.stage - 1, builder.config.build); - - // When using `download-rustc` and a stage0 build_compiler, copying rustc doesn't actually - // build stage0 libstd (because the libstd in sysroot has the wrong ABI). Explicitly build - // it. - builder.ensure(compile::Std::new(build_compiler, target_compiler.host)); - builder.ensure(compile::Rustc::new(build_compiler, target_compiler.host)); - // NOTE: this implies that `download-rustc` is pretty useless when compiling with the stage0 - // compiler, since you do just as much work. - if !builder.config.dry_run() && builder.download_rustc() && build_compiler.stage == 0 { - println!( - "warning: `download-rustc` does nothing when building stage1 tools; consider using `--stage 2` instead" - ); - } - - // The presence of `target_compiler` ensures that the necessary libraries (codegen backends, - // compiler libraries, ...) are built. Rustdoc does not require the presence of any - // libraries within sysroot_libdir (i.e., rustlib), though doctests may want it (since - // they'll be linked to those libraries). As such, don't explicitly `ensure` any additional - // libraries here. The intuition here is that If we've built a compiler, we should be able - // to build rustdoc. - // - let mut features = Vec::new(); - if builder.config.jemalloc { - features.push("jemalloc".to_string()); - } - - let mut cargo = prepare_tool_cargo( - builder, - build_compiler, - Mode::ToolRustc, - target, - "build", - "src/tools/rustdoc", - SourceType::InTree, - features.as_slice(), - ); - - if builder.config.rustc_parallel { - cargo.rustflag("--cfg=parallel_compiler"); - } - - let _guard = builder.msg_tool( - Mode::ToolRustc, - "rustdoc", - build_compiler.stage, - &self.compiler.host, - &target, - ); - builder.run(&mut cargo.into()); - - // Cargo adds a number of paths to the dylib search path on windows, which results in - // the wrong rustdoc being executed. To avoid the conflicting rustdocs, we name the "tool" - // rustdoc a different name. - let tool_rustdoc = builder - .cargo_out(build_compiler, Mode::ToolRustc, target) - .join(exe("rustdoc_tool_binary", target_compiler.host)); - - // don't create a stage0-sysroot/bin directory. - if target_compiler.stage > 0 { - let sysroot = builder.sysroot(target_compiler); - let bindir = sysroot.join("bin"); - t!(fs::create_dir_all(&bindir)); - let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host)); - let _ = fs::remove_file(&bin_rustdoc); - builder.copy(&tool_rustdoc, &bin_rustdoc); - bin_rustdoc - } else { - tool_rustdoc - } - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct Cargo { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for Cargo { - type Output = PathBuf; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/cargo").default_condition( - builder.config.extended - && builder.config.tools.as_ref().map_or( - true, - // If `tools` is set, search list for this tool. - |tools| tools.iter().any(|tool| tool == "cargo"), - ), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(Cargo { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - let cargo_bin_path = builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "cargo", - mode: Mode::ToolRustc, - path: "src/tools/cargo", - is_optional_tool: false, - source_type: SourceType::Submodule, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool"); - cargo_bin_path - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct LldWrapper { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for LldWrapper { - type Output = PathBuf; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.never() - } - - fn run(self, builder: &Builder<'_>) -> PathBuf { - let src_exe = builder - .ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "lld-wrapper", - mode: Mode::ToolStd, - path: "src/tools/lld-wrapper", - is_optional_tool: false, - source_type: SourceType::InTree, - extra_features: Vec::new(), - allow_features: "", - }) - .expect("expected to build -- essential tool"); - - src_exe - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustAnalyzer { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl RustAnalyzer { - pub const ALLOW_FEATURES: &'static str = - "proc_macro_internals,proc_macro_diagnostic,proc_macro_span,proc_macro_span_shrink"; -} - -impl Step for RustAnalyzer { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path("src/tools/rust-analyzer").default_condition( - builder.config.extended - && builder - .config - .tools - .as_ref() - .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzer { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer", - mode: Mode::ToolStd, - path: "src/tools/rust-analyzer", - extra_features: vec!["rust-analyzer/in-rust-tree".to_owned()], - is_optional_tool: false, - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - }) - } -} - -#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] -pub struct RustAnalyzerProcMacroSrv { - pub compiler: Compiler, - pub target: TargetSelection, -} - -impl Step for RustAnalyzerProcMacroSrv { - type Output = Option; - const DEFAULT: bool = true; - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - // Allow building `rust-analyzer-proc-macro-srv` both as part of the `rust-analyzer` and as a stand-alone tool. - run.path("src/tools/rust-analyzer") - .path("src/tools/rust-analyzer/crates/proc-macro-srv-cli") - .default_condition(builder.config.tools.as_ref().map_or(true, |tools| { - tools - .iter() - .any(|tool| tool == "rust-analyzer" || tool == "rust-analyzer-proc-macro-srv") - })) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(RustAnalyzerProcMacroSrv { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - }); - } - - fn run(self, builder: &Builder<'_>) -> Option { - let path = builder.ensure(ToolBuild { - compiler: self.compiler, - target: self.target, - tool: "rust-analyzer-proc-macro-srv", - mode: Mode::ToolStd, - path: "src/tools/rust-analyzer/crates/proc-macro-srv-cli", - extra_features: vec!["sysroot-abi".to_owned()], - is_optional_tool: false, - source_type: SourceType::InTree, - allow_features: RustAnalyzer::ALLOW_FEATURES, - })?; - - // Copy `rust-analyzer-proc-macro-srv` to `/libexec/` - // so that r-a can use it. - let libexec_path = builder.sysroot(self.compiler).join("libexec"); - t!(fs::create_dir_all(&libexec_path)); - builder.copy(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); - - Some(path) - } -} - -macro_rules! tool_extended { - (($sel:ident, $builder:ident), - $($name:ident, - $path:expr, - $tool_name:expr, - stable = $stable:expr - $(,tool_std = $tool_std:literal)? - $(,allow_features = $allow_features:expr)? - $(,add_bins_to_sysroot = $add_bins_to_sysroot:expr)? - ;)+) => { - $( - #[derive(Debug, Clone, Hash, PartialEq, Eq)] - pub struct $name { - pub compiler: Compiler, - pub target: TargetSelection, - pub extra_features: Vec, - } - - impl Step for $name { - type Output = Option; - const DEFAULT: bool = true; // Overwritten below - const ONLY_HOSTS: bool = true; - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - let builder = run.builder; - run.path($path).default_condition( - builder.config.extended - && builder.config.tools.as_ref().map_or( - // By default, on nightly/dev enable all tools, else only - // build stable tools. - $stable || builder.build.unstable_features(), - // If `tools` is set, search list for this tool. - |tools| { - tools.iter().any(|tool| match tool.as_ref() { - "clippy" => $tool_name == "clippy-driver", - x => $tool_name == x, - }) - }), - ) - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure($name { - compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build), - target: run.target, - extra_features: Vec::new(), - }); - } - - #[allow(unused_mut)] - fn run(mut $sel, $builder: &Builder<'_>) -> Option { - let tool = $builder.ensure(ToolBuild { - compiler: $sel.compiler, - target: $sel.target, - tool: $tool_name, - mode: if false $(|| $tool_std)? { Mode::ToolStd } else { Mode::ToolRustc }, - path: $path, - extra_features: $sel.extra_features, - is_optional_tool: true, - source_type: SourceType::InTree, - allow_features: concat!($($allow_features)*), - })?; - - if (false $(|| !$add_bins_to_sysroot.is_empty())?) && $sel.compiler.stage > 0 { - let bindir = $builder.sysroot($sel.compiler).join("bin"); - t!(fs::create_dir_all(&bindir)); - - #[allow(unused_variables)] - let tools_out = $builder - .cargo_out($sel.compiler, Mode::ToolRustc, $sel.target); - - $(for add_bin in $add_bins_to_sysroot { - let bin_source = tools_out.join(exe(add_bin, $sel.target)); - let bin_destination = bindir.join(exe(add_bin, $sel.compiler.host)); - $builder.copy(&bin_source, &bin_destination); - })? - - let tool = bindir.join(exe($tool_name, $sel.compiler.host)); - Some(tool) - } else { - Some(tool) - } - } - } - )+ - } -} - -// Note: tools need to be also added to `Builder::get_step_descriptions` in `builder.rs` -// to make `./x.py build ` work. -// Note: Most submodule updates for tools are handled by bootstrap.py, since they're needed just to -// invoke Cargo to build bootstrap. See the comment there for more details. -tool_extended!((self, builder), - Cargofmt, "src/tools/rustfmt", "cargo-fmt", stable=true; - CargoClippy, "src/tools/clippy", "cargo-clippy", stable=true; - Clippy, "src/tools/clippy", "clippy-driver", stable=true, add_bins_to_sysroot = ["clippy-driver", "cargo-clippy"]; - Miri, "src/tools/miri", "miri", stable=false, add_bins_to_sysroot = ["miri"]; - CargoMiri, "src/tools/miri/cargo-miri", "cargo-miri", stable=true, add_bins_to_sysroot = ["cargo-miri"]; - // FIXME: tool_std is not quite right, we shouldn't allow nightly features. - // But `builder.cargo` doesn't know how to handle ToolBootstrap in stages other than 0, - // and this is close enough for now. - Rls, "src/tools/rls", "rls", stable=true, tool_std=true; - RustDemangler, "src/tools/rust-demangler", "rust-demangler", stable=false, tool_std=true; - Rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, add_bins_to_sysroot = ["rustfmt", "cargo-fmt"]; -); - -impl<'a> Builder<'a> { - /// Gets a `Command` which is ready to run `tool` in `stage` built for - /// `host`. - pub fn tool_cmd(&self, tool: Tool) -> Command { - let mut cmd = Command::new(self.tool_exe(tool)); - let compiler = self.compiler(0, self.config.build); - let host = &compiler.host; - // Prepares the `cmd` provided to be able to run the `compiler` provided. - // - // Notably this munges the dynamic library lookup path to point to the - // right location to run `compiler`. - let mut lib_paths: Vec = vec![ - self.build.rustc_snapshot_libdir(), - self.cargo_out(compiler, Mode::ToolBootstrap, *host).join("deps"), - ]; - - // On MSVC a tool may invoke a C compiler (e.g., compiletest in run-make - // mode) and that C compiler may need some extra PATH modification. Do - // so here. - if compiler.host.contains("msvc") { - let curpaths = env::var_os("PATH").unwrap_or_default(); - let curpaths = env::split_paths(&curpaths).collect::>(); - for &(ref k, ref v) in self.cc.borrow()[&compiler.host].env() { - if k != "PATH" { - continue; - } - for path in env::split_paths(v) { - if !curpaths.contains(&path) { - lib_paths.push(path); - } - } - } - } - - add_dylib_path(lib_paths, &mut cmd); - - // Provide a RUSTC for this command to use. - cmd.env("RUSTC", &self.initial_rustc); - - cmd - } -} diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs deleted file mode 100644 index 308023537..000000000 --- a/src/bootstrap/toolstate.rs +++ /dev/null @@ -1,478 +0,0 @@ -use crate::builder::{Builder, RunConfig, ShouldRun, Step}; -use crate::util::t; -use serde_derive::{Deserialize, Serialize}; -use std::collections::HashMap; -use std::env; -use std::fmt; -use std::fs; -use std::io::{Seek, SeekFrom}; -use std::path::{Path, PathBuf}; -use std::process::Command; -use std::time; - -// Each cycle is 42 days long (6 weeks); the last week is 35..=42 then. -const BETA_WEEK_START: u64 = 35; - -#[cfg(target_os = "linux")] -const OS: Option<&str> = Some("linux"); - -#[cfg(windows)] -const OS: Option<&str> = Some("windows"); - -#[cfg(all(not(target_os = "linux"), not(windows)))] -const OS: Option<&str> = None; - -type ToolstateData = HashMap, ToolState>; - -#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, PartialOrd)] -#[serde(rename_all = "kebab-case")] -/// Whether a tool can be compiled, tested or neither -pub enum ToolState { - /// The tool compiles successfully, but the test suite fails - TestFail = 1, - /// The tool compiles successfully and its test suite passes - TestPass = 2, - /// The tool can't even be compiled - BuildFail = 0, -} - -impl fmt::Display for ToolState { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!( - f, - "{}", - match self { - ToolState::TestFail => "test-fail", - ToolState::TestPass => "test-pass", - ToolState::BuildFail => "build-fail", - } - ) - } -} - -/// Number of days after the last promotion of beta. -/// Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens. -/// The Wednesday after this has value 0. -/// We track this value to prevent regressing tools in the last week of the 6-week cycle. -fn days_since_beta_promotion() -> u64 { - let since_epoch = t!(time::SystemTime::UNIX_EPOCH.elapsed()); - (since_epoch.as_secs() / 86400 - 20) % 42 -} - -// These tools must test-pass on the beta/stable channels. -// -// On the nightly channel, their build step must be attempted, but they may not -// be able to build successfully. -static STABLE_TOOLS: &[(&str, &str)] = &[ - ("book", "src/doc/book"), - ("nomicon", "src/doc/nomicon"), - ("reference", "src/doc/reference"), - ("rust-by-example", "src/doc/rust-by-example"), - ("edition-guide", "src/doc/edition-guide"), -]; - -// These tools are permitted to not build on the beta/stable channels. -// -// We do require that we checked whether they build or not on the tools builder, -// though, as otherwise we will be unable to file an issue if they start -// failing. -static NIGHTLY_TOOLS: &[(&str, &str)] = &[ - ("embedded-book", "src/doc/embedded-book"), - // ("rustc-dev-guide", "src/doc/rustc-dev-guide"), -]; - -fn print_error(tool: &str, submodule: &str) { - eprintln!(); - eprintln!("We detected that this PR updated '{tool}', but its tests failed."); - eprintln!(); - eprintln!("If you do intend to update '{tool}', please check the error messages above and"); - eprintln!("commit another update."); - eprintln!(); - eprintln!("If you do NOT intend to update '{tool}', please ensure you did not accidentally"); - eprintln!("change the submodule at '{submodule}'. You may ask your reviewer for the"); - eprintln!("proper steps."); - crate::exit!(3); -} - -fn check_changed_files(toolstates: &HashMap, ToolState>) { - // Changed files - let output = std::process::Command::new("git") - .arg("diff") - .arg("--name-status") - .arg("HEAD") - .arg("HEAD^") - .output(); - let output = match output { - Ok(o) => o, - Err(e) => { - eprintln!("Failed to get changed files: {e:?}"); - crate::exit!(1); - } - }; - - let output = t!(String::from_utf8(output.stdout)); - - for (tool, submodule) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { - let changed = output.lines().any(|l| l.starts_with('M') && l.ends_with(submodule)); - eprintln!("Verifying status of {tool}..."); - if !changed { - continue; - } - - eprintln!("This PR updated '{submodule}', verifying if status is 'test-pass'..."); - if toolstates[*tool] != ToolState::TestPass { - print_error(tool, submodule); - } - } -} - -#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)] -pub struct ToolStateCheck; - -impl Step for ToolStateCheck { - type Output = (); - - /// Checks tool state status. - /// - /// This is intended to be used in the `checktools.sh` script. To use - /// this, set `save-toolstates` in `config.toml` so that tool status will - /// be saved to a JSON file. Then, run `x.py test --no-fail-fast` for all - /// of the tools to populate the JSON file. After that is done, this - /// command can be run to check for any status failures, and exits with an - /// error if there are any. - /// - /// This also handles publishing the results to the `history` directory of - /// the toolstate repo - /// if the env var `TOOLSTATE_PUBLISH` is set. Note that there is a - /// *separate* step of updating the `latest.json` file and creating GitHub - /// issues and comments in `src/ci/publish_toolstate.sh`, which is only - /// performed on master. (The shell/python code is intended to be migrated - /// here eventually.) - /// - /// The rules for failure are: - /// * If the PR modifies a tool, the status must be test-pass. - /// NOTE: There is intent to change this, see - /// . - /// * All "stable" tools must be test-pass on the stable or beta branches. - /// * During beta promotion week, a PR is not allowed to "regress" a - /// stable tool. That is, the status is not allowed to get worse - /// (test-pass to test-fail or build-fail). - fn run(self, builder: &Builder<'_>) { - if builder.config.dry_run() { - return; - } - - let days_since_beta_promotion = days_since_beta_promotion(); - let in_beta_week = days_since_beta_promotion >= BETA_WEEK_START; - let is_nightly = !(builder.config.channel == "beta" || builder.config.channel == "stable"); - let toolstates = builder.toolstates(); - - let mut did_error = false; - - for (tool, _) in STABLE_TOOLS.iter().chain(NIGHTLY_TOOLS.iter()) { - if !toolstates.contains_key(*tool) { - did_error = true; - eprintln!("error: Tool `{tool}` was not recorded in tool state."); - } - } - - if did_error { - crate::exit!(1); - } - - check_changed_files(&toolstates); - checkout_toolstate_repo(); - let old_toolstate = read_old_toolstate(); - - for (tool, _) in STABLE_TOOLS.iter() { - let state = toolstates[*tool]; - - if state != ToolState::TestPass { - if !is_nightly { - did_error = true; - eprintln!("error: Tool `{tool}` should be test-pass but is {state}"); - } else if in_beta_week { - let old_state = old_toolstate - .iter() - .find(|ts| ts.tool == *tool) - .expect("latest.json missing tool") - .state(); - if state < old_state { - did_error = true; - eprintln!( - "error: Tool `{tool}` has regressed from {old_state} to {state} during beta week." - ); - } else { - // This warning only appears in the logs, which most - // people won't read. It's mostly here for testing and - // debugging. - eprintln!( - "warning: Tool `{tool}` is not test-pass (is `{state}`), \ - this should be fixed before beta is branched." - ); - } - } - // `publish_toolstate.py` is responsible for updating - // `latest.json` and creating comments/issues warning people - // if there is a regression. That all happens in a separate CI - // job on the master branch once the PR has passed all tests - // on the `auto` branch. - } - } - - if did_error { - crate::exit!(1); - } - - if builder.config.channel == "nightly" && env::var_os("TOOLSTATE_PUBLISH").is_some() { - commit_toolstate_change(&toolstates); - } - } - - fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { - run.alias("check-tools") - } - - fn make_run(run: RunConfig<'_>) { - run.builder.ensure(ToolStateCheck); - } -} - -impl Builder<'_> { - fn toolstates(&self) -> HashMap, ToolState> { - if let Some(ref path) = self.config.save_toolstates { - if let Some(parent) = path.parent() { - // Ensure the parent directory always exists - t!(std::fs::create_dir_all(parent)); - } - let mut file = - t!(fs::OpenOptions::new().create(true).write(true).read(true).open(path)); - - serde_json::from_reader(&mut file).unwrap_or_default() - } else { - Default::default() - } - } - - /// Updates the actual toolstate of a tool. - /// - /// The toolstates are saved to the file specified by the key - /// `rust.save-toolstates` in `config.toml`. If unspecified, nothing will be - /// done. The file is updated immediately after this function completes. - pub fn save_toolstate(&self, tool: &str, state: ToolState) { - use std::io::Write; - - // If we're in a dry run setting we don't want to save toolstates as - // that means if we e.g. panic down the line it'll look like we tested - // everything (but we actually haven't). - if self.config.dry_run() { - return; - } - // Toolstate isn't tracked for clippy or rustfmt, but since most tools do, we avoid checking - // in all the places we could save toolstate and just do so here. - if tool == "clippy-driver" || tool == "rustfmt" { - return; - } - if let Some(ref path) = self.config.save_toolstates { - if let Some(parent) = path.parent() { - // Ensure the parent directory always exists - t!(std::fs::create_dir_all(parent)); - } - let mut file = - t!(fs::OpenOptions::new().create(true).read(true).write(true).open(path)); - - let mut current_toolstates: HashMap, ToolState> = - serde_json::from_reader(&mut file).unwrap_or_default(); - current_toolstates.insert(tool.into(), state); - t!(file.seek(SeekFrom::Start(0))); - t!(file.set_len(0)); - t!(serde_json::to_writer(&file, ¤t_toolstates)); - t!(writeln!(file)); // make sure this ends in a newline - } - } -} - -fn toolstate_repo() -> String { - env::var("TOOLSTATE_REPO") - .unwrap_or_else(|_| "https://github.com/rust-lang-nursery/rust-toolstate.git".to_string()) -} - -/// Directory where the toolstate repo is checked out. -const TOOLSTATE_DIR: &str = "rust-toolstate"; - -/// Checks out the toolstate repo into `TOOLSTATE_DIR`. -fn checkout_toolstate_repo() { - if let Ok(token) = env::var("TOOLSTATE_REPO_ACCESS_TOKEN") { - prepare_toolstate_config(&token); - } - if Path::new(TOOLSTATE_DIR).exists() { - eprintln!("Cleaning old toolstate directory..."); - t!(fs::remove_dir_all(TOOLSTATE_DIR)); - } - - let status = Command::new("git") - .arg("clone") - .arg("--depth=1") - .arg(toolstate_repo()) - .arg(TOOLSTATE_DIR) - .status(); - let success = match status { - Ok(s) => s.success(), - Err(_) => false, - }; - if !success { - panic!("git clone unsuccessful (status: {status:?})"); - } -} - -/// Sets up config and authentication for modifying the toolstate repo. -fn prepare_toolstate_config(token: &str) { - fn git_config(key: &str, value: &str) { - let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status(); - let success = match status { - Ok(s) => s.success(), - Err(_) => false, - }; - if !success { - panic!("git config key={key} value={value} failed (status: {status:?})"); - } - } - - // If changing anything here, then please check that `src/ci/publish_toolstate.sh` is up to date - // as well. - git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com"); - git_config("user.name", "Rust Toolstate Update"); - git_config("credential.helper", "store"); - - let credential = format!("https://{token}:x-oauth-basic@github.com\n",); - let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials"); - t!(fs::write(&git_credential_path, credential)); -} - -/// Reads the latest toolstate from the toolstate repo. -fn read_old_toolstate() -> Vec { - let latest_path = Path::new(TOOLSTATE_DIR).join("_data").join("latest.json"); - let old_toolstate = t!(fs::read(latest_path)); - t!(serde_json::from_slice(&old_toolstate)) -} - -/// This function `commit_toolstate_change` provides functionality for pushing a change -/// to the `rust-toolstate` repository. -/// -/// The function relies on a GitHub bot user, which should have a Personal access -/// token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for -/// some reason you need to change the token, please update the Azure Pipelines -/// variable group. -/// -/// 1. Generate a new Personal access token: -/// -/// * Login to the bot account, and go to Settings -> Developer settings -> -/// Personal access tokens -/// * Click "Generate new token" -/// * Enable the "public_repo" permission, then click "Generate token" -/// * Copy the generated token (should be a 40-digit hexadecimal number). -/// Save it somewhere secure, as the token would be gone once you leave -/// the page. -/// -/// 2. Update the variable group in Azure Pipelines -/// -/// * Ping a member of the infrastructure team to do this. -/// -/// 4. Replace the email address below if the bot account identity is changed -/// -/// * See -/// if a private email by GitHub is wanted. -fn commit_toolstate_change(current_toolstate: &ToolstateData) { - let message = format!("({} CI update)", OS.expect("linux/windows only")); - let mut success = false; - for _ in 1..=5 { - // Upload the test results (the new commit-to-toolstate mapping) to the toolstate repo. - // This does *not* change the "current toolstate"; that only happens post-landing - // via `src/ci/docker/publish_toolstate.sh`. - publish_test_results(¤t_toolstate); - - // `git commit` failing means nothing to commit. - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) - .arg("commit") - .arg("-a") - .arg("-m") - .arg(&message) - .status()); - if !status.success() { - success = true; - break; - } - - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) - .arg("push") - .arg("origin") - .arg("master") - .status()); - // If we successfully push, exit. - if status.success() { - success = true; - break; - } - eprintln!("Sleeping for 3 seconds before retrying push"); - std::thread::sleep(std::time::Duration::from_secs(3)); - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) - .arg("fetch") - .arg("origin") - .arg("master") - .status()); - assert!(status.success()); - let status = t!(Command::new("git") - .current_dir(TOOLSTATE_DIR) - .arg("reset") - .arg("--hard") - .arg("origin/master") - .status()); - assert!(status.success()); - } - - if !success { - panic!("Failed to update toolstate repository with new data"); - } -} - -/// Updates the "history" files with the latest results. -/// -/// These results will later be promoted to `latest.json` by the -/// `publish_toolstate.py` script if the PR passes all tests and is merged to -/// master. -fn publish_test_results(current_toolstate: &ToolstateData) { - let commit = t!(std::process::Command::new("git").arg("rev-parse").arg("HEAD").output()); - let commit = t!(String::from_utf8(commit.stdout)); - - let toolstate_serialized = t!(serde_json::to_string(¤t_toolstate)); - - let history_path = Path::new(TOOLSTATE_DIR) - .join("history") - .join(format!("{}.tsv", OS.expect("linux/windows only"))); - let mut file = t!(fs::read_to_string(&history_path)); - let end_of_first_line = file.find('\n').unwrap(); - file.insert_str(end_of_first_line, &format!("\n{}\t{}", commit.trim(), toolstate_serialized)); - t!(fs::write(&history_path, file)); -} - -#[derive(Debug, Deserialize)] -struct RepoState { - tool: String, - windows: ToolState, - linux: ToolState, -} - -impl RepoState { - fn state(&self) -> ToolState { - if cfg!(target_os = "linux") { - self.linux - } else if cfg!(windows) { - self.windows - } else { - unimplemented!() - } - } -} diff --git a/src/bootstrap/util.rs b/src/bootstrap/util.rs deleted file mode 100644 index 3c4a21434..000000000 --- a/src/bootstrap/util.rs +++ /dev/null @@ -1,497 +0,0 @@ -//! Various utility functions used throughout rustbuild. -//! -//! Simple things like testing the various filesystem operations here and there, -//! not a lot of interesting happenings here unfortunately. - -use build_helper::util::{fail, try_run}; -use std::env; -use std::fs; -use std::io; -use std::path::{Path, PathBuf}; -use std::process::{Command, Stdio}; -use std::str; -use std::time::{Instant, SystemTime, UNIX_EPOCH}; - -use crate::builder::Builder; -use crate::config::{Config, TargetSelection}; -use crate::OnceCell; - -/// A helper macro to `unwrap` a result except also print out details like: -/// -/// * The file/line of the panic -/// * The expression that failed -/// * The error itself -/// -/// This is currently used judiciously throughout the build system rather than -/// using a `Result` with `try!`, but this may change one day... -#[macro_export] -macro_rules! t { - ($e:expr) => { - match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {}", stringify!($e), e), - } - }; - // it can show extra info in the second parameter - ($e:expr, $extra:expr) => { - match $e { - Ok(e) => e, - Err(e) => panic!("{} failed with {} ({:?})", stringify!($e), e, $extra), - } - }; -} -pub use t; - -/// Given an executable called `name`, return the filename for the -/// executable for a particular target. -pub fn exe(name: &str, target: TargetSelection) -> String { - if target.contains("windows") { - format!("{name}.exe") - } else if target.contains("uefi") { - format!("{name}.efi") - } else { - name.to_string() - } -} - -/// Returns `true` if the file name given looks like a dynamic library. -pub fn is_dylib(name: &str) -> bool { - name.ends_with(".dylib") || name.ends_with(".so") || name.ends_with(".dll") -} - -/// Returns `true` if the file name given looks like a debug info file -pub fn is_debug_info(name: &str) -> bool { - // FIXME: consider split debug info on other platforms (e.g., Linux, macOS) - name.ends_with(".pdb") -} - -/// Returns the corresponding relative library directory that the compiler's -/// dylibs will be found in. -pub fn libdir(target: TargetSelection) -> &'static str { - if target.contains("windows") { "bin" } else { "lib" } -} - -/// Adds a list of lookup paths to `cmd`'s dynamic library lookup path. -/// If the dylib_path_var is already set for this cmd, the old value will be overwritten! -pub fn add_dylib_path(path: Vec, cmd: &mut Command) { - let mut list = dylib_path(); - for path in path { - list.insert(0, path); - } - cmd.env(dylib_path_var(), t!(env::join_paths(list))); -} - -include!("dylib_util.rs"); - -/// Adds a list of lookup paths to `cmd`'s link library lookup path. -pub fn add_link_lib_path(path: Vec, cmd: &mut Command) { - let mut list = link_lib_path(); - for path in path { - list.insert(0, path); - } - cmd.env(link_lib_path_var(), t!(env::join_paths(list))); -} - -/// Returns the environment variable which the link library lookup path -/// resides in for this platform. -fn link_lib_path_var() -> &'static str { - if cfg!(target_env = "msvc") { "LIB" } else { "LIBRARY_PATH" } -} - -/// Parses the `link_lib_path_var()` environment variable, returning a list of -/// paths that are members of this lookup path. -fn link_lib_path() -> Vec { - let var = match env::var_os(link_lib_path_var()) { - Some(v) => v, - None => return vec![], - }; - env::split_paths(&var).collect() -} - -pub struct TimeIt(bool, Instant); - -/// Returns an RAII structure that prints out how long it took to drop. -pub fn timeit(builder: &Builder<'_>) -> TimeIt { - TimeIt(builder.config.dry_run(), Instant::now()) -} - -impl Drop for TimeIt { - fn drop(&mut self) { - let time = self.1.elapsed(); - if !self.0 { - println!("\tfinished in {}.{:03} seconds", time.as_secs(), time.subsec_millis()); - } - } -} - -/// Used for download caching -pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { - if !stamp.exists() { - return true; - } - t!(fs::read_to_string(stamp)) != key -} - -/// Symlinks two directories, using junctions on Windows and normal symlinks on -/// Unix. -pub fn symlink_dir(config: &Config, original: &Path, link: &Path) -> io::Result<()> { - if config.dry_run() { - return Ok(()); - } - let _ = fs::remove_dir(link); - return symlink_dir_inner(original, link); - - #[cfg(not(windows))] - fn symlink_dir_inner(original: &Path, link: &Path) -> io::Result<()> { - use std::os::unix::fs; - fs::symlink(original, link) - } - - #[cfg(windows)] - fn symlink_dir_inner(target: &Path, junction: &Path) -> io::Result<()> { - junction::create(&target, &junction) - } -} - -pub fn forcing_clang_based_tests() -> bool { - if let Some(var) = env::var_os("RUSTBUILD_FORCE_CLANG_BASED_TESTS") { - match &var.to_string_lossy().to_lowercase()[..] { - "1" | "yes" | "on" => true, - "0" | "no" | "off" => false, - other => { - // Let's make sure typos don't go unnoticed - panic!( - "Unrecognized option '{other}' set in \ - RUSTBUILD_FORCE_CLANG_BASED_TESTS" - ) - } - } - } else { - false - } -} - -pub fn use_host_linker(target: TargetSelection) -> bool { - // FIXME: this information should be gotten by checking the linker flavor - // of the rustc target - !(target.contains("emscripten") - || target.contains("wasm32") - || target.contains("nvptx") - || target.contains("fortanix") - || target.contains("fuchsia") - || target.contains("bpf") - || target.contains("switch")) -} - -pub fn is_valid_test_suite_arg<'a, P: AsRef>( - path: &'a Path, - suite_path: P, - builder: &Builder<'_>, -) -> Option<&'a str> { - let suite_path = suite_path.as_ref(); - let path = match path.strip_prefix(".") { - Ok(p) => p, - Err(_) => path, - }; - if !path.starts_with(suite_path) { - return None; - } - let abs_path = builder.src.join(path); - let exists = abs_path.is_dir() || abs_path.is_file(); - if !exists { - panic!( - "Invalid test suite filter \"{}\": file or directory does not exist", - abs_path.display() - ); - } - // Since test suite paths are themselves directories, if we don't - // specify a directory or file, we'll get an empty string here - // (the result of the test suite directory without its suite prefix). - // Therefore, we need to filter these out, as only the first --test-args - // flag is respected, so providing an empty --test-args conflicts with - // any following it. - match path.strip_prefix(suite_path).ok().and_then(|p| p.to_str()) { - Some(s) if !s.is_empty() => Some(s), - _ => None, - } -} - -pub fn run(cmd: &mut Command, print_cmd_on_fail: bool) { - if try_run(cmd, print_cmd_on_fail).is_err() { - crate::exit!(1); - } -} - -pub fn check_run(cmd: &mut Command, print_cmd_on_fail: bool) -> bool { - let status = match cmd.status() { - Ok(status) => status, - Err(e) => { - println!("failed to execute command: {cmd:?}\nerror: {e}"); - return false; - } - }; - if !status.success() && print_cmd_on_fail { - println!( - "\n\ncommand did not execute successfully: {cmd:?}\n\ - expected success, got: {status}\n\n" - ); - } - status.success() -} - -pub fn run_suppressed(cmd: &mut Command) { - if !try_run_suppressed(cmd) { - crate::exit!(1); - } -} - -pub fn try_run_suppressed(cmd: &mut Command) -> bool { - let output = match cmd.output() { - Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")), - }; - if !output.status.success() { - println!( - "\n\ncommand did not execute successfully: {:?}\n\ - expected success, got: {}\n\n\ - stdout ----\n{}\n\ - stderr ----\n{}\n\n", - cmd, - output.status, - String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) - ); - } - output.status.success() -} - -pub fn make(host: &str) -> PathBuf { - if host.contains("dragonfly") - || host.contains("freebsd") - || host.contains("netbsd") - || host.contains("openbsd") - { - PathBuf::from("gmake") - } else { - PathBuf::from("make") - } -} - -#[track_caller] -pub fn output(cmd: &mut Command) -> String { - let output = match cmd.stderr(Stdio::inherit()).output() { - Ok(status) => status, - Err(e) => fail(&format!("failed to execute command: {cmd:?}\nerror: {e}")), - }; - if !output.status.success() { - panic!( - "command did not execute successfully: {:?}\n\ - expected success, got: {}", - cmd, output.status - ); - } - String::from_utf8(output.stdout).unwrap() -} - -pub fn output_result(cmd: &mut Command) -> Result { - let output = match cmd.stderr(Stdio::inherit()).output() { - Ok(status) => status, - Err(e) => return Err(format!("failed to run command: {cmd:?}: {e}")), - }; - if !output.status.success() { - return Err(format!( - "command did not execute successfully: {:?}\n\ - expected success, got: {}\n{}", - cmd, - output.status, - String::from_utf8(output.stderr).map_err(|err| format!("{err:?}"))? - )); - } - Ok(String::from_utf8(output.stdout).map_err(|err| format!("{err:?}"))?) -} - -/// Returns the last-modified time for `path`, or zero if it doesn't exist. -pub fn mtime(path: &Path) -> SystemTime { - fs::metadata(path).and_then(|f| f.modified()).unwrap_or(UNIX_EPOCH) -} - -/// Returns `true` if `dst` is up to date given that the file or files in `src` -/// are used to generate it. -/// -/// Uses last-modified time checks to verify this. -pub fn up_to_date(src: &Path, dst: &Path) -> bool { - if !dst.exists() { - return false; - } - let threshold = mtime(dst); - let meta = match fs::metadata(src) { - Ok(meta) => meta, - Err(e) => panic!("source {src:?} failed to get metadata: {e}"), - }; - if meta.is_dir() { - dir_up_to_date(src, threshold) - } else { - meta.modified().unwrap_or(UNIX_EPOCH) <= threshold - } -} - -fn dir_up_to_date(src: &Path, threshold: SystemTime) -> bool { - t!(fs::read_dir(src)).map(|e| t!(e)).all(|e| { - let meta = t!(e.metadata()); - if meta.is_dir() { - dir_up_to_date(&e.path(), threshold) - } else { - meta.modified().unwrap_or(UNIX_EPOCH) < threshold - } - }) -} - -/// Copied from `std::path::absolute` until it stabilizes. -/// -/// FIXME: this shouldn't exist. -pub(crate) fn absolute(path: &Path) -> PathBuf { - if path.as_os_str().is_empty() { - panic!("can't make empty path absolute"); - } - #[cfg(unix)] - { - t!(absolute_unix(path), format!("could not make path absolute: {}", path.display())) - } - #[cfg(windows)] - { - t!(absolute_windows(path), format!("could not make path absolute: {}", path.display())) - } - #[cfg(not(any(unix, windows)))] - { - println!("warning: bootstrap is not supported on non-unix platforms"); - t!(std::fs::canonicalize(t!(std::env::current_dir()))).join(path) - } -} - -#[cfg(unix)] -/// Make a POSIX path absolute without changing its semantics. -fn absolute_unix(path: &Path) -> io::Result { - // This is mostly a wrapper around collecting `Path::components`, with - // exceptions made where this conflicts with the POSIX specification. - // See 4.13 Pathname Resolution, IEEE Std 1003.1-2017 - // https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 - - use std::os::unix::prelude::OsStrExt; - let mut components = path.components(); - let path_os = path.as_os_str().as_bytes(); - - let mut normalized = if path.is_absolute() { - // "If a pathname begins with two successive characters, the - // first component following the leading characters may be - // interpreted in an implementation-defined manner, although more than - // two leading characters shall be treated as a single - // character." - if path_os.starts_with(b"//") && !path_os.starts_with(b"///") { - components.next(); - PathBuf::from("//") - } else { - PathBuf::new() - } - } else { - env::current_dir()? - }; - normalized.extend(components); - - // "Interfaces using pathname resolution may specify additional constraints - // when a pathname that does not name an existing directory contains at - // least one non- character and contains one or more trailing - // characters". - // A trailing is also meaningful if "a symbolic link is - // encountered during pathname resolution". - - if path_os.ends_with(b"/") { - normalized.push(""); - } - - Ok(normalized) -} - -#[cfg(windows)] -fn absolute_windows(path: &std::path::Path) -> std::io::Result { - use std::ffi::OsString; - use std::io::Error; - use std::os::windows::ffi::{OsStrExt, OsStringExt}; - use std::ptr::null_mut; - #[link(name = "kernel32")] - extern "system" { - fn GetFullPathNameW( - lpFileName: *const u16, - nBufferLength: u32, - lpBuffer: *mut u16, - lpFilePart: *mut *const u16, - ) -> u32; - } - - unsafe { - // encode the path as UTF-16 - let path: Vec = path.as_os_str().encode_wide().chain([0]).collect(); - let mut buffer = Vec::new(); - // Loop until either success or failure. - loop { - // Try to get the absolute path - let len = GetFullPathNameW( - path.as_ptr(), - buffer.len().try_into().unwrap(), - buffer.as_mut_ptr(), - null_mut(), - ); - match len as usize { - // Failure - 0 => return Err(Error::last_os_error()), - // Buffer is too small, resize. - len if len > buffer.len() => buffer.resize(len, 0), - // Success! - len => { - buffer.truncate(len); - return Ok(OsString::from_wide(&buffer).into()); - } - } - } - } -} - -/// Adapted from -/// -/// When `clang-cl` is used with instrumentation, we need to add clang's runtime library resource -/// directory to the linker flags, otherwise there will be linker errors about the profiler runtime -/// missing. This function returns the path to that directory. -pub fn get_clang_cl_resource_dir(clang_cl_path: &str) -> PathBuf { - // Similar to how LLVM does it, to find clang's library runtime directory: - // - we ask `clang-cl` to locate the `clang_rt.builtins` lib. - let mut builtins_locator = Command::new(clang_cl_path); - builtins_locator.args(&["/clang:-print-libgcc-file-name", "/clang:--rtlib=compiler-rt"]); - - let clang_rt_builtins = output(&mut builtins_locator); - let clang_rt_builtins = Path::new(clang_rt_builtins.trim()); - assert!( - clang_rt_builtins.exists(), - "`clang-cl` must correctly locate the library runtime directory" - ); - - // - the profiler runtime will be located in the same directory as the builtins lib, like - // `$LLVM_DISTRO_ROOT/lib/clang/$LLVM_VERSION/lib/windows`. - let clang_rt_dir = clang_rt_builtins.parent().expect("The clang lib folder should exist"); - clang_rt_dir.to_path_buf() -} - -pub fn lld_flag_no_threads(is_windows: bool) -> &'static str { - static LLD_NO_THREADS: OnceCell<(&'static str, &'static str)> = OnceCell::new(); - let (windows, other) = LLD_NO_THREADS.get_or_init(|| { - let out = output(Command::new("lld").arg("-flavor").arg("ld").arg("--version")); - let newer = match (out.find(char::is_numeric), out.find('.')) { - (Some(b), Some(e)) => out.as_str()[b..e].parse::().ok().unwrap_or(14) > 10, - _ => true, - }; - if newer { ("/threads:1", "--threads=1") } else { ("/no-threads", "--no-threads") } - }); - if is_windows { windows } else { other } -} - -pub fn dir_is_empty(dir: &Path) -> bool { - t!(std::fs::read_dir(dir)).next().is_none() -} diff --git a/src/ci/docker/host-x86_64/arm-android/Dockerfile b/src/ci/docker/host-x86_64/arm-android/Dockerfile index db11700af..abca06fb9 100644 --- a/src/ci/docker/host-x86_64/arm-android/Dockerfile +++ b/src/ci/docker/host-x86_64/arm-android/Dockerfile @@ -30,7 +30,7 @@ ENV PATH=$PATH:/android/sdk/platform-tools ENV TARGETS=arm-linux-androideabi -ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ +ENV RUST_CONFIGURE_ARGS --android-ndk=/android/ndk/ ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-android/Dockerfile b/src/ci/docker/host-x86_64/dist-android/Dockerfile index b09b6edb0..20b72b377 100644 --- a/src/ci/docker/host-x86_64/dist-android/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-android/Dockerfile @@ -19,12 +19,7 @@ ENV TARGETS=$TARGETS,x86_64-linux-android ENV RUST_CONFIGURE_ARGS \ --enable-extended \ --enable-profiler \ - --arm-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --armv7-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --thumbv7neon-linux-androideabi-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --i686-linux-android-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --aarch64-linux-android-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ - --x86_64-linux-android-ndk=/android/ndk/toolchains/llvm/prebuilt/linux-x86_64/ \ + --android-ndk=/android/ndk/ \ --disable-docs ENV SCRIPT python3 ../x.py dist --host='' --target $TARGETS diff --git a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile index 8f4ad0f4e..3372baed9 100644 --- a/src/ci/docker/host-x86_64/dist-various-1/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-various-1/Dockerfile @@ -29,8 +29,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ g++-arm-linux-gnueabi \ g++-arm-linux-gnueabihf \ g++-aarch64-linux-gnu \ - g++-mips64-linux-gnuabi64 \ - g++-mips64el-linux-gnuabi64 \ gcc-arm-none-eabi \ gcc-sparc64-linux-gnu \ libc6-dev-sparc64-cross \ @@ -48,12 +46,6 @@ WORKDIR /build COPY host-x86_64/dist-various-1/install-x86_64-redox.sh /build RUN ./install-x86_64-redox.sh -COPY host-x86_64/dist-various-1/install-mips-musl.sh /build -RUN ./install-mips-musl.sh - -COPY host-x86_64/dist-various-1/install-mipsel-musl.sh /build -RUN ./install-mipsel-musl.sh - COPY host-x86_64/dist-various-1/install-aarch64-none-elf.sh /build RUN ./install-aarch64-none-elf.sh @@ -76,32 +68,7 @@ RUN env \ env \ CC=arm-linux-gnueabihf-gcc CFLAGS="-march=armv7-a+fp" \ CXX=arm-linux-gnueabihf-g++ CXXFLAGS="-march=armv7-a+fp" \ - bash musl.sh armv7hf && \ - env \ - CC=mips-openwrt-linux-gcc \ - CXX=mips-openwrt-linux-g++ \ - bash musl.sh mips && \ - env \ - CC=mipsel-openwrt-linux-gcc \ - CXX=mipsel-openwrt-linux-g++ \ - bash musl.sh mipsel && \ - env \ - CC=mips64-linux-gnuabi64-gcc \ - CXX=mips64-linux-gnuabi64-g++ \ - bash musl.sh mips64 && \ - env \ - CC=mips64el-linux-gnuabi64-gcc \ - CXX=mips64el-linux-gnuabi64-g++ \ - bash musl.sh mips64el && \ - rm -rf /build/* - -# FIXME(mozilla/sccache#235) this shouldn't be necessary but is currently -# necessary to disambiguate the mips compiler with the mipsel compiler. We want -# to give these two wrapper scripts (currently identical ones) different hashes -# to ensure that sccache understands that they're different compilers. -RUN \ - echo "# a" >> /usr/local/mips-linux-musl/bin/mips-openwrt-linux-musl-wrapper.sh && \ - echo "# b" >> /usr/local/mipsel-linux-musl/bin/mipsel-openwrt-linux-musl-wrapper.sh + bash musl.sh armv7hf ENV RUN_MAKE_TARGETS=thumbv6m-none-eabi ENV RUN_MAKE_TARGETS=$RUN_MAKE_TARGETS,thumbv7m-none-eabi @@ -110,10 +77,6 @@ ENV RUN_MAKE_TARGETS=$RUN_MAKE_TARGETS,thumbv7em-none-eabihf ENV TARGETS=asmjs-unknown-emscripten ENV TARGETS=$TARGETS,wasm32-unknown-emscripten -ENV TARGETS=$TARGETS,mips-unknown-linux-musl -ENV TARGETS=$TARGETS,mipsel-unknown-linux-musl -ENV TARGETS=$TARGETS,mips64-unknown-linux-muslabi64 -ENV TARGETS=$TARGETS,mips64el-unknown-linux-muslabi64 ENV TARGETS=$TARGETS,arm-unknown-linux-musleabi ENV TARGETS=$TARGETS,arm-unknown-linux-musleabihf ENV TARGETS=$TARGETS,armv5te-unknown-linux-gnueabi @@ -149,10 +112,6 @@ ENV CFLAGS_armv5te_unknown_linux_musleabi="-march=armv5te -marm -mfloat-abi=soft CFLAGS_arm_unknown_linux_musleabi="-march=armv6 -marm" \ CFLAGS_arm_unknown_linux_musleabihf="-march=armv6 -marm -mfpu=vfp" \ CFLAGS_armv7_unknown_linux_musleabihf="-march=armv7-a+fp" \ - CC_mipsel_unknown_linux_musl=mipsel-openwrt-linux-gcc \ - CC_mips_unknown_linux_musl=mips-openwrt-linux-gcc \ - CC_mips64el_unknown_linux_muslabi64=mips64el-linux-gnuabi64-gcc \ - CC_mips64_unknown_linux_muslabi64=mips64-linux-gnuabi64-gcc \ CC_sparc64_unknown_linux_gnu=sparc64-linux-gnu-gcc \ CC_x86_64_unknown_redox=x86_64-unknown-redox-gcc \ CC_thumbv7neon_unknown_linux_gnueabihf=arm-linux-gnueabihf-gcc \ @@ -177,10 +136,6 @@ ENV RUST_CONFIGURE_ARGS \ --musl-root-arm=/musl-arm \ --musl-root-armhf=/musl-armhf \ --musl-root-armv7hf=/musl-armv7hf \ - --musl-root-mips=/musl-mips \ - --musl-root-mipsel=/musl-mipsel \ - --musl-root-mips64=/musl-mips64 \ - --musl-root-mips64el=/musl-mips64el \ --disable-docs ENV SCRIPT \ diff --git a/src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh b/src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh deleted file mode 100755 index abab18093..000000000 --- a/src/ci/docker/host-x86_64/dist-various-1/install-mips-musl.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -ex - -mkdir /usr/local/mips-linux-musl - -# originally from -# https://downloads.openwrt.org/snapshots/trunk/ar71xx/generic/ -# OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2 -URL="https://ci-mirrors.rust-lang.org/rustc" -FILE="OpenWrt-Toolchain-ar71xx-generic_gcc-5.3.0_musl-1.1.16.Linux-x86_64.tar.bz2" -curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mips-linux-musl --strip-components=2 - -for file in /usr/local/mips-linux-musl/bin/mips-openwrt-linux-*; do - ln -s $file /usr/local/bin/`basename $file` -done diff --git a/src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh b/src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh deleted file mode 100755 index 779acb2d8..000000000 --- a/src/ci/docker/host-x86_64/dist-various-1/install-mipsel-musl.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh -set -ex - -mkdir /usr/local/mipsel-linux-musl - -# Note that this originally came from: -# https://downloads.openwrt.org/snapshots/trunk/malta/generic/ -# OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2 -URL="https://ci-mirrors.rust-lang.org/rustc" -FILE="OpenWrt-Toolchain-malta-le_gcc-5.3.0_musl-1.1.15.Linux-x86_64.tar.bz2" -curl -L "$URL/$FILE" | tar xjf - -C /usr/local/mipsel-linux-musl --strip-components=2 - -for file in /usr/local/mipsel-linux-musl/bin/mipsel-openwrt-linux-*; do - ln -s $file /usr/local/bin/`basename $file` -done diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 6f1b2a6a6..9a2fcb0ce 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -57,9 +57,9 @@ COPY host-x86_64/dist-x86_64-linux/build-clang.sh /tmp/ RUN ./build-clang.sh ENV CC=clang CXX=clang++ -# rustc-perf version from 2023-05-30 +# rustc-perf version from 2023-10-22 # Should also be changed in the opt-dist tool for other environments. -ENV PERF_COMMIT 8b2ac3042e1ff2c0074455a0a3618adef97156b1 +ENV PERF_COMMIT 4f313add609f43e928e98132358e8426ed3969ae RUN curl -LS -o perf.zip https://ci-mirrors.rust-lang.org/rustc/rustc-perf-$PERF_COMMIT.zip && \ unzip perf.zip && \ mv rustc-perf-$PERF_COMMIT rustc-perf && \ @@ -84,7 +84,8 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.ninja=false \ --set rust.jemalloc \ --set rust.use-lld=true \ - --set rust.lto=thin + --set rust.lto=thin \ + --set rust.codegen-units=1 ENV SCRIPT python3 ../x.py build --set rust.debug=true opt-dist && \ ./build/$HOSTS/stage0-tools-bin/opt-dist linux-ci -- python3 ../x.py dist \ diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh index 02b023fe7..1d9568702 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-clang.sh @@ -4,7 +4,7 @@ set -ex source shared.sh -LLVM=llvmorg-17.0.0-rc3 +LLVM=llvmorg-17.0.4 mkdir llvm-project cd llvm-project diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh index 6da3f8922..e939a5d7e 100755 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/build-gcc.sh @@ -3,7 +3,8 @@ set -ex source shared.sh -GCC=8.5.0 +# Note: in the future when bumping to version 10.1.0, also take care of the sed block below. +GCC=9.5.0 curl https://ftp.gnu.org/gnu/gcc/gcc-$GCC/gcc-$GCC.tar.xz | xzcat | tar xf - cd gcc-$GCC @@ -22,15 +23,25 @@ cd gcc-$GCC # latter host is presented to `wget`! Therefore, we choose to download from the insecure HTTP server # instead here. # +# Note: in version 10.1.0, the URL used in `download_prerequisites` has changed from using FTP to +# using HTTP. When bumping to that gcc version, we can likely remove the sed replacement below, or +# the expression will need to be updated. That new URL is available at: +# https://github.com/gcc-mirror/gcc/blob/6e6e3f144a33ae504149dc992453b4f6dea12fdb/contrib/download_prerequisites#L35 +# sed -i'' 's|ftp://gcc\.gnu\.org/|https://gcc.gnu.org/|g' ./contrib/download_prerequisites ./contrib/download_prerequisites mkdir ../gcc-build cd ../gcc-build + +# '-fno-reorder-blocks-and-partition' is required to +# enable BOLT optimization of the C++ standard library, +# which is included in librustc_driver.so hide_output ../gcc-$GCC/configure \ --prefix=/rustroot \ --enable-languages=c,c++ \ - --disable-gnu-unique-object + --disable-gnu-unique-object \ + --enable-cxx-flags='-fno-reorder-blocks-and-partition' hide_output make -j$(nproc) hide_output make install ln -s gcc /rustroot/bin/cc diff --git a/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock new file mode 100644 index 000000000..e983edf20 --- /dev/null +++ b/src/ci/docker/host-x86_64/test-various/uefi_qemu_test/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "r-efi" +version = "4.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "575fc2d9b3da54adbdfaddf6eca48fec256d977c8630a1750b8991347d1ac911" + +[[package]] +name = "uefi_qemu_test" +version = "0.0.0" +dependencies = [ + "r-efi", +] diff --git a/src/ci/docker/host-x86_64/wasm32/Dockerfile b/src/ci/docker/host-x86_64/wasm32/Dockerfile deleted file mode 100644 index 0d0f1edd0..000000000 --- a/src/ci/docker/host-x86_64/wasm32/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -FROM ubuntu:22.04 - -ARG DEBIAN_FRONTEND=noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends \ - g++ \ - make \ - ninja-build \ - file \ - curl \ - ca-certificates \ - python3 \ - git \ - cmake \ - sudo \ - gdb \ - xz-utils \ - libssl-dev \ - bzip2 \ - && rm -rf /var/lib/apt/lists/* - -COPY scripts/emscripten.sh /scripts/ -RUN bash /scripts/emscripten.sh - -COPY scripts/sccache.sh /scripts/ -RUN sh /scripts/sccache.sh - -# emcc seems to need python to specifically be "python" and not "python3" -RUN ln `which python3` /usr/bin/python - -ENV PATH=$PATH:/emsdk-portable -ENV PATH=$PATH:/emsdk-portable/upstream/emscripten/ - -# Rust's build system requires NodeJS to be in the path, but the directory in -# which emsdk stores it contains the version number. This caused breakages in -# the past when emsdk bumped the node version causing the path to point to a -# missing directory. -# -# To avoid the problem this symlinks the latest NodeJs version available to -# "latest", and adds that to the path. -RUN ln -s /emsdk-portable/node/$(ls /emsdk-portable/node | sort -V | tail -n 1) \ - /emsdk-portable/node/latest -ENV PATH=$PATH:/emsdk-portable/node/latest/bin/ - -ENV BINARYEN_ROOT=/emsdk-portable/upstream/ -ENV EMSDK=/emsdk-portable -ENV EM_CONFIG=/emsdk-portable/.emscripten -ENV EM_CACHE=/emsdk-portable/upstream/emscripten/cache - -ENV TARGETS=wasm32-unknown-emscripten - -# Use -O1 optimizations in the link step to reduce time spent optimizing. -ENV EMCC_CFLAGS=-O1 - -COPY static/gitconfig /etc/gitconfig - -# Emscripten installation is user-specific -ENV NO_CHANGE_USER=1 -RUN chown 10719 -R /emsdk-portable/ - -# Exclude library/alloc due to OOM in benches. -# FIXME: Fix std tests -ENV SCRIPT python3 ../x.py test --stage 2 --host='' --target $TARGETS \ - --skip library/alloc --skip library/std diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile index 444e0275d..cefdcad76 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/Dockerfile @@ -24,6 +24,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ xz-utils \ nodejs \ mingw-w64 \ + libgccjit-12-dev \ && rm -rf /var/lib/apt/lists/* # Install powershell (universal package) so we can test x.ps1 on Linux @@ -34,6 +35,9 @@ RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# Make `libgccjit.so` accessible to the linker. +RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so + # We are disabling CI LLVM since this builder is intentionally using a host # LLVM, rather than the typical src/llvm-project LLVM. ENV NO_DOWNLOAD_CI_LLVM 1 @@ -47,6 +51,7 @@ ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ --llvm-root=/usr/lib/llvm-15 \ --enable-llvm-link-shared \ + $USE_NEW_MANGLING \ --set rust.thin-lto-import-instr-limit=10 COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh index 918b19612..2eb751ca3 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-15/script.sh @@ -4,34 +4,46 @@ set -ex # Only run the stage 1 tests on merges, not on PR CI jobs. if [[ -z "${PR_CI_JOB}" ]]; then - ../x.py --stage 1 test --skip src/tools/tidy && \ - # Run the `mir-opt` tests again but this time for a 32-bit target. - # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have - # both 32-bit and 64-bit outputs updated by the PR author, before - # the PR is approved and tested for merging. - # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, - # despite having different output on 32-bit vs 64-bit targets. - ../x.py --stage 1 test tests/mir-opt \ - --host='' --target=i686-unknown-linux-gnu && \ - # Run `ui-fulldeps` in `--stage=1`, which actually uses the stage0 - # compiler, and is sensitive to the addition of new flags. - ../x.py --stage 1 test tests/ui-fulldeps + # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen + # tests as it will fail them. + if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then + ../x.py --stage 1 test --skip src/tools/tidy --skip tests/codegen + else + ../x.py --stage 1 test --skip src/tools/tidy + fi + + # Run the `mir-opt` tests again but this time for a 32-bit target. + # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have + # both 32-bit and 64-bit outputs updated by the PR author, before + # the PR is approved and tested for merging. + # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, + # despite having different output on 32-bit vs 64-bit targets. + ../x.py --stage 1 test tests/mir-opt --host='' --target=i686-unknown-linux-gnu + + # Run `ui-fulldeps` in `--stage=1`, which actually uses the stage0 + # compiler, and is sensitive to the addition of new flags. + ../x.py --stage 1 test tests/ui-fulldeps fi +# When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen +# tests as it will fail them. # NOTE: intentionally uses all of `x.py`, `x`, and `x.ps1` to make sure they all work on Linux. -../x.py --stage 2 test --skip src/tools/tidy && \ - # Run the `mir-opt` tests again but this time for a 32-bit target. - # This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have - # both 32-bit and 64-bit outputs updated by the PR author, before - # the PR is approved and tested for merging. - # It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, - # despite having different output on 32-bit vs 64-bit targets. - ../x --stage 2 test tests/mir-opt \ - --host='' --target=i686-unknown-linux-gnu && \ - # Run the UI test suite again, but in `--pass=check` mode - # - # This is intended to make sure that both `--pass=check` continues to - # work. - # - ../x.ps1 --stage 2 test tests/ui --pass=check \ - --host='' --target=i686-unknown-linux-gnu +if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then + ../x.py --stage 2 test --skip src/tools/tidy --skip tests/codegen +else + ../x.py --stage 2 test --skip src/tools/tidy +fi + +# Run the `mir-opt` tests again but this time for a 32-bit target. +# This enforces that tests using `// EMIT_MIR_FOR_EACH_BIT_WIDTH` have +# both 32-bit and 64-bit outputs updated by the PR author, before +# the PR is approved and tested for merging. +# It will also detect tests lacking `// EMIT_MIR_FOR_EACH_BIT_WIDTH`, +# despite having different output on 32-bit vs 64-bit targets. +../x --stage 2 test tests/mir-opt --host='' --target=i686-unknown-linux-gnu + +# Run the UI test suite again, but in `--pass=check` mode +# +# This is intended to make sure that both `--pass=check` continues to +# work. +../x.ps1 --stage 2 test tests/ui --pass=check --host='' --target=i686-unknown-linux-gnu diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile index 1e2b802e6..c177e7387 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-16/Dockerfile @@ -38,6 +38,10 @@ RUN sh /scripts/sccache.sh # LLVM, rather than the typical src/llvm-project LLVM. ENV NO_DOWNLOAD_CI_LLVM 1 +# This is not the latest LLVM version, so some components required by tests may +# be missing. +ENV IS_NOT_LATEST_LLVM 1 + # Using llvm-link-shared due to libffi issues -- see #34486 ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile new file mode 100644 index 000000000..76846f1fe --- /dev/null +++ b/src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile @@ -0,0 +1,50 @@ +FROM ubuntu:23.10 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends \ + g++ \ + gcc-multilib \ + make \ + ninja-build \ + file \ + curl \ + ca-certificates \ + python3 \ + git \ + cmake \ + sudo \ + gdb \ + llvm-17-tools \ + llvm-17-dev \ + libedit-dev \ + libssl-dev \ + pkg-config \ + zlib1g-dev \ + xz-utils \ + nodejs \ + mingw-w64 \ + && rm -rf /var/lib/apt/lists/* + +# Install powershell (universal package) so we can test x.ps1 on Linux +RUN curl -sL "https://github.com/PowerShell/PowerShell/releases/download/v7.3.1/powershell_7.3.1-1.deb_amd64.deb" > powershell.deb && \ + dpkg -i powershell.deb && \ + rm -f powershell.deb + +COPY scripts/sccache.sh /scripts/ +RUN sh /scripts/sccache.sh + +# We are disabling CI LLVM since this builder is intentionally using a host +# LLVM, rather than the typical src/llvm-project LLVM. +ENV NO_DOWNLOAD_CI_LLVM 1 + +# Using llvm-link-shared due to libffi issues -- see #34486 +ENV RUST_CONFIGURE_ARGS \ + --build=x86_64-unknown-linux-gnu \ + --llvm-root=/usr/lib/llvm-17 \ + --enable-llvm-link-shared \ + --set rust.thin-lto-import-instr-limit=10 + +COPY host-x86_64/x86_64-gnu-llvm-15/script.sh /tmp/ + +ENV SCRIPT /tmp/script.sh diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile index 85f2f84a4..82385ea15 100644 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile @@ -15,6 +15,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ sudo \ xz-utils \ tidy \ + libgccjit-12-dev \ \ # Install dependencies for chromium browser gconf-service \ @@ -61,6 +62,11 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ COPY scripts/sccache.sh /scripts/ RUN sh /scripts/sccache.sh +# Make `libgccjit.so` accessible. +RUN ln -s /usr/lib/gcc/x86_64-linux-gnu/12/libgccjit.so /usr/lib/x86_64-linux-gnu/libgccjit.so +# Fix rustc_codegen_gcc lto issues. +ENV GCC_EXEC_PREFIX="/usr/lib/gcc/" + COPY host-x86_64/x86_64-gnu-tools/checktools.sh /tmp/ RUN curl -sL https://nodejs.org/dist/v14.20.0/node-v14.20.0-linux-x64.tar.xz | tar -xJ @@ -81,7 +87,10 @@ RUN npm install -g browser-ui-test@$(head -n 1 /tmp/browser-ui-test.version) --u ENV RUST_CONFIGURE_ARGS \ --build=x86_64-unknown-linux-gnu \ - --save-toolstates=/tmp/toolstate/toolstates.json + --save-toolstates=/tmp/toolstate/toolstates.json \ + --enable-new-symbol-mangling + +ENV HOST_TARGET x86_64-unknown-linux-gnu ENV SCRIPT /tmp/checktools.sh ../x.py && \ NODE_PATH=`npm root -g` python3 ../x.py test tests/rustdoc-gui --stage 2 \ diff --git a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh index 7dde63709..821a09feb 100755 --- a/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh +++ b/src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh @@ -1,4 +1,5 @@ #!/bin/sh +# ignore-tidy-linelength set -eu @@ -26,8 +27,30 @@ python3 "$X_PY" test --stage 2 src/tools/clippy python3 "$X_PY" test --stage 2 src/tools/rustfmt python3 "$X_PY" test --stage 2 src/tools/miri # We natively run this script on x86_64-unknown-linux-gnu and x86_64-pc-windows-msvc. -# Also cover some other targets (on both of these hosts) via cross-testing. +# Also cover some other targets via cross-testing, in particular all tier 1 targets. export BOOTSTRAP_SKIP_TARGET_SANITY=1 # we don't need `cc` for these targets -python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc -python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin +case $HOST_TARGET in + x86_64-unknown-linux-gnu) + # Only this branch runs in PR CI. + # Fully test all main OSes, including a 32bit target. + python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-apple-darwin + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-msvc + # Only run "pass" tests for the remaining targets, which is quite a bit faster. + python3 "$X_PY" test --stage 2 src/tools/miri --target x86_64-pc-windows-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-unknown-linux-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-unknown-linux-gnu --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target s390x-unknown-linux-gnu --test-args pass + ;; + x86_64-pc-windows-msvc) + # Strangely, Linux targets do not work here. cargo always says + # "error: cannot produce cdylib for ... as the target ... does not support these crate types". + # Only run "pass" tests, which is quite a bit faster. + python3 "$X_PY" test --stage 2 src/tools/miri --target aarch64-apple-darwin --test-args pass + python3 "$X_PY" test --stage 2 src/tools/miri --target i686-pc-windows-gnu --test-args pass + ;; + *) + echo "FATAL: unexpected host $HOST_TARGET" + exit 1 + ;; +esac unset BOOTSTRAP_SKIP_TARGET_SANITY diff --git a/src/ci/docker/run.sh b/src/ci/docker/run.sh index e9c155b13..cedbc0390 100755 --- a/src/ci/docker/run.sh +++ b/src/ci/docker/run.sh @@ -235,7 +235,7 @@ else args="$args --volume /tmp/toolstate:/tmp/toolstate" id=$(id -u) - if [[ "$id" != 0 && "$(docker -v)" =~ ^podman ]]; then + if [[ "$id" != 0 && "$(docker version)" =~ Podman ]]; then # Rootless podman creates a separate user namespace, where an inner # LOCAL_USER_ID will map to a different subuid range on the host. # The "keep-id" mode maps the current UID directly into the container. @@ -264,10 +264,27 @@ else BASE_COMMIT="" fi +SUMMARY_FILE=github-summary.md +touch $objdir/${SUMMARY_FILE} + +extra_env="" +if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then + extra_env="$EXTRA_ENV --env ENABLE_GCC_CODEGEN=1" + # If `ENABLE_GCC_CODEGEN` is set and not empty, we add the `--enable-new-symbol-mangling` + # argument to `RUST_CONFIGURE_ARGS` and set the `GCC_EXEC_PREFIX` environment variable. + # `cg_gcc` doesn't support the legacy mangling so we need to enforce the new one + # if we run `cg_gcc` tests. + extra_env="$EXTRA_ENV --env USE_NEW_MANGLING=--enable-new-symbol-mangling" + # Fix rustc_codegen_gcc lto issues. + extra_env="$EXTRA_ENV --env GCC_EXEC_PREFIX=/usr/lib/gcc/" + echo "Setting extra environment values for docker: $extra_env" +fi + docker \ run \ --workdir /checkout/obj \ --env SRC=/checkout \ + $extra_env \ $args \ --env CARGO_HOME=/cargo \ --env DEPLOY \ @@ -275,6 +292,7 @@ docker \ --env CI \ --env GITHUB_ACTIONS \ --env GITHUB_REF \ + --env GITHUB_STEP_SUMMARY="/checkout/obj/${SUMMARY_FILE}" \ --env TOOLSTATE_REPO_ACCESS_TOKEN \ --env TOOLSTATE_REPO \ --env TOOLSTATE_PUBLISH \ @@ -284,11 +302,14 @@ docker \ --env DIST_TRY_BUILD \ --env PR_CI_JOB \ --env OBJDIR_ON_HOST="$objdir" \ + --env CODEGEN_BACKENDS \ --init \ --rm \ rust-ci \ $command +cat $objdir/${SUMMARY_FILE} >> "${GITHUB_STEP_SUMMARY}" + if [ -f /.dockerenv ]; then rm -rf $objdir docker cp checkout:/checkout/obj $objdir diff --git a/src/ci/docker/scripts/fuchsia-test-runner.py b/src/ci/docker/scripts/fuchsia-test-runner.py index f78d446c8..437b51641 100755 --- a/src/ci/docker/scripts/fuchsia-test-runner.py +++ b/src/ci/docker/scripts/fuchsia-test-runner.py @@ -12,6 +12,7 @@ from dataclasses import dataclass import fcntl import glob import hashlib +import io import json import os import platform @@ -276,27 +277,60 @@ class TestEnvironment: stderr=self.subprocess_output(), ) - # Start emulator - self.log_info("Starting emulator...") - product_bundle = "terminal.qemu-" + self.triple_to_arch(self.target) + # Look up the product bundle transfer manifest. + self.log_info("Looking up the product bundle transfer manifest...") + product_name = "minimal." + self.triple_to_arch(self.target) + fuchsia_version = "14.20230811.2.1" + + # FIXME: We should be able to replace this with the machine parsable + # `ffx --machine json product lookup ...` once F15 is released. + out = subprocess.check_output( + [ + ffx_path, + "product", + "lookup", + product_name, + fuchsia_version, + "--base-url", + "gs://fuchsia/development/" + fuchsia_version, + ], + env=ffx_env, + stderr=self.subprocess_output(), + ) + + self.log_debug(out) + + for line in io.BytesIO(out): + if line.startswith(b"gs://"): + transfer_manifest_url = line.rstrip() + break + else: + raise Exception("Unable to parse transfer manifest") + + # Download the product bundle. + product_bundle_dir = os.path.join(self.tmp_dir(), 'product-bundle') subprocess.check_call( [ ffx_path, - "product-bundle", - "get", - product_bundle, + "product", + "download", + transfer_manifest_url, + product_bundle_dir, + "--force", ], env=ffx_env, stdout=self.subprocess_output(), stderr=self.subprocess_output(), ) + + # Start emulator # FIXME: condition --accel hyper on target arch matching host arch subprocess.check_call( [ ffx_path, "emu", "start", - product_bundle, + product_bundle_dir, "--headless", "--log", self.emulator_log_path(), diff --git a/src/ci/github-actions/ci.yml b/src/ci/github-actions/ci.yml index 858ebf72a..da29ffb8e 100644 --- a/src/ci/github-actions/ci.yml +++ b/src/ci/github-actions/ci.yml @@ -91,6 +91,10 @@ x--expand-yaml-anchors--remove: os: macos-13 # We use the standard runner for now <<: *base-job + - &job-macos-m1 + os: macos-13-xlarge + <<: *base-job + - &job-windows-8c os: windows-2019-8core-32gb <<: *base-job @@ -153,6 +157,10 @@ x--expand-yaml-anchors--remove: run: src/ci/scripts/dump-environment.sh <<: *step + - name: install awscli + run: src/ci/scripts/install-awscli.sh + <<: *step + - name: install sccache run: src/ci/scripts/install-sccache.sh <<: *step @@ -165,6 +173,10 @@ x--expand-yaml-anchors--remove: run: src/ci/scripts/install-clang.sh <<: *step + - name: install tidy + run: src/ci/scripts/install-tidy.sh + <<: *step + - name: install WIX run: src/ci/scripts/install-wix.sh <<: *step @@ -281,6 +293,7 @@ on: - auto - try - try-perf + - automation/bors/try - master pull_request: branches: @@ -322,6 +335,8 @@ jobs: <<: *job-linux-4c - name: x86_64-gnu-llvm-15 + env: + ENABLE_GCC_CODEGEN: "1" <<: *job-linux-16c - name: x86_64-gnu-tools @@ -350,6 +365,8 @@ jobs: <<: *job-linux-8c - name: dist-aarch64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-8c - name: dist-android @@ -402,14 +419,19 @@ jobs: - &dist-x86_64-linux name: dist-x86_64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - name: dist-x86_64-linux-alt env: IMAGE: dist-x86_64-linux + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c - name: dist-x86_64-musl + env: + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-8c - name: dist-x86_64-netbsd @@ -427,20 +449,6 @@ jobs: - name: test-various <<: *job-linux-8c - - name: wasm32 - env: - # Running emscripten tests currently requires that we are - # building a nightly toolchain. Otherwise, we cannot pass - # -Zunstable-options to libtest. Normally we workaround this by - # setting RUSTC_BOOTSTRAP in the environment, but that doesn't - # work for emscripten as environment variables are not threaded - # into the compiled code. - # - # For more details see: - # https://emscripten.org/docs/porting/connecting_cpp_and_javascript/Interacting-with-code.html#environment-variables - RUST_CI_OVERRIDE_RELEASE_CHANNEL: nightly - <<: *job-linux-8c - - name: x86_64-gnu <<: *job-linux-4c @@ -468,6 +476,11 @@ jobs: - name: x86_64-gnu-distcheck <<: *job-linux-8c + - name: x86_64-gnu-llvm-17 + env: + RUST_BACKTRACE: 1 + <<: *job-linux-8c + - name: x86_64-gnu-llvm-16 env: RUST_BACKTRACE: 1 @@ -501,6 +514,7 @@ jobs: NO_DEBUG_ASSERTIONS: 1 NO_OVERFLOW_CHECKS: 1 DIST_REQUIRE_ALL_TOOLS: 1 + CODEGEN_BACKENDS: llvm,cranelift <<: *job-macos-xl - name: dist-apple-various @@ -536,17 +550,14 @@ jobs: # This target only needs to support 11.0 and up as nothing else supports the hardware - name: dist-aarch64-apple env: - SCRIPT: ./x.py dist bootstrap --include-default-paths --stage 2 + SCRIPT: ./x.py dist bootstrap --include-default-paths --host=aarch64-apple-darwin --target=aarch64-apple-darwin RUST_CONFIGURE_ARGS: >- - --build=x86_64-apple-darwin - --host=aarch64-apple-darwin - --target=aarch64-apple-darwin --enable-full-tools --enable-sanitizers --enable-profiler - --disable-docs --set rust.jemalloc --set llvm.ninja=false + --set rust.lto=thin RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 SELECT_XCODE: /Applications/Xcode_13.4.1.app USE_XCODE_CLANG: 1 @@ -556,15 +567,26 @@ jobs: NO_DEBUG_ASSERTIONS: 1 NO_OVERFLOW_CHECKS: 1 DIST_REQUIRE_ALL_TOOLS: 1 - # Corresponds to 16K page size - # - # Shouldn't be needed if jemalloc-sys is updated to - # handle this platform like iOS or if we build on - # aarch64-apple-darwin itself. - # - # https://github.com/gnzlbg/jemallocator/blob/c27a859e98e3cb790dc269773d9da71a1e918458/jemalloc-sys/build.rs#L237 - JEMALLOC_SYS_WITH_LG_PAGE: 14 - <<: *job-macos-xl + <<: *job-macos-m1 + + # This target only needs to support 11.0 and up as nothing else supports the hardware + - name: aarch64-apple + env: + SCRIPT: ./x.py --stage 2 test --host=aarch64-apple-darwin --target=aarch64-apple-darwin + RUST_CONFIGURE_ARGS: >- + --enable-sanitizers + --enable-profiler + --set rust.jemalloc + --set llvm.ninja=false + RUSTC_RETRY_LINKER_ON_SEGFAULT: 1 + SELECT_XCODE: /Applications/Xcode_13.4.1.app + USE_XCODE_CLANG: 1 + MACOSX_DEPLOYMENT_TARGET: 11.0 + MACOSX_STD_DEPLOYMENT_TARGET: 11.0 + NO_LLVM_ASSERTIONS: 1 + NO_DEBUG_ASSERTIONS: 1 + NO_OVERFLOW_CHECKS: 1 + <<: *job-macos-m1 ###################### # Windows Builders # @@ -585,6 +607,7 @@ jobs: - name: x86_64-msvc-ext env: SCRIPT: python x.py --stage 2 test src/tools/cargotest src/tools/cargo && src/ci/docker/host-x86_64/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows + HOST_TARGET: x86_64-pc-windows-msvc RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-lld --save-toolstates=/tmp/toolstate/toolstates.json DEPLOY_TOOLSTATES_JSON: toolstates-windows.json <<: *job-windows-8c @@ -702,12 +725,14 @@ jobs: env: DIST_TRY_BUILD: 1 <<: [*shared-ci-variables, *prod-variables] - if: github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust' + if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust')) strategy: matrix: include: - &dist-x86_64-linux name: dist-x86_64-linux + env: + CODEGEN_BACKENDS: llvm,cranelift <<: *job-linux-16c master: diff --git a/src/ci/run.sh b/src/ci/run.sh index 98f2cdac5..ce0dd6018 100755 --- a/src/ci/run.sh +++ b/src/ci/run.sh @@ -47,7 +47,8 @@ source "$ci_dir/shared.sh" export CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse -if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf; then +if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf || \ + isCiBranch automation/bors/try; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.metrics" HAS_METRICS=1 @@ -97,12 +98,14 @@ if [ "$DEPLOY$DEPLOY_ALT" = "1" ]; then if [ "$NO_LLVM_ASSERTIONS" = "1" ]; then RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --disable-llvm-assertions" elif [ "$DEPLOY_ALT" != "" ]; then - if [ "$NO_PARALLEL_COMPILER" = "" ]; then - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-compiler" + if [ "$ALT_PARALLEL_COMPILER" = "" ]; then + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.parallel-compiler=false" fi RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --enable-llvm-assertions" RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" fi + + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=${CODEGEN_BACKENDS:-llvm}" else # We almost always want debug assertions enabled, but sometimes this takes too # long for too little benefit, so we just turn them off. @@ -123,8 +126,15 @@ else RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.verify-llvm-ir" - # Test the Cranelift backend in on CI, but don't ship it. - RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" + # When running gcc backend tests, we need to install `libgccjit` and to not run llvm codegen + # tests as it will fail them. + if [[ "${ENABLE_GCC_CODEGEN}" == "1" ]]; then + # Test the Cranelift and GCC backends in CI. Bootstrap knows which targets to run tests on. + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift,gcc" + else + # Test the Cranelift backend in CI. Bootstrap knows which targets to run tests on. + RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.codegen-backends=llvm,cranelift" + fi # We enable this for non-dist builders, since those aren't trying to produce # fresh binaries. We currently don't entirely support distributing a fresh diff --git a/src/ci/scripts/install-awscli.sh b/src/ci/scripts/install-awscli.sh new file mode 100755 index 000000000..b4a239fd3 --- /dev/null +++ b/src/ci/scripts/install-awscli.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# This script downloads and installs the awscli binaries directly from +# Amazon. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +AWS_VERSION="2.13.25" + +# Only the macOS arm64/aarch64 GitHub Actions runner needs to have AWS +# installed; other platforms have it preinstalled. + +if isMacOS; then + platform=$(uname -m) + case $platform in + x86_64) + ;; + arm64) + file="https://awscli.amazonaws.com/AWSCLIV2-${AWS_VERSION}.pkg" + retry curl -f "${file}" -o "AWSCLIV2.pkg" + sudo installer -pkg "AWSCLIV2.pkg" -target / + ;; + *) + echo "unsupported architecture: ${platform}" + exit 1 + esac +fi diff --git a/src/ci/scripts/install-tidy.sh b/src/ci/scripts/install-tidy.sh new file mode 100755 index 000000000..fab126453 --- /dev/null +++ b/src/ci/scripts/install-tidy.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# This script downloads and installs the tidy binary from Homebrew. + +set -euo pipefail +IFS=$'\n\t' + +source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" + +# Only the macOS arm64/aarch64 GitHub Actions runner needs to have tidy +# installed; other platforms have it preinstalled. + +if isMacOS; then + platform=$(uname -m) + case $platform in + x86_64) + ;; + arm64) + brew install tidy-html5 + ;; + *) + echo "unsupported architecture: ${platform}" + exit 1 + esac +fi diff --git a/src/ci/scripts/verify-channel.sh b/src/ci/scripts/verify-channel.sh index cd28748a4..edeea2014 100755 --- a/src/ci/scripts/verify-channel.sh +++ b/src/ci/scripts/verify-channel.sh @@ -8,7 +8,7 @@ IFS=$'\n\t' source "$(cd "$(dirname "$0")" && pwd)/../shared.sh" -if isCiBranch auto || isCiBranch try || isCiBranch try-perf; then +if isCiBranch auto || isCiBranch try || isCiBranch try-perf || isCiBranch automation/bors/try; then echo "channel verification is only executed on PR builds" exit fi diff --git a/src/doc/book/redirects/compiler-plugins.md b/src/doc/book/redirects/compiler-plugins.md index 66061adf5..67187f5f6 100644 --- a/src/doc/book/redirects/compiler-plugins.md +++ b/src/doc/book/redirects/compiler-plugins.md @@ -2,12 +2,5 @@ There is a new edition of the book and this is an old link. -> Compiler plugins are user-provided libraries that extend the compiler's behavior with new syntax extensions, lint checks, etc. - ---- - -This particular chapter has moved to [the Unstable Book][2]. - -* **[In the Unstable Rust Book: `plugin`][2]** - -[2]: ../unstable-book/language-features/plugin.html +> Compiler plugins were user-provided libraries that extended the compiler's behavior in certain ways. +> Support for them has been removed. diff --git a/src/doc/book/src/ch02-00-guessing-game-tutorial.md b/src/doc/book/src/ch02-00-guessing-game-tutorial.md index 4f2857333..5e27fb114 100644 --- a/src/doc/book/src/ch02-00-guessing-game-tutorial.md +++ b/src/doc/book/src/ch02-00-guessing-game-tutorial.md @@ -930,8 +930,8 @@ discusses structs and method syntax, and Chapter 6 explains how enums work. [randcrate]: https://crates.io/crates/rand [semver]: http://semver.org [cratesio]: https://crates.io/ -[doccargo]: http://doc.crates.io -[doccratesio]: http://doc.crates.io/crates-io.html +[doccargo]: https://doc.rust-lang.org/cargo/ +[doccratesio]: https://doc.rust-lang.org/cargo/reference/publishing.html [match]: ch06-02-match.html [shadowing]: ch03-01-variables-and-mutability.html#shadowing [parse]: ../std/primitive.str.html#method.parse diff --git a/src/doc/embedded-book/src/start/hardware.md b/src/doc/embedded-book/src/start/hardware.md index 8166e62e5..ba9fe6e76 100644 --- a/src/doc/embedded-book/src/start/hardware.md +++ b/src/doc/embedded-book/src/start/hardware.md @@ -64,6 +64,9 @@ target = "thumbv7em-none-eabihf" # Cortex-M4F and Cortex-M7F (with FPU) ``` We'll use `thumbv7em-none-eabihf` as that covers the Cortex-M4F core. +> **NOTE**: As you may remember from the previous chapter, we have to install +> all targets and this is a new one. So don't forget to run the installation +> process `rustup target add thumbv7em-none-eabihf` for this target. The second step is to enter the memory region information into the `memory.x` file. diff --git a/src/doc/guide-plugins.md b/src/doc/guide-plugins.md index 6c5115487..ccb9bb2ee 100644 --- a/src/doc/guide-plugins.md +++ b/src/doc/guide-plugins.md @@ -1,4 +1,3 @@ % The (old) Rust Compiler Plugins Guide -This content has moved into -[the Unstable Book](unstable-book/language-features/plugin.html). +Support for plugins has been removed. diff --git a/src/doc/nomicon/src/exception-safety.md b/src/doc/nomicon/src/exception-safety.md index 8404bb859..762b38b98 100644 --- a/src/doc/nomicon/src/exception-safety.md +++ b/src/doc/nomicon/src/exception-safety.md @@ -172,7 +172,7 @@ impl<'a, T> Hole<'a, T> { fn removed(&self) -> &T { self.elt.as_ref().unwrap() } - unsafe fn get(&self, index: usize) -> &T { &self.data[index] } + fn get(&self, index: usize) -> &T { &self.data[index] } unsafe fn move_to(&mut self, index: usize) { let index_ptr: *const _ = &self.data[index]; diff --git a/src/doc/reference/src/attributes.md b/src/doc/reference/src/attributes.md index 92ce1cd09..a1ad5c60c 100644 --- a/src/doc/reference/src/attributes.md +++ b/src/doc/reference/src/attributes.md @@ -275,8 +275,8 @@ The following is an index of all built-in attributes. - [`debugger_visualizer`] — Embeds a file that specifies debugger output for a type. [Doc comments]: comments.md#doc-comments -[ECMA-334]: https://www.ecma-international.org/publications/standards/Ecma-334.htm -[ECMA-335]: https://www.ecma-international.org/publications/standards/Ecma-335.htm +[ECMA-334]: https://www.ecma-international.org/publications-and-standards/standards/ecma-334/ +[ECMA-335]: https://www.ecma-international.org/publications-and-standards/standards/ecma-335/ [Expression Attributes]: expressions.md#expression-attributes [IDENTIFIER]: identifiers.md [RAW_STRING_LITERAL]: tokens.md#raw-string-literals diff --git a/src/doc/reference/src/attributes/codegen.md b/src/doc/reference/src/attributes/codegen.md index c929f979c..7a50fe26f 100644 --- a/src/doc/reference/src/attributes/codegen.md +++ b/src/doc/reference/src/attributes/codegen.md @@ -204,6 +204,66 @@ Feature | Implicitly Enables | Feature Name `tme` | | FEAT_TME - Transactional Memory Extension `vh` | | FEAT_VHE - Virtualization Host Extensions +#### `riscv32` or `riscv64` + +This platform requires that `#[target_feature]` is only applied to [`unsafe` +functions][unsafe function]. + +Further documentation on these features can be found in their respective +specification. Many specifications are described in the [RISC-V ISA Manual] or +in another manual hosted on the [RISC-V GitHub Account]. + +[RISC-V ISA Manual]: https://github.com/riscv/riscv-isa-manual +[RISC-V GitHub Account]: https://github.com/riscv + +Feature | Implicitly Enables | Description +------------|---------------------|------------------- +`a` | | [A][rv-a] — Atomic instructions +`c` | | [C][rv-c] — Compressed instructions +`m` | | [M][rv-m] — Integer Multiplication and Division instructions +`zb` | `zba`, `zbc`, `zbs` | [Zb][rv-zb] — Bit Manipulation instructions +`zba` | | [Zba][rv-zb-zba] — Address Generation instructions +`zbb` | | [Zbb][rv-zb-zbb] — Basic bit-manipulation +`zbc` | | [Zbc][rv-zb-zbc] — Carry-less multiplication +`zbkb` | | [Zbkb][rv-zb-zbkb] — Bit Manipulation Instructions for Cryptography +`zbkc` | | [Zbkc][rv-zb-zbc] — Carry-less multiplication for Cryptography +`zbkx` | | [Zbkx][rv-zb-zbkx] — Crossbar permutations +`zbs` | | [Zbs][rv-zb-zbs] — Single-bit instructions +`zk` | `zkn`, `zkr`, `zks`, `zkt`, `zbkb`, `zbkc`, `zkbx` | [Zk][rv-zk] — Scalar Cryptography +`zkn` | `zknd`, `zkne`, `zknh`, `zbkb`, `zbkc`, `zkbx` | [Zkn][rv-zkn] — NIST Algorithm suite extension +`zknd` | | [Zknd][rv-zknd] — NIST Suite: AES Decryption +`zkne` | | [Zkne][rv-zkne] — NIST Suite: AES Encryption +`zknh` | | [Zknh][rv-zknh] — NIST Suite: Hash Function Instructions +`zkr` | | [Zkr][rv-zkr] — Entropy Source Extension +`zks` | `zksed`, `zksh`, `zbkb`, `zbkc`, `zkbx` | [Zks][rv-zks] — ShangMi Algorithm Suite +`zksed` | | [Zksed][rv-zksed] — ShangMi Suite: SM4 Block Cipher Instructions +`zksh` | | [Zksh][rv-zksh] — ShangMi Suite: SM3 Hash Function Instructions +`zkt` | | [Zkt][rv-zkt] — Data Independent Execution Latency Subset + + + +[rv-a]: https://github.com/riscv/riscv-isa-manual/blob/de46343a245c6ee1f7b1a40c92fe1a86bd4f4978/src/a-st-ext.adoc +[rv-c]: https://github.com/riscv/riscv-isa-manual/blob/de46343a245c6ee1f7b1a40c92fe1a86bd4f4978/src/c-st-ext.adoc +[rv-m]: https://github.com/riscv/riscv-isa-manual/blob/de46343a245c6ee1f7b1a40c92fe1a86bd4f4978/src/m-st-ext.adoc +[rv-zb]: https://github.com/riscv/riscv-bitmanip +[rv-zb-zba]: https://github.com/riscv/riscv-bitmanip/blob/main/bitmanip/zba.adoc +[rv-zb-zbb]: https://github.com/riscv/riscv-bitmanip/blob/main/bitmanip/zbb.adoc +[rv-zb-zbc]: https://github.com/riscv/riscv-bitmanip/blob/main/bitmanip/zbc.adoc +[rv-zb-zbkb]: https://github.com/riscv/riscv-bitmanip/blob/main/bitmanip/zbkb.adoc +[rv-zb-zbkc]: https://github.com/riscv/riscv-bitmanip/blob/main/bitmanip/zbkc.adoc +[rv-zb-zbkx]: https://github.com/riscv/riscv-bitmanip/blob/main/bitmanip/zbkx.adoc +[rv-zb-zbs]: https://github.com/riscv/riscv-bitmanip/blob/main/bitmanip/zbs.adoc +[rv-zk]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zk.adoc +[rv-zkn]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zkn.adoc +[rv-zkne]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zkne.adoc +[rv-zknd]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zknd.adoc +[rv-zknh]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zknh.adoc +[rv-zkr]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zkr.adoc +[rv-zks]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zks.adoc +[rv-zksed]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zksed.adoc +[rv-zksh]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zksh.adoc +[rv-zkt]: https://github.com/riscv/riscv-crypto/blob/e2dd7d98b7f34d477e38cb5fd7a3af4379525189/doc/scalar/riscv-crypto-scalar-zkr.adoc + #### `wasm32` or `wasm64` `#[target_feature]` may be used with both safe and diff --git a/src/doc/reference/src/behavior-considered-undefined.md b/src/doc/reference/src/behavior-considered-undefined.md index f140fb20c..756b86db0 100644 --- a/src/doc/reference/src/behavior-considered-undefined.md +++ b/src/doc/reference/src/behavior-considered-undefined.md @@ -27,9 +27,12 @@ Please read the [Rustonomicon] before writing unsafe code. * Data races. -* Evaluating a [dereference expression] (`*expr`) on a raw pointer that is - [dangling] or unaligned, even in [place expression context] - (e.g. `addr_of!(*expr)`). +* Accessing (loading from or storing to) a place that is [dangling] or [based on + a misaligned pointer]. +* Performing a place projection that violates the requirements of [in-bounds + pointer arithmetic][offset]. A place projection is a [field + expression][project-field], a [tuple index expression][project-tuple], or an + [array/slice index expression][project-slice]. * Breaking the [pointer aliasing rules]. `Box`, `&mut T` and `&T` follow LLVM’s scoped [noalias] model, except if the `&T` contains an [`UnsafeCell`]. References and boxes must not be [dangling] while they are @@ -68,7 +71,7 @@ Please read the [Rustonomicon] before writing unsafe code. * A `!` (all values are invalid for this type). * An integer (`i*`/`u*`), floating point value (`f*`), or raw pointer obtained from [uninitialized memory][undef], or uninitialized memory in a `str`. - * A reference or `Box` that is [dangling], unaligned, or points to an invalid value. + * A reference or `Box` that is [dangling], misaligned, or points to an invalid value. * Invalid metadata in a wide reference, `Box`, or raw pointer: * `dyn Trait` metadata is invalid if it is not a pointer to a vtable for `Trait` that matches the actual dynamic trait the pointer or reference points to. @@ -102,6 +105,36 @@ reading uninitialized memory is permitted are inside `union`s and in "padding" The span of bytes a pointer or reference "points to" is determined by the pointer value and the size of the pointee type (using `size_of_val`). +### Places based on misaligned pointers +[based on a misaligned pointer]: #places-based-on-misaligned-pointers + +A place is said to be "based on a misaligned pointer" if the last `*` projection +during place computation was performed on a pointer that was not aligned for its +type. (If there is no `*` projection in the place expression, then this is +accessing the field of a local and rustc will guarantee proper alignment. If +there are multiple `*` projection, then each of them incurs a load of the +pointer-to-be-dereferenced itself from memory, and each of these loads is +subject to the alignment constraint. Note that some `*` projections can be +omitted in surface Rust syntax due to automatic dereferencing; we are +considering the fully expanded place expression here.) + +For instance, if `ptr` has type `*const S` where `S` has an alignment of 8, then +`ptr` must be 8-aligned or else `(*ptr).f` is "based on an misaligned pointer". +This is true even if the type of the field `f` is `u8` (i.e., a type with +alignment 1). In other words, the alignment requirement derives from the type of +the pointer that was dereferenced, *not* the type of the field that is being +accessed. + +Note that a place based on a misaligned pointer only leads to Undefined Behavior +when it is loaded from or stored to. `addr_of!`/`addr_of_mut!` on such a place +is allowed. `&`/`&mut` on a place requires the alignment of the field type (or +else the program would be "producing an invalid value"), which generally is a +less restrictive requirement than being based on an aligned pointer. Taking a +reference will lead to a compiler error in cases where the field type might be +more aligned than the type that contains it, i.e., `repr(packed)`. This means +that being based on an aligned pointer is always sufficient to ensure that the +new reference is aligned, but it is not always necessary. + ### Dangling pointers [dangling]: #dangling-pointers @@ -128,8 +161,11 @@ must never exceed `isize::MAX`. [Rustonomicon]: ../nomicon/index.html [`NonNull`]: ../core/ptr/struct.NonNull.html [`NonZero*`]: ../core/num/index.html -[dereference expression]: expressions/operator-expr.md#the-dereference-operator [place expression context]: expressions.md#place-expressions-and-value-expressions [rules]: inline-assembly.md#rules-for-inline-assembly [points to]: #pointed-to-bytes [pointed to]: #pointed-to-bytes +[offset]: ../std/primitive.pointer.html#method.offset +[project-field]: expressions/field-expr.md +[project-tuple]: expressions/tuple-expr.md#tuple-indexing-expressions +[project-slice]: expressions/array-expr.md#array-and-slice-indexing-expressions diff --git a/src/doc/reference/src/destructors.md b/src/doc/reference/src/destructors.md index 17afc3676..9c426426c 100644 --- a/src/doc/reference/src/destructors.md +++ b/src/doc/reference/src/destructors.md @@ -156,7 +156,7 @@ temporary variable that holds the result of that expression when used in a Apart from lifetime extension, the temporary scope of an expression is the smallest scope that contains the expression and is one of the following: -* The entire function body. +* The entire function. * A statement. * The body of an [`if`], [`while`] or [`loop`] expression. * The `else` block of an `if` expression. @@ -168,8 +168,8 @@ smallest scope that contains the expression and is one of the following: > **Notes**: > > Temporaries that are created in the final expression of a function -> body are dropped *after* any named variables bound in the function body, as -> there is no smaller enclosing temporary scope. +> body are dropped *after* any named variables bound in the function body. +> Their drop scope is the entire function, as there is no smaller enclosing temporary scope. > > The [scrutinee] of a `match` expression is not a temporary scope, so > temporaries in the scrutinee can be dropped after the `match` expression. For diff --git a/src/doc/reference/src/expressions/operator-expr.md b/src/doc/reference/src/expressions/operator-expr.md index 8b6429636..bd4998af5 100644 --- a/src/doc/reference/src/expressions/operator-expr.md +++ b/src/doc/reference/src/expressions/operator-expr.md @@ -478,6 +478,16 @@ unsafe { assert_eq!(values[1], 3); ``` +#### Slice DST pointer to pointer cast + +For slice types like `[T]` and `[U]`, the raw pointer types `*const [T]`, `*mut [T]`, +`*const [U]`, and `*mut [U]` encode the number of elements in this slice. Casts between +these raw pointer types preserve the number of elements. Note that, as a consequence, +such casts do *not* necessarily preserve the size of the pointer's referent (e.g., +casting `*const [u16]` to `*const [u8]` will result in a raw pointer which refers to an +object of half the size of the original). The same holds for `str` and any compound type +whose unsized tail is a slice type, such as struct `Foo(i32, [u8])` or `(u64, Foo)`. + ## Assignment expressions > **Syntax**\ diff --git a/src/doc/reference/src/inline-assembly.md b/src/doc/reference/src/inline-assembly.md index 26f1acedc..e905fcb26 100644 --- a/src/doc/reference/src/inline-assembly.md +++ b/src/doc/reference/src/inline-assembly.md @@ -414,10 +414,13 @@ Flags are used to further influence the behavior of the inline assembly block. Currently the following options are defined: - `pure`: The `asm!` block has no side effects, and its outputs depend only on its direct inputs (i.e. the values themselves, not what they point to) or values read from memory (unless the `nomem` options is also set). This allows the compiler to execute the `asm!` block fewer times than specified in the program (e.g. by hoisting it out of a loop) or even eliminate it entirely if the outputs are not used. + The `pure` option must be combined with either the `nomem` or `readonly` options, otherwise a compile-time error is emitted. - `nomem`: The `asm!` blocks does not read or write to any memory. This allows the compiler to cache the values of modified global variables in registers across the `asm!` block since it knows that they are not read or written to by the `asm!`. + The compiler also assumes that this `asm!` block does not perform any kind of synchronization with other threads, e.g. via fences. - `readonly`: The `asm!` block does not write to any memory. This allows the compiler to cache the values of unmodified global variables in registers across the `asm!` block since it knows that they are not written to by the `asm!`. + The compiler also assumes that this `asm!` block does not perform any kind of synchronization with other threads, e.g. via fences. - `preserves_flags`: The `asm!` block does not modify the flags register (defined in the rules below). This allows the compiler to avoid recomputing the condition flags after the `asm!` block. - `noreturn`: The `asm!` block never returns, and its return type is defined as `!` (never). @@ -432,7 +435,6 @@ Currently the following options are defined: The compiler performs some additional checks on options: - The `nomem` and `readonly` options are mutually exclusive: it is a compile-time error to specify both. -- The `pure` option must be combined with either the `nomem` or `readonly` options, otherwise a compile-time error is emitted. - It is a compile-time error to specify `pure` on an asm block with no outputs or only discarded outputs (`_`). - It is a compile-time error to specify `noreturn` on an asm block with outputs. diff --git a/src/doc/reference/src/items/traits.md b/src/doc/reference/src/items/traits.md index 3c5d31d5c..828a8a35f 100644 --- a/src/doc/reference/src/items/traits.md +++ b/src/doc/reference/src/items/traits.md @@ -43,7 +43,7 @@ trait Example { } ``` -Trait functions are not allowed to be [`async`] or [`const`]. +Trait functions are not allowed to be [`const`]. ## Trait bounds diff --git a/src/doc/reference/src/types/impl-trait.md b/src/doc/reference/src/types/impl-trait.md index af900408e..b46c209b7 100644 --- a/src/doc/reference/src/types/impl-trait.md +++ b/src/doc/reference/src/types/impl-trait.md @@ -88,6 +88,12 @@ which also avoids the drawbacks of using a boxed trait object. Similarly, the concrete types of iterators could become very complex, incorporating the types of all previous iterators in a chain. Returning `impl Iterator` means that a function only exposes the `Iterator` trait as a bound on its return type, instead of explicitly specifying all of the other iterator types involved. +## Return-position `impl Trait` in traits and trait implementations + +Functions in traits may also use `impl Trait` as a syntax for an anonymous associated type. + +Every `impl Trait` in the return type of an associated function in a trait is desugared to an anonymous associated type. The return type that appears in the implementation's function signature is used to determine the value of the associated type. + ### Differences between generics and `impl Trait` in return position In argument position, `impl Trait` is very similar in semantics to a generic type parameter. @@ -121,8 +127,8 @@ Instead, the function chooses the return type, but only promises that it will im ## Limitations -`impl Trait` can only appear as a parameter or return type of a free or inherent function. -It cannot appear inside implementations of traits, nor can it be the type of a let binding or appear inside a type alias. +`impl Trait` can only appear as a parameter or return type of a non-`extern` function. +It cannot be the type of a `let` binding, field type, or appear inside a type alias. [closures]: closure.md [_GenericArgs_]: ../paths.md#paths-in-expressions diff --git a/src/doc/reference/src/types/textual.md b/src/doc/reference/src/types/textual.md index 41ed35ea1..fcbec0823 100644 --- a/src/doc/reference/src/types/textual.md +++ b/src/doc/reference/src/types/textual.md @@ -17,7 +17,9 @@ is valid UTF-8. Calling a `str` method with a non-UTF-8 buffer can cause Since `str` is a [dynamically sized type], it can only be instantiated through a pointer type, such as `&str`. -## Bit validity +## Layout and bit validity + +`char` is guaranteed to have the same size and alignment as `u32` on all platforms. Every byte of a `char` is guaranteed to be initialized (in other words, `transmute::()]>(...)` is always sound -- but since diff --git a/src/doc/rust-by-example/src/SUMMARY.md b/src/doc/rust-by-example/src/SUMMARY.md index b1bcf223d..b8e6ada91 100644 --- a/src/doc/rust-by-example/src/SUMMARY.md +++ b/src/doc/rust-by-example/src/SUMMARY.md @@ -161,7 +161,7 @@ - [Unpacking options with `?`](error/option_unwrap/question_mark.md) - [Combinators: `map`](error/option_unwrap/map.md) - [Combinators: `and_then`](error/option_unwrap/and_then.md) - - [Defaults: `or`, `or_else`, `get_or_insert`, 'get_or_insert_with`](error/option_unwrap/defaults.md) + - [Defaults: `or`, `or_else`, `get_or_insert`, `get_or_insert_with`](error/option_unwrap/defaults.md) - [`Result`](error/result.md) - [`map` for `Result`](error/result/result_map.md) - [aliases for `Result`](error/result/result_alias.md) @@ -197,7 +197,7 @@ - [File I/O](std_misc/file.md) - [`open`](std_misc/file/open.md) - [`create`](std_misc/file/create.md) - - [`read lines`](std_misc/file/read_lines.md) + - [`read_lines`](std_misc/file/read_lines.md) - [Child processes](std_misc/process.md) - [Pipes](std_misc/process/pipe.md) - [Wait](std_misc/process/wait.md) diff --git a/src/doc/rust-by-example/src/attribute.md b/src/doc/rust-by-example/src/attribute.md index dcb7d6c1a..d6a49dc01 100644 --- a/src/doc/rust-by-example/src/attribute.md +++ b/src/doc/rust-by-example/src/attribute.md @@ -14,9 +14,34 @@ can be used to/for: * mark functions that will be part of a benchmark * [attribute like macros][macros] -When attributes apply to a whole crate, their syntax is `#![crate_attribute]`, -and when they apply to a module or item, the syntax is `#[item_attribute]` -(notice the missing bang `!`). +Attributes look like `#[outer_attribute]` or `#![inner_attribute]`, +with the difference between them being where they apply. + +- `#[outer_attribute]` applies to the [item][item] immediately + following it. Some examples of items are: a function, a module + declaration, a constant, a structure, an enum. Here is an example + where attribute `#[derive(Debug)]` applies to the struct + `Rectangle`: + ```rust + #[derive(Debug)] + struct Rectangle { + width: u32, + height: u32, + } + ``` + +- `#![inner_attribute]` applies to the enclosing [item][item] (typically a + module or a crate). In other words, this attribute is intepreted as + applying to the entire scope in which it's place. Here is an example + where `#![allow(unusude_variables)]` applies to the whole crate (if + placed in `main.rs`): + ```rust + #![allow(unused_variables)] + + fn main() { + let x = 3; // This would normally warn about an unused variable. + } + ``` Attributes can take arguments with different syntaxes: @@ -36,5 +61,6 @@ Attributes can have multiple values and can be separated over multiple lines, to [cfg]: attribute/cfg.md [crate]: attribute/crate.md +[item]: https://doc.rust-lang.org/stable/reference/items.html [lint]: https://en.wikipedia.org/wiki/Lint_%28software%29 [macros]: https://doc.rust-lang.org/book/ch19-06-macros.html#attribute-like-macros diff --git a/src/doc/rust-by-example/src/custom_types/constants.md b/src/doc/rust-by-example/src/custom_types/constants.md index 8878ba834..c060db734 100644 --- a/src/doc/rust-by-example/src/custom_types/constants.md +++ b/src/doc/rust-by-example/src/custom_types/constants.md @@ -4,7 +4,7 @@ Rust has two different types of constants which can be declared in any scope including global. Both require explicit type annotation: * `const`: An unchangeable value (the common case). -* `static`: A possibly `mut`able variable with [`'static`][static] lifetime. +* `static`: A possibly mutable variable with [`'static`][static] lifetime. The static lifetime is inferred and does not have to be specified. Accessing or modifying a mutable static variable is [`unsafe`][unsafe]. diff --git a/src/doc/rust-by-example/src/error/option_unwrap/question_mark.md b/src/doc/rust-by-example/src/error/option_unwrap/question_mark.md index 831625229..04de93cc7 100644 --- a/src/doc/rust-by-example/src/error/option_unwrap/question_mark.md +++ b/src/doc/rust-by-example/src/error/option_unwrap/question_mark.md @@ -8,7 +8,8 @@ function is being executed and return `None`. ```rust,editable fn next_birthday(current_age: Option) -> Option { // If `current_age` is `None`, this returns `None`. - // If `current_age` is `Some`, the inner `u8` gets assigned to `next_age` + // If `current_age` is `Some`, the inner `u8` value + 1 + // gets assigned to `next_age` let next_age: u8 = current_age? + 1; Some(format!("Next year I will be {}", next_age)) } diff --git a/src/doc/rust-by-example/src/flow_control/while_let.md b/src/doc/rust-by-example/src/flow_control/while_let.md index 897375a8e..745b7ae75 100644 --- a/src/doc/rust-by-example/src/flow_control/while_let.md +++ b/src/doc/rust-by-example/src/flow_control/while_let.md @@ -31,26 +31,24 @@ loop { Using `while let` makes this sequence much nicer: ```rust,editable -fn main() { - // Make `optional` of type `Option` - let mut optional = Some(0); - - // This reads: "while `let` destructures `optional` into - // `Some(i)`, evaluate the block (`{}`). Else `break`. - while let Some(i) = optional { - if i > 9 { - println!("Greater than 9, quit!"); - optional = None; - } else { - println!("`i` is `{:?}`. Try again.", i); - optional = Some(i + 1); - } - // ^ Less rightward drift and doesn't require - // explicitly handling the failing case. +// Make `optional` of type `Option` +let mut optional = Some(0); + +// This reads: "while `let` destructures `optional` into +// `Some(i)`, evaluate the block (`{}`). Else `break`. +while let Some(i) = optional { + if i > 9 { + println!("Greater than 9, quit!"); + optional = None; + } else { + println!("`i` is `{:?}`. Try again.", i); + optional = Some(i + 1); } - // ^ `if let` had additional optional `else`/`else if` - // clauses. `while let` does not have these. + // ^ Less rightward drift and doesn't require + // explicitly handling the failing case. } +// ^ `if let` had additional optional `else`/`else if` +// clauses. `while let` does not have these. ``` ### See also: diff --git a/src/doc/rust-by-example/src/fn/closures.md b/src/doc/rust-by-example/src/fn/closures.md index 82286003b..e7b8c9867 100644 --- a/src/doc/rust-by-example/src/fn/closures.md +++ b/src/doc/rust-by-example/src/fn/closures.md @@ -14,7 +14,7 @@ variable names *must* be specified. Other characteristics of closures include: * using `||` instead of `()` around input variables. -* optional body delimination (`{}`) for a single expression (mandatory otherwise). +* optional body delimitation (`{}`) for a single expression (mandatory otherwise). * the ability to capture the outer environment variables. ```rust,editable @@ -26,7 +26,7 @@ fn main() { // TODO: uncomment the line above and see the compiler error. The compiler // suggests that we define a closure instead. - // Closures are anonymous, here we are binding them to references + // Closures are anonymous, here we are binding them to references. // Annotation is identical to function annotation but is optional // as are the `{}` wrapping the body. These nameless functions // are assigned to appropriately named variables. diff --git a/src/doc/rust-by-example/src/fn/hof.md b/src/doc/rust-by-example/src/fn/hof.md index 88918cb52..9be5b41fe 100644 --- a/src/doc/rust-by-example/src/fn/hof.md +++ b/src/doc/rust-by-example/src/fn/hof.md @@ -10,7 +10,7 @@ fn is_odd(n: u32) -> bool { } fn main() { - println!("Find the sum of all the squared odd numbers under 1000"); + println!("Find the sum of all the numbers with odd squares under 1000"); let upper = 1000; // Imperative approach diff --git a/src/doc/rust-by-example/src/meta/doc.md b/src/doc/rust-by-example/src/meta/doc.md index e9e51186f..b1732f837 100644 --- a/src/doc/rust-by-example/src/meta/doc.md +++ b/src/doc/rust-by-example/src/meta/doc.md @@ -2,7 +2,8 @@ Use `cargo doc` to build documentation in `target/doc`. -Use `cargo test` to run all tests (including documentation tests), and `cargo test --doc` to only run documentation tests. +Use `cargo test` to run all tests (including documentation tests), and `cargo +test --doc` to only run documentation tests. These commands will appropriately invoke `rustdoc` (and `rustc`) as required. @@ -67,7 +68,8 @@ $ rustdoc --test --extern doc="libdoc.rlib" doc.rs ## Doc attributes -Below are a few examples of the most common `#[doc]` attributes used with `rustdoc`. +Below are a few examples of the most common `#[doc]` attributes used with +`rustdoc`. ### `inline` @@ -104,7 +106,8 @@ Using this tells `rustdoc` not to include this in documentation: pub use self::async_await::*; ``` -For documentation, `rustdoc` is widely used by the community. It's what is used to generate the [std library docs](https://doc.rust-lang.org/std/). +For documentation, `rustdoc` is widely used by the community. It's what is used +to generate the [std library docs](https://doc.rust-lang.org/std/). ### See also: diff --git a/src/doc/rust-by-example/src/meta/playground.md b/src/doc/rust-by-example/src/meta/playground.md index 7fcfad1a7..e78552d29 100644 --- a/src/doc/rust-by-example/src/meta/playground.md +++ b/src/doc/rust-by-example/src/meta/playground.md @@ -1,6 +1,7 @@ # Playground -The [Rust Playground](https://play.rust-lang.org/) is a way to experiment with Rust code through a web interface. +The [Rust Playground](https://play.rust-lang.org/) is a way to experiment with +Rust code through a web interface. ## Using it with `mdbook` @@ -12,7 +13,9 @@ fn main() { } ``` -This allows the reader to both run your code sample, but also modify and tweak it. The key here is the adding the word `editable` to your codefence block separated by a comma. +This allows the reader to both run your code sample, but also modify and tweak +it. The key here is the adding the word `editable` to your codefence block +separated by a comma. ````markdown ```rust,editable @@ -20,7 +23,8 @@ This allows the reader to both run your code sample, but also modify and tweak i ``` ```` -Additionally, you can add `ignore` if you want `mdbook` to skip your code when it builds and tests. +Additionally, you can add `ignore` if you want `mdbook` to skip your code when +it builds and tests. ````markdown ```rust,editable,ignore @@ -30,7 +34,10 @@ Additionally, you can add `ignore` if you want `mdbook` to skip your code when i ## Using it with docs -You may have noticed in some of the [official Rust docs][official-rust-docs] a button that says "Run", which opens the code sample up in a new tab in Rust Playground. This feature is enabled if you use the #[doc] attribute called [`html_playground_url`][html-playground-url]. +You may have noticed in some of the [official Rust docs][official-rust-docs] a +button that says "Run", which opens the code sample up in a new tab in Rust +Playground. This feature is enabled if you use the `#[doc]` attribute called +[`html_playground_url`][html-playground-url]. ### See also: diff --git a/src/doc/rust-by-example/src/primitives/array.md b/src/doc/rust-by-example/src/primitives/array.md index 5f5e69944..704a2131c 100644 --- a/src/doc/rust-by-example/src/primitives/array.md +++ b/src/doc/rust-by-example/src/primitives/array.md @@ -6,7 +6,7 @@ at compile time, is part of their type signature `[T; length]`. Slices are similar to arrays, but their length is not known at compile time. Instead, a slice is a two-word object; the first word is a pointer to the data, -the second word the length of the slice. The word size is the same as usize, +the second word is the length of the slice. The word size is the same as usize, determined by the processor architecture, e.g. 64 bits on an x86-64. Slices can be used to borrow a section of an array and have the type signature `&[T]`. diff --git a/src/doc/rust-by-example/src/scope/lifetime.md b/src/doc/rust-by-example/src/scope/lifetime.md index 68b42d380..01c4bf405 100644 --- a/src/doc/rust-by-example/src/scope/lifetime.md +++ b/src/doc/rust-by-example/src/scope/lifetime.md @@ -1,6 +1,6 @@ # Lifetimes -A *lifetime* is a construct of the compiler (or more specifically, its *borrow +A *lifetime* is a construct the compiler (or more specifically, its *borrow checker*) uses to ensure all borrows are valid. Specifically, a variable's lifetime begins when it is created and ends when it is destroyed. While lifetimes and scopes are often referred to together, they are not the same. diff --git a/src/doc/rust-by-example/src/scope/move.md b/src/doc/rust-by-example/src/scope/move.md index 0433e8ac1..df631ee08 100644 --- a/src/doc/rust-by-example/src/scope/move.md +++ b/src/doc/rust-by-example/src/scope/move.md @@ -1,7 +1,7 @@ # Ownership and moves Because variables are in charge of freeing their own resources, -**resources can only have one owner**. This also prevents resources +**resources can only have one owner**. This prevents resources from being freed more than once. Note that not all variables own resources (e.g. [references]). diff --git a/src/doc/rust-by-example/src/std/hash/alt_key_types.md b/src/doc/rust-by-example/src/std/hash/alt_key_types.md index ab94819b2..7e68e8c67 100644 --- a/src/doc/rust-by-example/src/std/hash/alt_key_types.md +++ b/src/doc/rust-by-example/src/std/hash/alt_key_types.md @@ -3,7 +3,7 @@ Any type that implements the `Eq` and `Hash` traits can be a key in `HashMap`. This includes: -* `bool` (though not very useful since there is only two possible keys) +* `bool` (though not very useful since there are only two possible keys) * `int`, `uint`, and all variations thereof * `String` and `&str` (protip: you can have a `HashMap` keyed by `String` and call `.get()` with an `&str`) diff --git a/src/doc/rust-by-example/src/std_misc/arg/matching.md b/src/doc/rust-by-example/src/std_misc/arg/matching.md index 4cb68eabf..4a96fbc21 100644 --- a/src/doc/rust-by-example/src/std_misc/arg/matching.md +++ b/src/doc/rust-by-example/src/std_misc/arg/matching.md @@ -70,6 +70,9 @@ fn main() { } ``` +If you named your program `match_args.rs` and compile it like this `rustc +match_args.rs`, you can execute it as follows: + ```shell $ ./match_args Rust This is not the answer. diff --git a/src/doc/rust-by-example/src/std_misc/file/read_lines.md b/src/doc/rust-by-example/src/std_misc/file/read_lines.md index 216b0181c..1a2a64241 100644 --- a/src/doc/rust-by-example/src/std_misc/file/read_lines.md +++ b/src/doc/rust-by-example/src/std_misc/file/read_lines.md @@ -56,15 +56,13 @@ fn main() { // File hosts.txt must exist in the current path if let Ok(lines) = read_lines("./hosts.txt") { // Consumes the iterator, returns an (Optional) String - for line in lines { - if let Ok(ip) = line { - println!("{}", ip); - } + for line in lines.flatten() { + println!("{}", line); } } } -// The output is wrapped in a Result to allow matching on errors +// The output is wrapped in a Result to allow matching on errors. // Returns an Iterator to the Reader of the lines of the file. fn read_lines

(filename: P) -> io::Result>> where P: AsRef, { diff --git a/src/doc/rust-by-example/src/std_misc/threads/testcase_mapreduce.md b/src/doc/rust-by-example/src/std_misc/threads/testcase_mapreduce.md index ee25b1661..9cc162f51 100644 --- a/src/doc/rust-by-example/src/std_misc/threads/testcase_mapreduce.md +++ b/src/doc/rust-by-example/src/std_misc/threads/testcase_mapreduce.md @@ -30,7 +30,7 @@ use std::thread; fn main() { // This is our data to process. - // We will calculate the sum of all digits via a threaded map-reduce algorithm. + // We will calculate the sum of all digits via a threaded map-reduce algorithm. // Each whitespace separated chunk will be handled in a different thread. // // TODO: see what happens to the output if you insert spaces! diff --git a/src/doc/rustc-dev-guide/examples/rustc-driver-example.rs b/src/doc/rustc-dev-guide/examples/rustc-driver-example.rs index 70e77fd5b..fc0a5f469 100644 --- a/src/doc/rustc-dev-guide/examples/rustc-driver-example.rs +++ b/src/doc/rustc-dev-guide/examples/rustc-driver-example.rs @@ -63,6 +63,8 @@ fn main() { // Registry of diagnostics codes. registry: registry::Registry::new(&rustc_error_codes::DIAGNOSTICS), make_codegen_backend: None, + expanded_args: Vec::new(), + ice_file: None, }; rustc_interface::run_compiler(config, |compiler| { compiler.enter(|queries| { diff --git a/src/doc/rustc-dev-guide/examples/rustc-driver-getting-diagnostics.rs b/src/doc/rustc-dev-guide/examples/rustc-driver-getting-diagnostics.rs index 888674aaf..0195224de 100644 --- a/src/doc/rustc-dev-guide/examples/rustc-driver-getting-diagnostics.rs +++ b/src/doc/rustc-dev-guide/examples/rustc-driver-getting-diagnostics.rs @@ -73,6 +73,8 @@ fn main() { override_queries: None, registry: registry::Registry::new(&rustc_error_codes::DIAGNOSTICS), make_codegen_backend: None, + expanded_args: Vec::new(), + ice_file: None, }; rustc_interface::run_compiler(config, |compiler| { compiler.enter(|queries| { diff --git a/src/doc/rustc-dev-guide/examples/rustc-driver-interacting-with-the-ast.rs b/src/doc/rustc-dev-guide/examples/rustc-driver-interacting-with-the-ast.rs index df0e0385d..3d3827e5d 100644 --- a/src/doc/rustc-dev-guide/examples/rustc-driver-interacting-with-the-ast.rs +++ b/src/doc/rustc-dev-guide/examples/rustc-driver-interacting-with-the-ast.rs @@ -51,6 +51,8 @@ fn main() { override_queries: None, make_codegen_backend: None, registry: registry::Registry::new(&rustc_error_codes::DIAGNOSTICS), + expanded_args: Vec::new(), + ice_file: None, }; rustc_interface::run_compiler(config, |compiler| { compiler.enter(|queries| { diff --git a/src/doc/rustc-dev-guide/src/SUMMARY.md b/src/doc/rustc-dev-guide/src/SUMMARY.md index 41e16c5e6..4a7fd2b77 100644 --- a/src/doc/rustc-dev-guide/src/SUMMARY.md +++ b/src/doc/rustc-dev-guide/src/SUMMARY.md @@ -124,6 +124,7 @@ - [Goals and clauses](./traits/goals-and-clauses.md) - [Canonical queries](./traits/canonical-queries.md) - [Next-gen trait solving](./solve/trait-solving.md) + - [Invariants of the type system](./solve/invariants.md) - [The solver](./solve/the-solver.md) - [Canonicalization](./solve/canonicalization.md) - [Coinduction](./solve/coinduction.md) @@ -135,6 +136,7 @@ - [Opaque Types](./opaque-types-type-alias-impl-trait.md) - [Inference details](./opaque-types-impl-trait-inference.md) - [Return Position Impl Trait In Trait](./return-position-impl-trait-in-trait.md) +- [Effect checking](./effects.md) - [Pattern and Exhaustiveness Checking](./pat-exhaustive-checking.md) - [MIR dataflow](./mir/dataflow.md) - [Drop elaboration](./mir/drop-elaboration.md) diff --git a/src/doc/rustc-dev-guide/src/appendix/bibliography.md b/src/doc/rustc-dev-guide/src/appendix/bibliography.md index c2bb00e3b..cc262abef 100644 --- a/src/doc/rustc-dev-guide/src/appendix/bibliography.md +++ b/src/doc/rustc-dev-guide/src/appendix/bibliography.md @@ -79,7 +79,7 @@ Rust, as well as publications about Rust. Rust](https://munksgaard.me/papers/laumann-munksgaard-larsen.pdf). Philip Munksgaard's master's thesis. Research for Servo. * [Ownership is Theft: Experiences Building an Embedded OS in Rust - Amit Levy, et. al.](https://amitlevy.com/papers/tock-plos2015.pdf) -* [You can't spell trust without Rust](https://raw.githubusercontent.com/Gankro/thesis/master/thesis.pdf). Alexis Beingessner's master's thesis. +* [You can't spell trust without Rust](https://faultlore.com/blah/papers/thesis.pdf). Aria Beingessner's master's thesis. * [Rust-Bio: a fast and safe bioinformatics library](https://academic.oup.com/bioinformatics/article/32/3/444/1743419). Johannes Köster * [Safe, Correct, and Fast Low-Level Networking](https://octarineparrot.com/assets/msci_paper.pdf). Robert Clipsham's master's thesis. * [Formalizing Rust traits](https://open.library.ubc.ca/cIRcle/collections/ubctheses/24/items/1.0220521). Jonatan Milewski's master's thesis. diff --git a/src/doc/rustc-dev-guide/src/appendix/glossary.md b/src/doc/rustc-dev-guide/src/appendix/glossary.md index 27b6cddf2..ee3a3a720 100644 --- a/src/doc/rustc-dev-guide/src/appendix/glossary.md +++ b/src/doc/rustc-dev-guide/src/appendix/glossary.md @@ -25,6 +25,7 @@ Term | Meaning drop glue   | (internal) compiler-generated instructions that handle calling the destructors (`Drop`) for data types. DST   | Short for Dynamically-Sized Type, this is a type for which the compiler cannot statically know the size in memory (e.g. `str` or `[u8]`). Such types don't implement `Sized` and cannot be allocated on the stack. They can only occur as the last field in a struct. They can only be used behind a pointer (e.g. `&str` or `&[u8]`). early-bound lifetime   | A lifetime region that is substituted at its definition site. Bound in an item's `Generics` and substituted using a `GenericArgs`. Contrast with **late-bound lifetime**. ([see more](https://doc.rust-lang.org/nightly/nightly-rustc/rustc_type_ir/sty/enum.RegionKind.html#bound-regions)) +effects   | Right now only means const traits and `~const` bounds. ([see more](../effects.md)) empty type   | see "uninhabited type". fat pointer   | A two word value carrying the address of some value, along with some further information necessary to put the value to use. Rust includes two kinds of "fat pointers": references to slices, and trait objects. A reference to a slice carries the starting address of the slice and its length. A trait object carries a value's address and a pointer to the trait's implementation appropriate to that value. "Fat pointers" are also known as "wide pointers", and "double pointers". free variable   | A "free variable" is one that is not bound within an expression or term; see [the background chapter for more](./background.md#free-vs-bound) @@ -93,6 +94,7 @@ Term | Meaning upvar   | A variable captured by a closure from outside the closure. variance   | Determines how changes to a generic type/lifetime parameter affect subtyping; for example, if `T` is a subtype of `U`, then `Vec` is a subtype `Vec` because `Vec` is *covariant* in its generic parameter. See [the background chapter](./background.md#variance) for a more general explanation. See the [variance chapter](../variance.md) for an explanation of how type checking handles variance. variant index   | In an enum, identifies a variant by assigning them indices starting at 0. This is purely internal and not to be confused with the ["discriminant"](#discriminant) which can be overwritten by the user (e.g. `enum Bool { True = 42, False = 0 }`). +Well-formedness   | Semantically:An expression that evaluates to meaningful result. In Type Systems: A type related construct which follows rules of the type system. wide pointer   | A pointer with additional metadata. See "fat pointer" for more. ZST   | Zero-Sized Type. A type whose values have size 0 bytes. Since `2^0 = 1`, such types can have exactly one value. For example, `()` (unit) is a ZST. `struct Foo;` is also a ZST. The compiler can do some nice optimizations around ZSTs. diff --git a/src/doc/rustc-dev-guide/src/building/suggested.md b/src/doc/rustc-dev-guide/src/building/suggested.md index cb722696e..1336ff629 100644 --- a/src/doc/rustc-dev-guide/src/building/suggested.md +++ b/src/doc/rustc-dev-guide/src/building/suggested.md @@ -277,7 +277,6 @@ let # `config.toml.example`) from `1bd30ce2aac40c7698aa4a1b9520aa649ff2d1c5` config = pkgs.writeText "rustc-config" '' profile = "compiler" # you may want to choose a different profile, like `library` or `tools` - changelog-seen = 2 [build] patch-binaries-for-nix = true diff --git a/src/doc/rustc-dev-guide/src/effects.md b/src/doc/rustc-dev-guide/src/effects.md new file mode 100644 index 000000000..1fda7bcbb --- /dev/null +++ b/src/doc/rustc-dev-guide/src/effects.md @@ -0,0 +1,66 @@ +# Effects and effect checking + +Note: all of this describes the implementation of the unstable `effects` and +`const_trait_impl` features. None of this implementation is usable or visible from +stable Rust. + +The implementation of const traits and `~const` bounds is a limited effect system. +It is used to allow trait bounds on `const fn` to be used within the `const fn` for +method calls. Within the function, in order to know whether a method on a trait +bound is `const`, we need to know whether there is a `~const` bound for the trait. +In order to know whether we can instantiate a `~const` bound on a `const fn`, we +need to know whether there is a `const_trait` impl for the type and trait being +used (or whether the `const fn` is used at runtime, then any type implementing the +trait is ok, just like with other bounds). + +We perform these checks via a const generic boolean that gets attached to all +`const fn` and `const trait`. The following sections will explain the desugarings +and the way we perform the checks at call sites. + +The const generic boolean is inverted to the meaning of `const`. In the compiler +it is called `host`, because it enables "host APIs" like `static` items, network +access, disk access, random numbers and everything else that isn't available in +`const` contexts. So `false` means "const", `true` means "not const" and if it's +a generic parameter, it means "maybe const" (meaning we're in a const fn or const +trait). + +## `const fn` + +All `const fn` have a `#[rustc_host] const host: bool` generic parameter that is +hidden from users. Any `~const Trait` bounds in the generics list or `where` bounds +of a `const fn` get converted to `Trait + Trait` bounds. The `Trait` +exists so that associated types of the generic param can be used from projections +like `::Assoc`, because there are no `` projections for now. + +## `#[const_trait] trait`s + +The `#[const_trait]` attribute gives the marked trait a `#[rustc_host] const host: bool` +generic parameter. All functions of the trait "inherit" this generic parameter, just like +they have all the regular generic parameters of the trait. Any `~const Trait` super-trait +bounds get desugared to `Trait + Trait` in order to allow using associated +types and consts of the super traits in the trait declaration. This is necessary, because +`::Assoc` is always `>::Assoc` as there is +no `` syntax. + +## `typeck` performing method and function call checks. + +When generic parameters are instantiated for any items, the `host` generic parameter +is always instantiated as an inference variable. This is a special kind of inference var +that is not part of the type or const inference variables, similar to how we have +special inference variables for type variables that we know to be an integer, but not +yet which one. These separate inference variables fall back to `true` at +the end of typeck (in `fallback_effects`) to ensure that `let _ = some_fn_item_name;` +will keep compiling. + +All actually used (in function calls, casts, or anywhere else) function items, will +have the `enforce_context_effects` method invoked. +It trivially returns if the function being called has no `host` generic parameter. + +In order to error if a non-const function is called in a const context, we have not +yet disabled the const-check logic that happens on MIR, because +`enforce_context_effects` does not yet perform this check. + +The function call's `host` parameter is then equated to the context's `host` value, +which almost always trivially succeeds, as it was an inference var. If the inference +var has already been bound (since the function item is invoked twice), the second +invocation checks it against the first. diff --git a/src/doc/rustc-dev-guide/src/feature-gates.md b/src/doc/rustc-dev-guide/src/feature-gates.md index 8ad4fea1f..788f93d66 100644 --- a/src/doc/rustc-dev-guide/src/feature-gates.md +++ b/src/doc/rustc-dev-guide/src/feature-gates.md @@ -20,12 +20,12 @@ See ["Stability in code"][adding] in the "Implementing new features" section for To remove a feature gate, follow these steps: -1. Remove the feature gate declaration in `rustc_feature/src/active.rs`. +1. Remove the feature gate declaration in `rustc_feature/src/unstable.rs`. It will look like this: ```rust,ignore /// description of feature - (active, $feature_name, "$version", Some($tracking_issue_number), $edition) + (unstable, $feature_name, "$version", Some($tracking_issue_number), $edition) ``` 2. Add a modified version of the feature gate declaration that you just @@ -45,12 +45,12 @@ To remove a feature gate, follow these steps: To rename a feature gate, follow these steps (the first two are the same steps to follow when [removing a feature gate][removing]): -1. Remove the old feature gate declaration in `rustc_feature/src/active.rs`. +1. Remove the old feature gate declaration in `rustc_feature/src/unstable.rs`. It will look like this: ```rust,ignore /// description of feature - (active, $old_feature_name, "$version", Some($tracking_issue_number), $edition) + (unstable, $old_feature_name, "$version", Some($tracking_issue_number), $edition) ``` 2. Add a modified version of the old feature gate declaration that you just @@ -64,12 +64,12 @@ to follow when [removing a feature gate][removing]): ``` 3. Add a feature gate declaration with the new name to - `rustc_feature/src/active.rs`. It should look very similar to the old + `rustc_feature/src/unstable.rs`. It should look very similar to the old declaration: ```rust,ignore /// description of feature - (active, $new_feature_name, "$version", Some($tracking_issue_number), $edition) + (unstable, $new_feature_name, "$version", Some($tracking_issue_number), $edition) ``` diff --git a/src/doc/rustc-dev-guide/src/hir-debugging.md b/src/doc/rustc-dev-guide/src/hir-debugging.md index c25a558a0..5a0bda208 100644 --- a/src/doc/rustc-dev-guide/src/hir-debugging.md +++ b/src/doc/rustc-dev-guide/src/hir-debugging.md @@ -1,6 +1,13 @@ # HIR Debugging -The `-Z unpretty=hir-tree` flag will dump out the HIR. +Use the `-Z unpretty=hir` flag to produce a human-readable representation of the HIR. +For cargo projects this can be done with `cargo rustc -- -Z unpretty=hir`. +This output is useful when you need to see at a glance how your code was desugared and transformed +during AST lowering. + +For a full `Debug` dump of the data in the HIR, use the `-Z unpretty=hir-tree` flag. +This may be useful when you need to see the full structure of the HIR from the perspective of the +compiler. If you are trying to correlate `NodeId`s or `DefId`s with source code, the `-Z unpretty=expanded,identified` flag may be useful. diff --git a/src/doc/rustc-dev-guide/src/implementing_new_features.md b/src/doc/rustc-dev-guide/src/implementing_new_features.md index 01508889f..427589dab 100644 --- a/src/doc/rustc-dev-guide/src/implementing_new_features.md +++ b/src/doc/rustc-dev-guide/src/implementing_new_features.md @@ -123,12 +123,12 @@ a new unstable feature: 1. Add the feature name to `rustc_span/src/symbol.rs` in the `Symbols {...}` block. -1. Add a feature gate declaration to `rustc_feature/src/active.rs` in the active +1. Add a feature gate declaration to `rustc_feature/src/unstable.rs` in the unstable `declare_features` block. ```rust ignore /// description of feature - (active, $feature_name, "CURRENT_RUSTC_VERSION", Some($tracking_issue_number), $edition) + (unstable, $feature_name, "CURRENT_RUSTC_VERSION", Some($tracking_issue_number), $edition) ``` where `$edition` has the type `Option`, and is typically just `None`. If you haven't yet @@ -140,7 +140,7 @@ a new unstable feature: ```rust ignore /// Allows defining identifiers beyond ASCII. - (active, non_ascii_idents, "CURRENT_RUSTC_VERSION", Some(55467), None), + (unstable, non_ascii_idents, "CURRENT_RUSTC_VERSION", Some(55467), None), ``` Features can be marked as incomplete, and trigger the warn-by-default [`incomplete_features` diff --git a/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md b/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md index 9379a57f6..fd215e3e9 100644 --- a/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md +++ b/src/doc/rustc-dev-guide/src/llvm-coverage-instrumentation.md @@ -73,21 +73,21 @@ When compiling with `-C instrument-coverage`, Coverage instrumentation is performed on the MIR with a [MIR pass][mir-passes] called [`InstrumentCoverage`][mir-instrument-coverage]. This MIR pass analyzes the control flow graph (CFG)--represented by MIR `BasicBlock`s--to identify -code branches, and injects additional [`Coverage`][coverage-statement] -statements into the `BasicBlock`s. +code branches, attaches [`FunctionCoverageInfo`] to the function's body, +and injects additional [`Coverage`][coverage-statement] statements into the +`BasicBlock`s. A MIR `Coverage` statement is a virtual instruction that indicates a counter should be incremented when its adjacent statements are executed, to count a span of code ([`CodeRegion`][code-region]). It counts the number of times a -branch is executed, and also specifies the exact location of that code span in -the Rust source code. +branch is executed, and is referred to by coverage mappings in the function's +coverage-info struct. -Note that many of these `Coverage` statements will _not_ be converted into +Note that many coverage counters will _not_ be converted into physical counters (or any other executable instructions) in the final binary. -Some of them will be (see [`CoverageKind::Counter`]), +Some of them will be (see [`CoverageKind::CounterIncrement`]), but other counters can be computed on the fly, when generating a coverage -report, by mapping a `CodeRegion` to a -[`CoverageKind::Expression`]. +report, by mapping a `CodeRegion` to a coverage-counter _expression_. As an example: @@ -121,8 +121,8 @@ determines when to break out of a loop (a `while` condition, or an `if` or `match` with a `break`). In MIR, this is typically lowered to a `SwitchInt`, with one branch to stay in the loop, and another branch to break out of the loop. The branch that breaks out will almost always execute less often, -so `InstrumentCoverage` chooses to add a `Counter` to that branch, and an -`Expression(continue) = Counter(loop) - Counter(break)` to the branch that +so `InstrumentCoverage` chooses to add a `CounterIncrement` to that branch, and +uses an expression (`Counter(loop) - Counter(break)`) for the branch that continues. The `InstrumentCoverage` MIR pass is documented in @@ -130,9 +130,9 @@ The `InstrumentCoverage` MIR pass is documented in [mir-passes]: mir/passes.md [mir-instrument-coverage]: https://github.com/rust-lang/rust/tree/master/compiler/rustc_mir_transform/src/coverage +[`FunctionCoverageInfo`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/struct.FunctionCoverageInfo.html [code-region]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/struct.CodeRegion.html -[`CoverageKind::Counter`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/enum.CoverageKind.html#variant.Counter -[`CoverageKind::Expression`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/enum.CoverageKind.html#variant.Expression +[`CoverageKind::CounterIncrement`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/coverage/enum.CoverageKind.html#variant.CounterIncrement [coverage-statement]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/mir/enum.StatementKind.html#variant.Coverage [instrument-coverage-pass-details]: #implementation-details-of-the-instrumentcoverage-mir-pass @@ -150,40 +150,38 @@ MIR `Statement` into some backend-specific action or instruction. match statement.kind { ... mir::StatementKind::Coverage(box ref coverage) => { - self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope); - bx + self.codegen_coverage(bx, coverage, statement.source_info.scope); } ``` -`codegen_coverage()` handles each `CoverageKind` as follows: +`codegen_coverage()` handles inlined statements and then forwards the coverage +statement to [`Builder::add_coverage`], which handles each `CoverageKind` as +follows: -- For all `CoverageKind`s, Coverage data (counter ID, expression equation - and ID, and code regions) are passed to the backend's `Builder`, to - populate data structures that will be used to generate the crate's - "Coverage Map". (See the [`FunctionCoverage`][function-coverage] `struct`.) -- For `CoverageKind::Counter`s, an instruction is injected in the backend + +- For both `CounterIncrement` and `ExpressionUsed`, the underlying counter or + expression ID is passed through to the corresponding [`FunctionCoverage`] + struct to indicate that the corresponding regions of code were not removed + by MIR optimizations. +- For `CoverageKind::CounterIncrement`s, an instruction is injected in the backend IR to increment the physical counter, by calling the `BuilderMethod` [`instrprof_increment()`][instrprof-increment]. ```rust - pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage, scope: SourceScope) { + fn add_coverage(&mut self, instance: Instance<'tcx>, coverage: &Coverage) { ... - let instance = ... // the scoped instance (current or inlined function) - let Coverage { kind, code_region } = coverage; - match kind { - CoverageKind::Counter { function_source_hash, id } => { - ... - bx.add_coverage_counter(instance, id, code_region); + let Coverage { kind } = coverage; + match *kind { + CoverageKind::CounterIncrement { id } => { + func_coverage.mark_counter_id_seen(id); ... bx.instrprof_increment(fn_name, hash, num_counters, index); } - CoverageKind::Expression { id, lhs, op, rhs } => { - bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region); + CoverageKind::ExpressionUsed { id } => { + func_coverage.mark_expression_id_seen(id); } - CoverageKind::Unreachable => { - bx.add_coverage_unreachable( - instance, - code_region.expect(... + } + } ``` > The function name `instrprof_increment()` is taken from the LLVM intrinsic @@ -199,7 +197,8 @@ statements is only implemented for LLVM, at this time. [backend-lowering-mir]: backend/lowering-mir.md [codegen-statement]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/struct.FunctionCx.html#method.codegen_statement [codegen-coverage]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/mir/struct.FunctionCx.html#method.codegen_coverage -[function-coverage]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/coverageinfo/map_data/struct.FunctionCoverage.html +[`Builder::add_coverage`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/builder/struct.Builder.html#method.add_coverage +[`FunctionCoverage`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_llvm/coverageinfo/map_data/struct.FunctionCoverage.html [instrprof-increment]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_codegen_ssa/traits/trait.BuilderMethods.html#tymethod.instrprof_increment ### Coverage Map Generation @@ -327,9 +326,10 @@ Instrumentor::new(&self.name(), tcx, mir_body).inject_counters(); The `CoverageGraph` is a coverage-specific simplification of the MIR control flow graph (CFG). Its nodes are [`BasicCoverageBlock`s][bcb], which encompass one or more sequentially-executed MIR `BasicBlock`s -(with no internal branching), plus a `CoverageKind` counter (to -be added, via coverage analysis), and an optional set of additional counters -to count incoming edges (if there are more than one). +(with no internal branching). + +Nodes and edges in the graph can have associated [`BcbCounter`]s, which are +stored in [`CoverageCounters`]. The `Instrumentor`'s `inject_counters()` uses the `CoverageGraph` to compute the best places to inject coverage counters, as MIR `Statement`s, @@ -338,16 +338,15 @@ with the following steps: 1. [`generate_coverage_spans()`][generate-coverage-spans] computes the minimum set of distinct, non-branching code regions, from the MIR. These `CoverageSpan`s represent a span of code that must be counted. -2. [`make_bcb_counters()`][make-bcb-counters] generates `CoverageKind::Counter`s and - `CoverageKind::Expression`s for each `CoverageSpan`, plus additional - `intermediate_expressions`[^intermediate-expressions], not associated with any `CodeRegion`, but +2. [`make_bcb_counters()`][make-bcb-counters] generates `BcbCounter::Counter`s and + `BcbCounter::Expression`s for each `CoverageSpan`, plus additional + _intermediate expressions_[^intermediate-expressions] that are not associated + with any `CodeRegion`, but are required to compute a final `Expression` value for a `CodeRegion`. 3. Inject the new counters into the MIR, as new `StatementKind::Coverage` - statements. This is done by three distinct functions: - - `inject_coverage_span_counters()` - - `inject_indirect_counters()` - - `inject_intermediate_expression()`, called for each intermediate expression - returned from `make_bcb_counters()` + statements. +4. Attach all other necessary coverage information to the function's body as + [`FunctionCoverageInfo`]. [^intermediate-expressions]: Intermediate expressions are sometimes required because `Expression`s are limited to binary additions or subtractions. For @@ -359,7 +358,8 @@ intermediate expression for `B - C`. [coverage-graph]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/graph/struct.CoverageGraph.html [inject-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/struct.Instrumentor.html#method.inject_counters [bcb]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/graph/struct.BasicCoverageBlock.html -[debug]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/debug +[`BcbCounter`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/counters/enum.BcbCounter.html +[`CoverageCounters`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/counters/struct.CoverageCounters.html [generate-coverage-spans]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/spans/struct.CoverageSpans.html#method.generate_coverage_spans [make-bcb-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/counters/struct.BcbCounters.html#method.make_bcb_counters @@ -505,34 +505,3 @@ its `Counter` or `Expression`. [bcb-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/counters/struct.BcbCounters.html [traverse-coverage-graph-with-loops]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/graph/struct.TraverseCoverageGraphWithLoops.html - -### Injecting counters into a MIR `BasicBlock` - -With the refined `CoverageSpan`s, and after all `Counter`s and `Expression`s are -created, the final step is to inject the `StatementKind::Coverage` statements -into the MIR. There are three distinct sources, handled by the following -functions: - -- [`inject_coverage_span_counters()`][inject-coverage-span-counters] injects the - counter from each `CoverageSpan`'s BCB. -- [`inject_indirect_counters()`][inject-indirect-counters] injects counters - for any BCB not assigned to a `CoverageSpan`, and for all edge counters. - These counters don't have `CoverageSpan`s. -- [`inject_intermediate_expression()`][inject-intermediate-expression] injects - the intermediate expressions returned from `make_bcb_counters()`. These - counters aren't associated with any BCB, edge, or `CoverageSpan`. - -These three functions inject the `Coverage` statements into the MIR. -`Counter`s and `Expression`s with `CoverageSpan`s add `Coverage` statements -to a corresponding `BasicBlock`, with a `CodeRegion` computed from the -refined `Span` and current `SourceMap`. - -All other `Coverage` statements have a `CodeRegion` of `None`, but they -still must be injected because they contribute to other `Expression`s. - -Finally, edge's with a `CoverageKind::Counter` require a new `BasicBlock`, -so the counter is only incremented when traversing the branch edge. - -[inject-coverage-span-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/struct.Instrumentor.html#method.inject_coverage_span_counters -[inject-indirect-counters]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/struct.Instrumentor.html#method.inject_indirect_counters -[inject-intermediate-expression]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_mir_transform/coverage/fn.inject_intermediate_expression.html diff --git a/src/doc/rustc-dev-guide/src/return-position-impl-trait-in-trait.md b/src/doc/rustc-dev-guide/src/return-position-impl-trait-in-trait.md index 03c7fb6b7..9ed62c87c 100644 --- a/src/doc/rustc-dev-guide/src/return-position-impl-trait-in-trait.md +++ b/src/doc/rustc-dev-guide/src/return-position-impl-trait-in-trait.md @@ -298,6 +298,24 @@ types in the impl, since this mapping describes the type that should come after the `=` in `type Assoc = ...` for each RPITIT. +##### Implied bounds in RPITIT hidden type inference + +Since `collect_return_position_impl_trait_in_trait_tys` does fulfillment and +region resolution, we must provide it `assumed_wf_types` so that we can prove +region obligations with the same expected implied bounds as +`compare_method_predicate_entailment` does. + +Since the return type of a method is understood to be one of the assumed WF +types, and we eagerly fold the return type with inference variables to do +opaque type inference, after opaque type inference, the return type will +resolve to contain the hidden types of the RPITITs. this would mean that the +hidden types of the RPITITs would be assumed to be well-formed without having +independently proven that they are. This resulted in a +[subtle unsoundness bug](https://github.com/rust-lang/rust/pull/116072). In +order to prevent this cyclic reasoning, we instead replace the hidden types of +the RPITITs in the return type of the method with *placeholders*, which lead +to no implied well-formedness bounds. + #### Default trait body Type-checking a default trait body, like: diff --git a/src/doc/rustc-dev-guide/src/rustc-driver-getting-diagnostics.md b/src/doc/rustc-dev-guide/src/rustc-driver-getting-diagnostics.md index 47b9fb5d9..95e3c7cc8 100644 --- a/src/doc/rustc-dev-guide/src/rustc-driver-getting-diagnostics.md +++ b/src/doc/rustc-dev-guide/src/rustc-driver-getting-diagnostics.md @@ -7,7 +7,7 @@ To get diagnostics from the compiler, configure `rustc_interface::Config` to output diagnostic to a buffer, and run `TyCtxt.analysis`. The following was tested -with `nightly-2023-03-27`: +with `nightly-2023-10-03`: ```rust {{#include ../examples/rustc-driver-getting-diagnostics.rs}} diff --git a/src/doc/rustc-dev-guide/src/rustc-driver-interacting-with-the-ast.md b/src/doc/rustc-dev-guide/src/rustc-driver-interacting-with-the-ast.md index 4edbbca00..fc119c1ec 100644 --- a/src/doc/rustc-dev-guide/src/rustc-driver-interacting-with-the-ast.md +++ b/src/doc/rustc-dev-guide/src/rustc-driver-interacting-with-the-ast.md @@ -5,7 +5,7 @@ ## Getting the type of an expression To get the type of an expression, use the `global_ctxt` to get a `TyCtxt`. -The following was tested with `nightly-2023-03-27`: +The following was tested with `nightly-2023-10-03`: ```rust {{#include ../examples/rustc-driver-interacting-with-the-ast.rs}} diff --git a/src/doc/rustc-dev-guide/src/solve/invariants.md b/src/doc/rustc-dev-guide/src/solve/invariants.md new file mode 100644 index 000000000..75ae53070 --- /dev/null +++ b/src/doc/rustc-dev-guide/src/solve/invariants.md @@ -0,0 +1,154 @@ +# Invariants of the type system + +FIXME: This file talks about invariants of the type system as a whole, not only the solver + +There are a lot of invariants - things the type system guarantees to be true at all times - +which are desirable or expected from other languages and type systems. Unfortunately, quite +a few of them do not hold in Rust right now. This is either a fundamental to its design or +caused by bugs and something that may change in the future. + +It is important to know about the things you can assume while working on - and with - the +type system, so here's an incomplete and inofficial list of invariants of +the core type system: + +- ✅: this invariant mostly holds, with some weird exceptions, you can rely on it outside +of these cases +- ❌: this invariant does not hold, either due to bugs or by design, you must not rely on +it for soundness or have to be incredibly careful when doing so + +### `wf(X)` implies `wf(normalize(X))` ✅ + +If a type containing aliases is well-formed, it should also be +well-formed after normalizing said aliases. We rely on this as +otherwise we would have to re-check for well-formedness for these +types. + +This is unfortunately broken for `>::Output` due to implied bounds, +resulting in [#114936]. + +### Structural equality modulo regions implies semantic equality ✅ + +If you have a some type and equate it to itself after replacing any regions with unique +inference variables in both the lhs and rhs, the now potentially structurally different +types should still be equal to each other. + +Needed to prevent goals from succeeding in HIR typeck and then failing in MIR borrowck. +If this does invariant is broken MIR typeck ends up failing with an ICE. + +### Applying inference results from a goal does not change its result ❌ + +TODO: this invariant is formulated in a weird way and needs to be elaborated. +Pretty much: I would like this check to only fail if there's a solver bug: +https://github.com/rust-lang/rust/blob/2ffeb4636b4ae376f716dc4378a7efb37632dc2d/compiler/rustc_trait_selection/src/solve/eval_ctxt.rs#L391-L407 + +If we prove some goal/equate types/whatever, apply the resulting inference constraints, +and then redo the original action, the result should be the same. + +This unfortunately does not hold - at least in the new solver - due to a few annoying reasons. + +### The trait solver has to be *locally sound* ✅ + +This means that we must never return *success* for goals for which no `impl` exists. That would +mean we assume a trait is implemented even though it is not, which is very likely to result in +actual unsoundness. When using `where`-bounds to prove a goal, the `impl` will be provided by the +user of the item. + +This invariant only holds if we check region constraints. As we do not check region constraints +during implicit negative overlap check in coherence, this invariant is broken there. As this check +relies on *completeness* of the trait solver, it is not able to use the current region constraints +check - `InferCtxt::resolve_regions` - as its handling of type outlives goals is incomplete. + +### Normalization of semantically equal aliases in empty environments results in a unique type ✅ + +Normalization for alias types/consts has to have a unique result. Otherwise we can easily +implement transmute in safe code. Given the following function, we have to make sure that +the input and output types always get normalized to the same concrete type. + +```rust +fn foo( + x: ::Assoc +) -> ::Assoc { + x +} +``` + +Many of the currently known unsound issues end up relying on this invariant being broken. +It is however very difficult to imagine a sound type system without this invariant, so +the issue is that the invariant is broken, not that we incorrectly rely on it. + +### Generic goals and their instantiations have the same result ✅ + +Pretty much: If we successfully typecheck a generic function concrete instantiations +of that function should also typeck. We should not get errors post-monomorphization. +We can however get overflow errors at that point. + +TODO: example for overflow error post-monomorphization + +This invariant is relied on to allow the normalization of generic aliases. Breaking +it can easily result in unsoundness, e.g. [#57893](https://github.com/rust-lang/rust/issues/57893) + +### Trait goals in empty environments are proven by a unique impl ✅ + +If a trait goal holds with an empty environment, there should be a unique `impl`, +either user-defined or builtin, which is used to prove that goal. This is +necessary to select a unique method. It + +We do however break this invariant in few cases, some of which are due to bugs, +some by design: +- *marker traits* are allowed to overlap as they do not have associated items +- *specialization* allows specializing impls to overlap with their parent +- the builtin trait object trait implementation can overlap with a user-defined impl: +[#57893] + +### The type system is complete ❌ + +The type system is not complete, it often adds unnecessary inference constraints, and errors +even though the goal could hold. + +- method selection +- opaque type inference +- handling type outlives constraints +- preferring `ParamEnv` candidates over `Impl` candidates during candidate selection +in the trait solver + +#### The type system is complete during the implicit negative overlap check in coherence ✅ + +During the implicit negative overlap check in coherence we must never return *error* for +goals which can be proven. This would allow for overlapping impls with potentially different +associated items, breaking a bunch of other invariants. + +This invariant is currently broken in many different ways while actually something we rely on. +We have to be careful as it is quite easy to break: +- generalization of aliases +- generalization during subtyping binders (luckily not exploitable in coherence) + +### Trait solving must be (free) lifetime agnostic ✅ + +Trait solving during codegen should have the same result as during typeck. As we erase +all free regions during codegen we must not rely on them during typeck. A noteworthy example +is special behavior for `'static`. + +We also have to be careful with relying on equality of regions in the trait solver. +This is fine for codegen, as we treat all erased regions as equal. We can however +lose equality information from HIR to MIR typeck. + +The new solver "uniquifies regions" during canonicalization, canonicalizing `u32: Trait<'x, 'x>` +as `exists<'0, '1> u32: Trait<'0, '1>`, to make it harder to rely on this property. + +### Removing ambiguity makes strictly more things compile ❌ + +Ideally we *should* not rely on ambiguity for things to compile. +Not doing that will cause future improvements to be breaking changes. + +Due to *incompleteness* this is not the case and improving inference can result in inference +changes, breaking existing projects. + +### Semantic equality implies structural equality ✅ + +Two types being equal in the type system must mean that they have the +same `TypeId` after instantiating their generic parameters with concrete +arguments. This currently does not hold: [#97156]. + +[#57893]: https://github.com/rust-lang/rust/issues/57893 +[#97156]: https://github.com/rust-lang/rust/issues/97156 +[#114936]: https://github.com/rust-lang/rust/issues/114936 \ No newline at end of file diff --git a/src/doc/rustc-dev-guide/src/solve/the-solver.md b/src/doc/rustc-dev-guide/src/solve/the-solver.md index 61e6cad1c..f7d82d117 100644 --- a/src/doc/rustc-dev-guide/src/solve/the-solver.md +++ b/src/doc/rustc-dev-guide/src/solve/the-solver.md @@ -6,12 +6,71 @@ approach. [chalk]: https://rust-lang.github.io/chalk/book/recursive.html -The basic structure of the solver is a pure function -`fn evaluate_goal(goal: Goal<'tcx>) -> Response`. -While the actual solver is not fully pure to deal with overflow and cycles, we are -going to defer that for now. +## A rough walkthrough -To deal with inference variables and to improve caching, we use -[canonicalization](./canonicalization.md). +The entry-point of the solver is `InferCtxtEvalExt::evaluate_root_goal`. This +function sets up the root `EvalCtxt` and then calls `EvalCtxt::evaluate_goal`, +to actually enter the trait solver. -TODO: write the remaining code for this as well. +`EvalCtxt::evaluate_goal` handles [canonicalization](./canonicalization.md), caching, +overflow, and solver cycles. Once that is done, it creates a nested `EvalCtxt` with a +separate local `InferCtxt` and calls `EvalCtxt::compute_goal`, which is responsible for the +'actual solver behavior'. We match on the `PredicateKind`, delegating to a separate function +for each one. + +For trait goals, such a `Vec: Clone`, `EvalCtxt::compute_trait_goal` has +to collect all the possible ways this goal can be proven via +`EvalCtxt::assemble_and_evaluate_candidates`. Each candidate is handled in +a separate "probe", to not leak inference constraints to the other candidates. +We then try to merge the assembled candidates via `EvalCtxt::merge_candidates`. + + +## Important concepts and design pattern + +### `EvalCtxt::add_goal` + +To prove nested goals, we don't directly call `EvalCtxt::compute_goal`, but instead +add the goal to the `EvalCtxt` with `EvalCtxt::all_goal`. We then prove all nested +goals together in either `EvalCtxt::try_evaluate_added_goals` or +`EvalCtxt::evaluate_added_goals_and_make_canonical_response`. This allows us to handle +inference constraints from later goals. + +E.g. if we have both `?x: Debug` and `(): ConstrainToU8` as nested goals, +then proving `?x: Debug` is initially ambiguous, but after proving `(): ConstrainToU8` +we constrain `?x` to `u8` and proving `u8: Debug` succeeds. + +### Matching on `TyKind` + +We lazily normalize types in the solver, so we always have to assume that any types +and constants are potentially unnormalized. This means that matching on `TyKind` can easily +be incorrect. + +We handle normalization in two different ways. When proving `Trait` goals when normalizing +associated types, we separately assemble candidates depending on whether they structurally +match the self type. Candidates which match on the self type are handled in +`EvalCtxt::assemble_candidates_via_self_ty` which recurses via +`EvalCtxt::assemble_candidates_after_normalizing_self_ty`, which normalizes the self type +by one level. In all other cases we have to match on a `TyKind` we first use +`EvalCtxt::try_normalize_ty` to normalize the type as much as possible. + +### Higher ranked goals + +In case the goal is higher-ranked, e.g. `for<'a> F: FnOnce(&'a ())`, `EvalCtxt::compute_goal` +eagerly instantiates `'a` with a placeholder and then recursively proves +`F: FnOnce(&'!a ())` as a nested goal. + +### Dealing with choice + +Some goals can be proven in multiple ways. In these cases we try each option in +a separate "probe" and then attempt to merge the resulting responses by using +`EvalCtxt::try_merge_responses`. If merging the responses fails, we use +`EvalCtxt::flounder` instead, returning ambiguity. For some goals, we try +incompletely prefer some choices over others in case `EvalCtxt::try_merge_responses` +fails. + +## Learning more + +The solver should be fairly self-contained. I hope that the above information provides a +good foundation when looking at the code itself. Please reach out on zulip if you get stuck +while doing so or there are some quirks and design decisions which were unclear and deserve +better comments or should be mentioned here. diff --git a/src/doc/rustc-dev-guide/src/solve/trait-solving.md b/src/doc/rustc-dev-guide/src/solve/trait-solving.md index c3089f4a8..7c1e0b684 100644 --- a/src/doc/rustc-dev-guide/src/solve/trait-solving.md +++ b/src/doc/rustc-dev-guide/src/solve/trait-solving.md @@ -39,77 +39,6 @@ which does not have any nested goals. Therefore `Vec: Clone` holds. The trait solver can either return success, ambiguity or an error as a [`CanonicalResponse`]. For success and ambiguity it also returns constraints inference and region constraints. -## Requirements - -Before we dive into the new solver lets first take the time to go through all of our requirements -on the trait system. We can then use these to guide our design later on. - -TODO: elaborate on these rules and get more precise about their meaning. -Also add issues where each of these rules have been broken in the past -(or still are). - -### 1. The trait solver has to be *sound* - -This means that we must never return *success* for goals for which no `impl` exists. That would -simply be unsound by assuming a trait is implemented even though it is not. When using predicates -from the `where`-bounds, the `impl` will be proved by the user of the item. - -### 2. If type checker solves generic goal concrete instantiations of that goal have the same result - -Pretty much: If we successfully typecheck a generic function concrete instantiations -of that function should also typeck. We should not get errors post-monomorphization. -We can however get overflow as in the following snippet: - -```rust -fn foo(x: ) -``` - -### 3. Trait goals in empty environments are proven by a unique impl - -If a trait goal holds with an empty environment, there is a unique `impl`, -either user-defined or builtin, which is used to prove that goal. - -This is necessary for codegen to select a unique method. -An exception here are *marker traits* which are allowed to overlap. - -### 4. Normalization in empty environments results in a unique type - -Normalization for alias types/consts has a unique result. Otherwise we can easily implement -transmute in safe code. Given the following function, we have to make sure that the input and -output types always get normalized to the same concrete type. -```rust -fn foo( - x: ::Assoc -) -> ::Assoc { - x -} -``` - -### 5. During coherence trait solving has to be complete - -During coherence we never return *error* for goals which can be proven. This allows overlapping -impls which would break rule 3. - -### 6. Trait solving must be (free) lifetime agnostic - -Trait solving during codegen should have the same result as during typeck. As we erase -all free regions during codegen we must not rely on them during typeck. A noteworthy example -is special behavior for `'static`. - -We also have to be careful with relying on equality of regions in the trait solver. -This is fine for codegen, as we treat all erased regions as equal. We can however -lose equality information from HIR to MIR typeck. - -### 7. Removing ambiguity makes strictly more things compile - -We *should* not rely on ambiguity for things to compile. -Not doing that will cause future improvements to be breaking changes. - -### 8. semantic equality implies structural equality - -Two types being equal in the type system must mean that they have the same `TypeId`. - - [solve]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/solve/index.html [`Goal`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_trait_selection/traits/solve/struct.Goal.html [`Predicate`]: https://doc.rust-lang.org/nightly/nightly-rustc/rustc_middle/ty/struct.Predicate.html diff --git a/src/doc/rustc-dev-guide/src/stabilization_guide.md b/src/doc/rustc-dev-guide/src/stabilization_guide.md index 001ed25a5..9bc70f65c 100644 --- a/src/doc/rustc-dev-guide/src/stabilization_guide.md +++ b/src/doc/rustc-dev-guide/src/stabilization_guide.md @@ -109,7 +109,7 @@ to stabilize, something like (this example is taken from ```rust,ignore // pub(restricted) visibilities (RFC 1422) -(active, pub_restricted, "CURRENT_RUSTC_VERSION", Some(32409)), +(unstable, pub_restricted, "CURRENT_RUSTC_VERSION", Some(32409)), ``` The above line should be moved down to the area for "accepted" diff --git a/src/doc/rustc-dev-guide/src/tests/headers.md b/src/doc/rustc-dev-guide/src/tests/headers.md index f066dbbb5..fce2397e5 100644 --- a/src/doc/rustc-dev-guide/src/tests/headers.md +++ b/src/doc/rustc-dev-guide/src/tests/headers.md @@ -190,7 +190,7 @@ The following headers are generally available, and not specific to particular test suites. * `compile-flags` passes extra command-line args to the compiler, - e.g. `compile-flags -g` which forces debuginfo to be enabled. + e.g. `// compile-flags: -g` which forces debuginfo to be enabled. * `run-flags` passes extra args to the test if the test is to be executed. * `edition` controls the edition the test should be compiled with (defaults to 2015). Example usage: `// edition:2018`. diff --git a/src/doc/rustc-dev-guide/src/traits/unsize.md b/src/doc/rustc-dev-guide/src/traits/unsize.md new file mode 100644 index 000000000..b11760992 --- /dev/null +++ b/src/doc/rustc-dev-guide/src/traits/unsize.md @@ -0,0 +1,84 @@ +# [`CoerceUnsized`](https://doc.rust-lang.org/std/ops/trait.CoerceUnsized.html) + +`CoerceUnsized` is primarily concerned with data containers. When a struct +(typically, a smart pointer) implements `CoerceUnsized`, that means that the +data it points to is being unsized. + +Some implementors of `CoerceUnsized` include: +* `&T` +* `Arc` +* `Box` + +This trait is (eventually) intended to be implemented by user-written smart +pointers, and there are rules about when a type is allowed to implement +`CoerceUnsized` that are explained in the trait's documentation. + +# [`Unsize`](https://doc.rust-lang.org/std/marker/trait.Unsize.html) + +To contrast, the `Unsize` trait is concerned the actual types that are allowed +to be unsized. + +This is not intended to be implemented by users ever, since `Unsize` does not +instruct the compiler (namely codegen) *how* to unsize a type, just whether it +is allowed to be unsized. This is paired somewhat intimately with codegen +which must understand how types are represented and unsized. + +## Primitive unsizing implementations + +Built-in implementations are provided for: +* `T` -> `dyn Trait + 'a` when `T: Trait` (and `T: Sized + 'a`, and `Trait` + is object safe). +* `[T; N]` -> `[T]` + +## Structural implementations + +There are two implementations of `Unsize` which can be thought of as +structural: +* `(A1, A2, .., An): Unsize<(A1, A2, .., U)>` given `An: Unsize`, which + allows the tail field of a tuple to be unsized. This is gated behind the + [`unsized_tuple_coercion`] feature. +* `Struct<.., Pi, .., Pj, ..>: Unsize>` given + `TailField: Unsize`, which allows the tail field of a + struct to be unsized if it is the only field that mentions generic parameters + `Pi`, .., `Pj` (which don't need to be contiguous). + +The rules for the latter implementation are slightly complicated, since they +may allow more than one parameter to be changed (not necessarily unsized) and +are best stated in terms of the tail field of the struct. + +[`unsized_tuple_coercion`]: https://doc.rust-lang.org/beta/unstable-book/language-features/unsized-tuple-coercion.html + +## Upcasting implementations + +Two things are called "upcasting" internally: +1. True upcasting `dyn SubTrait` -> `dyn SuperTrait` (this also allows + dropping auto traits and adjusting lifetimes, as below). +2. Dropping auto traits and adjusting the lifetimes of dyn trait + *without changing the principal[^1]*: + `dyn Trait + AutoTraits... + 'a` -> `dyn Trait + NewAutoTraits... + 'b` + when `AutoTraits` ⊇ `NewAutoTraits`, and `'a: 'b`. + +These may seem like different operations, since (1.) includes adjusting the +vtable of a dyn trait, while (2.) is a no-op. However, to the type system, +these are handled with much the same code. + +This built-in implementation of `Unsize` is the most involved, particularly +after [it was reworked](https://github.com/rust-lang/rust/pull/114036) to +support the complexities of associated types. + +Specifically, the upcasting algorithm involves: For each supertrait of the +source dyn trait's principal (including itself)... +1. Unify the super trait ref with the principal of the target (making sure + we only ever upcast to a true supertrait, and never [via an impl]). +2. For every auto trait in the source, check that it's present in the principal + (allowing us to drop auto traits, but never gain new ones). +3. For every projection in the source, check that it unifies with a single + projection in the target (since there may be more than one given + `trait Sub: Sup<.., A = i32> + Sup<.., A = u32>`). + +[via an impl]: https://github.com/rust-lang/rust/blob/f3457dbf84cd86d284454d12705861398ece76c3/tests/ui/traits/trait-upcasting/illegal-upcast-from-impl.rs#L19 + +Specifically, (3.) prevents a choice of projection bound to guide inference +unnecessarily, though it may guide inference when it is unambiguous. + +[^1]: The principal is the one non-auto trait of a `dyn Trait`. \ No newline at end of file diff --git a/src/doc/rustc/src/SUMMARY.md b/src/doc/rustc/src/SUMMARY.md index 5c6633864..1b27b77b3 100644 --- a/src/doc/rustc/src/SUMMARY.md +++ b/src/doc/rustc/src/SUMMARY.md @@ -33,7 +33,7 @@ - [\*-esp-espidf](platform-support/esp-idf.md) - [\*-unknown-fuchsia](platform-support/fuchsia.md) - [\*-kmc-solid_\*](platform-support/kmc-solid.md) - - [csky-unknown-linux-gnuabiv2](platform-support/csky-unknown-linux-gnuabiv2.md) + - [csky-unknown-linux-gnuabiv2\*](platform-support/csky-unknown-linux-gnuabiv2.md) - [loongarch\*-unknown-linux-\*](platform-support/loongarch-linux.md) - [loongarch\*-unknown-none\*](platform-support/loongarch-none.md) - [m68k-unknown-linux-gnu](platform-support/m68k-unknown-linux-gnu.md) @@ -41,6 +41,7 @@ - [mipsel-sony-psx](platform-support/mipsel-sony-psx.md) - [mipsisa\*r6\*-unknown-linux-gnu\*](platform-support/mips-release-6.md) - [nvptx64-nvidia-cuda](platform-support/nvptx64-nvidia-cuda.md) + - [powerpc64-ibm-aix](platform-support/aix.md) - [riscv32imac-unknown-xous-elf](platform-support/riscv32imac-unknown-xous-elf.md) - [sparc-unknown-none-elf](./platform-support/sparc-unknown-none-elf.md) - [*-pc-windows-gnullvm](platform-support/pc-windows-gnullvm.md) diff --git a/src/doc/rustc/src/codegen-options/index.md b/src/doc/rustc/src/codegen-options/index.md index d72543c7e..cfbe1e09c 100644 --- a/src/doc/rustc/src/codegen-options/index.md +++ b/src/doc/rustc/src/codegen-options/index.md @@ -249,11 +249,9 @@ flavor. Valid options are: * `gcc`: use the `cc` executable, which is typically gcc or clang on many systems. * `ld`: use the `ld` executable. * `msvc`: use the `link.exe` executable from Microsoft Visual Studio MSVC. -* `ptx-linker`: use - [`rust-ptx-linker`](https://github.com/denzp/rust-ptx-linker) for Nvidia - NVPTX GPGPU support. -* `bpf-linker`: use - [`bpf-linker`](https://github.com/alessandrod/bpf-linker) for eBPF support. +* `ptx`: use [`rust-ptx-linker`](https://github.com/denzp/rust-ptx-linker) + for Nvidia NVPTX GPGPU support. +* `bpf`: use [`bpf-linker`](https://github.com/alessandrod/bpf-linker) for eBPF support. * `wasm-ld`: use the [`wasm-ld`](https://lld.llvm.org/WebAssembly.html) executable, a port of LLVM `lld` for WebAssembly. * `ld64.lld`: use the LLVM `lld` executable with the [`-flavor darwin` diff --git a/src/doc/rustc/src/exploit-mitigations.md b/src/doc/rustc/src/exploit-mitigations.md index 172048704..d4e2fc52e 100644 --- a/src/doc/rustc/src/exploit-mitigations.md +++ b/src/doc/rustc/src/exploit-mitigations.md @@ -1,12 +1,12 @@ # Exploit Mitigations -This chapter documents the exploit mitigations supported by the Rust -compiler, and is by no means an extensive survey of the Rust programming -language’s security features. +This chapter documents the exploit mitigations supported by the Rust compiler, +and is by no means an extensive survey of the Rust programming language’s +security features. This chapter is for software engineers working with the Rust programming -language, and assumes prior knowledge of the Rust programming language and -its toolchain. +language, and assumes prior knowledge of the Rust programming language and its +toolchain. ## Introduction @@ -14,8 +14,8 @@ its toolchain. The Rust programming language provides memory[1] and thread[2] safety guarantees via its ownership[3], references and borrowing[4], and slice types[5] features. However, Unsafe Rust[6] introduces unsafe blocks, unsafe -functions and methods, unsafe traits, and new types that are not subject to -the borrowing rules. +functions and methods, unsafe traits, and new types that are not subject to the +borrowing rules. Parts of the Rust standard library are implemented as safe abstractions over unsafe code (and historically have been vulnerable to memory corruption[7]). @@ -23,33 +23,32 @@ Furthermore, the Rust code and documentation encourage creating safe abstractions over unsafe code. This can cause a false sense of security if unsafe code is not properly reviewed and tested. -Unsafe Rust introduces features that do not provide the same memory and -thread safety guarantees. This causes programs or libraries to be -susceptible to memory corruption (CWE-119)[8] and concurrency issues -(CWE-557)[9]. Modern C and C++ compilers provide exploit mitigations to -increase the difficulty to exploit vulnerabilities resulting from these -issues. Therefore, the Rust compiler must also support these exploit -mitigations in order to mitigate vulnerabilities resulting from the use of -Unsafe Rust. This chapter documents these exploit mitigations and how they -apply to Rust. +Unsafe Rust introduces features that do not provide the same memory and thread +safety guarantees. This causes programs or libraries to be susceptible to +memory corruption (CWE-119)[8] and concurrency issues (CWE-557)[9]. Modern C +and C++ compilers provide exploit mitigations to increase the difficulty to +exploit vulnerabilities resulting from these issues. Therefore, the Rust +compiler must also support these exploit mitigations in order to mitigate +vulnerabilities resulting from the use of Unsafe Rust. This chapter documents +these exploit mitigations and how they apply to Rust. -This chapter does not discuss the effectiveness of these exploit mitigations -as they vary greatly depending on several factors besides their design and -implementation, but rather describe what they do, so their effectiveness can -be understood within a given context. +This chapter does not discuss the effectiveness of these exploit mitigations as +they vary greatly depending on several factors besides their design and +implementation, but rather describe what they do, so their effectiveness can be +understood within a given context. ## Exploit mitigations -This section documents the exploit mitigations applicable to the Rust -compiler when building programs for the Linux operating system on the AMD64 -architecture and equivalent.1 +This section documents the exploit mitigations applicable to the Rust compiler +when building programs for the Linux operating system on the AMD64 architecture +and equivalent.1 All examples in this section were built using +nightly builds of the Rust compiler on Debian testing. -The Rust Programming Language currently has no specification. The Rust -compiler (i.e., rustc) is the language reference implementation. All -references to “the Rust compiler” in this chapter refer to the language -reference implementation. +The Rust Programming Language currently has no specification. The Rust compiler +(i.e., rustc) is the language reference implementation. All references to “the +Rust compiler” in this chapter refer to the language reference implementation. Table I \ Summary of exploit mitigations supported by the Rust compiler when building @@ -83,8 +82,8 @@ instructing the dynamic linker to load it similarly to a shared object at a random load address, thus also benefiting from address-space layout randomization (ASLR). This is also referred to as “full ASLR”. -The Rust compiler supports position-independent executable, and enables it -by default since version 0.12.0 (2014-10-09)[10]–[13]. +The Rust compiler supports position-independent executable, and enables it by +default since version 0.12.0 (2014-10-09)[10]–[13]. ```text $ readelf -h target/release/hello-rust | grep Type: @@ -93,8 +92,7 @@ $ readelf -h target/release/hello-rust | grep Type: Fig. 1. Checking if an executable is a position-independent executable. An executable with an object type of `ET_DYN` (i.e., shared object) and not -`ET_EXEC` (i.e., executable) is a position-independent executable (see Fig. -1). +`ET_EXEC` (i.e., executable) is a position-independent executable (see Fig. 1). ### Integer overflow checks @@ -104,8 +102,11 @@ behavior (which may cause vulnerabilities) by checking for results of signed and unsigned integer computations that cannot be represented in their type, resulting in an overflow or wraparound. -The Rust compiler supports integer overflow checks, and enables it when -debug assertions are enabled since version 1.1.0 (2015-06-25)[14]–[20]. +The Rust compiler supports integer overflow checks, and enables it when debug +assertions are enabled since version 1.0.0 (2015-05-15)[14]–[17], but support +for it was not completed until version 1.1.0 (2015-06-25)[16]. An option to +control integer overflow checks was later stabilized in version 1.17.0 +(2017-04-27)[18]–[20]. ```compile_fail fn main() { @@ -136,21 +137,21 @@ u: 0 Fig. 4. Build and execution of hello-rust-integer with debug assertions disabled. -Integer overflow checks are enabled when debug assertions are enabled (see -Fig. 3), and disabled when debug assertions are disabled (see Fig. 4). To -enable integer overflow checks independently, use the option to control -integer overflow checks, scoped attributes, or explicit checking methods -such as `checked_add`2. -It is recommended that explicit wrapping methods such as `wrapping_add` be -used when wrapping semantics are intended, and that explicit checking and -wrapping methods always be used when using Unsafe Rust. +It is recommended that explicit wrapping methods such as `wrapping_add` be used +when wrapping semantics are intended, and that explicit checking and wrapping +methods always be used when using Unsafe Rust. -2\. See [the `u32` docs](../std/primitive.u32.html) -for more information on the checked, overflowing, saturating, and wrapping -methods (using u32 as an example). ↩ +2\. See [the `u32` docs](../std/primitive.u32.html) for more +information on the checked, overflowing, saturating, and wrapping methods +(using u32 as an example). ↩ ### Non-executable memory regions @@ -158,17 +159,16 @@ class="reversefootnote" role="doc-backlink">↩ Non-executable memory regions increase the difficulty of exploitation by limiting the memory regions that can be used to execute arbitrary code. Most modern processors provide support for the operating system to mark memory -regions as non executable, but it was previously emulated by software, such -as in grsecurity/PaX's -[PAGEEXEC](https://pax.grsecurity.net/docs/pageexec.txt) and -[SEGMEXEC](https://pax.grsecurity.net/docs/segmexec.txt), on processors that -did not provide support for it. This is also known as “No Execute (NX) Bit”, -“Execute Disable (XD) Bit”, “Execute Never (XN) Bit”, and others. +regions as non executable, but it was previously emulated by software, such as +in grsecurity/PaX’s [PAGEEXEC](https://pax.grsecurity.net/docs/pageexec.txt) +and [SEGMEXEC](https://pax.grsecurity.net/docs/segmexec.txt), on processors +that did not provide support for it. This is also known as “No Execute (NX) +Bit”, “Execute Disable (XD) Bit”, “Execute Never (XN) Bit”, and others. The Rust compiler supports non-executable memory regions, and enables it by -default since its initial release, version 0.1 (2012-01-20)[21], [22], but -has regressed since then[23]–[25], and enforced by default since version -1.8.0 (2016-04-14)[25]. +default since its initial release, version 0.1 (2012-01-20)[21], [22], but has +regressed since then[23]–[25], and enforced by default since version 1.8.0 +(2016-04-14)[25]. ```text $ readelf -l target/release/hello-rust | grep -A 1 GNU_STACK @@ -178,9 +178,9 @@ $ readelf -l target/release/hello-rust | grep -A 1 GNU_STACK Fig. 5. Checking if non-executable memory regions are enabled for a given binary. -The presence of an element of type `PT_GNU_STACK` in the program header -table with the `PF_X` (i.e., executable) flag unset indicates non-executable -memory regions3 are enabled for a given binary (see Fig. 5). Conversely, the presence of an element of type `PT_GNU_STACK` in the program header table with the `PF_X` flag set or the absence of an element of type @@ -196,38 +196,40 @@ class="reversefootnote" role="doc-backlink">↩ Stack clashing protection protects the stack from overlapping with another memory region—allowing arbitrary data in both to be overwritten using each -other—by reading from the stack pages as the stack grows to cause a page -fault when attempting to read from the guard page/region. This is also -referred to as “stack probes” or “stack probing”. +other—by reading from the stack pages as the stack grows to cause a page fault +when attempting to read from the guard page/region. This is also referred to as +“stack probes” or “stack probing”. The Rust compiler supports stack clashing protection via stack probing, and enables it by default since version 1.20.0 (2017-08-31)[26]–[29]. -![Screenshot of IDA Pro listing cross references to __rust_probestack in hello-rust.](images/image1.png "Cross references to __rust_probestack in hello-rust.") -Fig. 6. IDA Pro listing cross references to `__rust_probestack` in -hello-rust. - ```rust -fn hello() { - println!("Hello, world!"); +fn main() { + let v: [u8; 16384] = [1; 16384]; + let first = &v[0]; + println!("The first element is: {first}"); } +``` +Fig. 6. hello-rust-stack-probe-1 program. +![Screenshot of IDA Pro listing the "unrolled loop" stack probe variant in modified hello-rust.](images/image1.png "The \"unrolled loop\" stack probe variant in modified hello-rust.") +Fig. 7. The "unrolled loop" stack probe variant in modified hello-rust. + +```rust fn main() { - let _: [u64; 1024] = [0; 1024]; - hello(); + let v: [u8; 65536] = [1; 65536]; + let first = &v[0]; + println!("The first element is: {first}"); } ``` -Fig 7. Modified hello-rust. +Fig. 8. hello-rust-stack-probe-2 program. -![Screenshot of IDA Pro listing cross references to __rust_probestack in modified hello-rust.](images/image2.png "Cross references to __rust_probestack in modified hello-rust.") -Fig. 8. IDA Pro listing cross references to `__rust_probestack` in modified -hello-rust. +![Screenshot of IDA Pro listing the "standard loop" stack probe variant in modified hello-rust.](images/image2.png "The \"standard loop\" stack probe variant in modified hello-rust.") +Fig. 9. The "standard loop" stack probe variant in modified hello-rust. -To check if stack clashing protection is enabled for a given binary, search -for cross references to `__rust_probestack`. The `__rust_probestack` is -called in the prologue of functions whose stack size is larger than a page -size (see Fig. 6), and can be forced for illustration purposes by modifying -the hello-rust example as seen in Fig. 7 and Fig. 8. +To check if stack clashing protection is enabled for a given binary, look for +any of the two stack probe variants in the prologue of functions whose stack +size is larger than a page size (see Figs. 6–9). ### Read-only relocations and immediate binding @@ -246,21 +248,20 @@ $ readelf -l target/release/hello-rust | grep GNU_RELRO ``` Fig. 9. Checking if read-only relocations is enabled for a given binary. -The presence of an element of type `PT_GNU_RELRO` in the program header -table indicates read-only relocations are enabled for a given binary (see -Fig. 9). Conversely, the absence of an element of type `PT_GNU_RELRO` in the -program header table indicates read-only relocations are not enabled for a -given binary. +The presence of an element of type `PT_GNU_RELRO` in the program header table +indicates read-only relocations are enabled for a given binary (see Fig. 9). +Conversely, the absence of an element of type `PT_GNU_RELRO` in the program +header table indicates read-only relocations are not enabled for a given +binary. **Immediate binding** protects additional segments containing relocations -(i.e., `.got.plt`) from being overwritten by instructing the dynamic linker -to perform all relocations before transferring control to the program during -startup, so all segments containing relocations can be marked read only -(when combined with read-only relocations). This is also referred to as -“full RELRO”. +(i.e., `.got.plt`) from being overwritten by instructing the dynamic linker to +perform all relocations before transferring control to the program during +startup, so all segments containing relocations can be marked read only (when +combined with read-only relocations). This is also referred to as “full RELRO”. -The Rust compiler supports immediate binding, and enables it by default -since version 1.21.0 (2017-10-12)[30], [31]. +The Rust compiler supports immediate binding, and enables it by default since +version 1.21.0 (2017-10-12)[30], [31]. ```text $ readelf -d target/release/hello-rust | grep BIND_NOW @@ -270,16 +271,15 @@ Fig. 10. Checking if immediate binding is enabled for a given binary. The presence of an element with the `DT_BIND_NOW` tag and the `DF_BIND_NOW` flag4 in the dynamic section indicates immediate -binding is enabled for a given binary (see Fig. 10). Conversely, the absence -of an element with the `DT_BIND_NOW` tag and the `DF_BIND_NOW` flag in the -dynamic section indicates immediate binding is not enabled for a given -binary. +class="footnote">4 in the dynamic section indicates immediate binding +is enabled for a given binary (see Fig. 10). Conversely, the absence of an +element with the `DT_BIND_NOW` tag and the `DF_BIND_NOW` flag in the dynamic +section indicates immediate binding is not enabled for a given binary. The presence of both an element of type `PT_GNU_RELRO` in the program header -table and of an element with the `DT_BIND_NOW` tag and the `DF_BIND_NOW` -flag in the dynamic section indicates full RELRO is enabled for a given -binary (see Fig. 9 and Fig. 10). +table and of an element with the `DT_BIND_NOW` tag and the `DF_BIND_NOW` flag +in the dynamic section indicates full RELRO is enabled for a given binary (see +Figs. 9–10). 4\. And the `DF_1_NOW` flag for some link editors. ↩ @@ -287,26 +287,24 @@ href="#fnref:4" class="reversefootnote" role="doc-backlink">↩ ### Heap corruption protection -Heap corruption protection protects memory allocated dynamically by -performing several checks, such as checks for corrupted links between list -elements, invalid pointers, invalid sizes, double/multiple “frees” of the -same memory allocated, and many corner cases of these. These checks are -implementation specific, and vary per allocator. +Heap corruption protection protects memory allocated dynamically by performing +several checks, such as checks for corrupted links between list elements, +invalid pointers, invalid sizes, double/multiple “frees” of the same memory +allocated, and many corner cases of these. These checks are implementation +specific, and vary per allocator. [ARM Memory Tagging Extension (MTE)](https://community.arm.com/developer/ip-products/processors/b/processors-ip-blog/posts/enhancing-memory-safety), -when available, will provide hardware assistance for a probabilistic -mitigation to detect memory safety violations by tagging memory allocations, -and automatically checking that the correct tag is used on every memory -access. +when available, will provide hardware assistance for a probabilistic mitigation +to detect memory safety violations by tagging memory allocations, and +automatically checking that the correct tag is used on every memory access. Rust’s default allocator has historically been -[jemalloc](http://jemalloc.net/), and it has long been the cause of issues -and the subject of much discussion[32]–[38]. Consequently, it has been -removed as the default allocator in favor of the operating system’s standard -C library default allocator5 since version 1.32.0 -(2019-01-17)[39]. +[jemalloc](http://jemalloc.net/), and it has long been the cause of issues and +the subject of much discussion[32]–[38]. Consequently, it has been removed as +the default allocator in favor of the operating system’s standard C library +default allocator5 since version 1.32.0 (2019-01-17)[39]. ```rust,no_run fn main() { @@ -330,8 +328,7 @@ $ cargo run free(): invalid next size (normal) Aborted ``` -Fig. 12. Build and execution of hello-rust-heap with debug assertions -enabled. +Fig. 12. Build and execution of hello-rust-heap with debug assertions enabled. ```text $ cargo run --release @@ -341,47 +338,41 @@ $ cargo run --release free(): invalid next size (normal) Aborted ``` -Fig. 13. Build and execution of hello-rust-heap with debug assertions -disabled. +Fig. 13. Build and execution of hello-rust-heap with debug assertions disabled. -Heap corruption checks are being performed when using the default allocator -(i.e., the GNU Allocator) as seen in Fig. 12 and Fig. 13. +Heap corruption checks are performed when using the default allocator (i.e., +the GNU Allocator) (see Figs. 12–13). 5\. Linux's standard C library default allocator is the GNU -Allocator, which is derived from ptmalloc (pthreads malloc) by Wolfram -Gloger, which in turn is derived from dlmalloc (Doug Lea malloc) by Doug -Lea. ↩ +Allocator, which is derived from ptmalloc (pthreads malloc) by Wolfram Gloger, +which in turn is derived from dlmalloc (Doug Lea malloc) by Doug Lea. ↩ ### Stack smashing protection -Stack smashing protection protects programs from stack-based buffer -overflows by inserting a random guard value between local variables and the -saved return instruction pointer, and checking if this value has changed -when returning from a function. This is also known as “Stack Protector” or -“Stack Smashing Protector (SSP)”. +Stack smashing protection protects programs from stack-based buffer overflows +by inserting a random guard value between local variables and the saved return +instruction pointer, and checking if this value has changed when returning from +a function. This is also known as “Stack Protector” or “Stack Smashing +Protector (SSP)”. -The Rust compiler supports stack smashing protection on nightly builds[42]. +The Rust compiler supports stack smashing protection on nightly builds[40]. ![Screenshot of IDA Pro listing cross references to __stack_chk_fail in hello-rust.](images/image3.png "Cross references to __stack_chk_fail in hello-rust.") -Fig. 14. IDA Pro listing cross references to `__stack_chk_fail` in -hello-rust. +Fig. 14. IDA Pro listing cross references to `__stack_chk_fail` in hello-rust. -To check if stack smashing protection is enabled for a given binary, search -for cross references to `__stack_chk_fail`. The presence of these -cross-references in Rust-compiled code (e.g., `hello_rust::main`) indicates -that the stack smashing protection is enabled (see Fig. 14). +To check if stack smashing protection is enabled for a given binary, search for +cross references to `__stack_chk_fail` (see Fig. 14). ### Forward-edge control flow protection -Forward-edge control flow protection protects programs from having its -control flow changed/hijacked by performing checks to ensure that -destinations of indirect branches are one of their valid destinations in the -control flow graph. The comprehensiveness of these checks vary per -implementation. This is also known as “forward-edge control flow integrity -(CFI)”. +Forward-edge control flow protection protects programs from having its control +flow changed/hijacked by performing checks to ensure that destinations of +indirect branches are one of their valid destinations in the control flow +graph. The comprehensiveness of these checks vary per implementation. This is +also known as “forward-edge control flow integrity (CFI)”. Newer processors provide hardware assistance for forward-edge control flow protection, such as ARM Branch Target Identification (BTI), ARM Pointer @@ -394,22 +385,19 @@ commercially available [grsecurity/PaX Reuse Attack Protector (RAP)](https://grsecurity.net/rap_faq). The Rust compiler supports forward-edge control flow protection on nightly -builds[40]-[41] 6. ```text -$ readelf -s -W target/debug/rust-cfi | grep "\.cfi" - 12: 0000000000005170 46 FUNC LOCAL DEFAULT 14 _RNvCsjaOHoaNjor6_8rust_cfi7add_one.cfi - 15: 00000000000051a0 16 FUNC LOCAL DEFAULT 14 _RNvCsjaOHoaNjor6_8rust_cfi7add_two.cfi - 17: 0000000000005270 396 FUNC LOCAL DEFAULT 14 _RNvCsjaOHoaNjor6_8rust_cfi4main.cfi -... +$ readelf -s -W target/release/hello-rust | grep "\.cfi" + 5: 0000000000006480 657 FUNC LOCAL DEFAULT 15 _ZN10hello_rust4main17h4e359f1dcd627c83E.cfi ``` -Fig. 15. Checking if LLVM CFI is enabled for a given binary[41]. +Fig. 15. Checking if LLVM CFI is enabled for a given binary. The presence of symbols suffixed with ".cfi" or the `__cfi_init` symbol (and -references to `__cfi_check`) indicates that LLVM CFI (i.e., forward-edge control -flow protection) is enabled for a given binary. Conversely, the absence of -symbols suffixed with ".cfi" or the `__cfi_init` symbol (and references to +references to `__cfi_check`) indicates that LLVM CFI (i.e., forward-edge +control flow protection) is enabled for a given binary. Conversely, the absence +of symbols suffixed with ".cfi" or the `__cfi_init` symbol (and references to `__cfi_check`) indicates that LLVM CFI is not enabled for a given binary (see Fig. 15). @@ -421,48 +409,47 @@ class="reversefootnote" role="doc-backlink">↩ ### Backward-edge control flow protection **Shadow stack** protects saved return instruction pointers from being -overwritten by storing a copy of them on a separate (shadow) stack, and -using these copies as authoritative values when returning from functions. -This is also known as “ShadowCallStack” and “Return Flow Guard”, and is -considered an implementation of backward-edge control flow protection (or -“backward-edge CFI”). +overwritten by storing a copy of them on a separate (shadow) stack, and using +these copies as authoritative values when returning from functions. This is +also known as “ShadowCallStack” and “Return Flow Guard”, and is considered an +implementation of backward-edge control flow protection (or “backward-edge +CFI”). **Safe stack** protects not only the saved return instruction pointers, but -also register spills and some local variables from being overwritten by -storing unsafe variables, such as large arrays, on a separate (unsafe) -stack, and using these unsafe variables on the separate stack instead. This -is also known as “SafeStack”, and is also considered an implementation of -backward-edge control flow protection. +also register spills and some local variables from being overwritten by storing +unsafe variables, such as large arrays, on a separate (unsafe) stack, and using +these unsafe variables on the separate stack instead. This is also known as +“SafeStack”, and is also considered an implementation of backward-edge control +flow protection. -Both shadow and safe stack are intended to be a more comprehensive -alternatives to stack smashing protection as they protect the saved return -instruction pointers (and other data in the case of safe stack) from -arbitrary writes and non-linear out-of-bounds writes. +Both shadow and safe stack are intended to be a more comprehensive alternatives +to stack smashing protection as they protect the saved return instruction +pointers (and other data in the case of safe stack) from arbitrary writes and +non-linear out-of-bounds writes. Newer processors provide hardware assistance for backward-edge control flow -protection, such as ARM Pointer Authentication, and Intel Shadow Stack as -part of Intel CET. +protection, such as ARM Pointer Authentication, and Intel Shadow Stack as part +of Intel CET. -The Rust compiler supports shadow stack for aarch64 only -7 -on nightly Rust compilers [43]-[44]. Safe stack is available on nightly -Rust compilers [45]-[46]. +The Rust compiler supports shadow stack for the AArch64 architecture7on +nightly builds[43]-[44], and also supports safe stack on nightly +builds[45]-[46]. ```text $ readelf -s target/release/hello-rust | grep __safestack_init - 1177: 00000000000057b0 444 FUNC GLOBAL DEFAULT 9 __safestack_init + 678: 0000000000008c80 426 FUNC GLOBAL DEFAULT 15 __safestack_init ``` Fig. 16. Checking if LLVM SafeStack is enabled for a given binary. -The presence of the `__safestack_init` symbol indicates that LLVM SafeStack -is enabled for a given binary (see Fig. 16). Conversely, the absence of the -`__safestack_init` symbol indicates that LLVM SafeStack is not enabled for a -given binary. +The presence of the `__safestack_init` symbol indicates that LLVM SafeStack is +enabled for a given binary. Conversely, the absence of the `__safestack_init` +symbol indicates that LLVM SafeStack is not enabled for a given binary (see +Fig. 16). -7\. The shadow stack implementation for the AMD64 -architecture and equivalent in LLVM was removed due to performance and -security issues. ↩ +7\. The shadow stack implementation for the AMD64 architecture +and equivalent in LLVM was removed due to performance and security issues. ↩ ## Appendix @@ -470,29 +457,28 @@ role="doc-backlink">↩ As of the latest version of the [Linux Standard Base (LSB) Core Specification](https://refspecs.linuxfoundation.org/LSB_5.0.0/LSB-Core-generic/LSB-Core-generic/progheader.html), the `PT_GNU_STACK` program header indicates whether the stack should be -executable, and the absence of this header indicates that the stack should -be executable. However, the Linux kernel currently sets the -`READ_IMPLIES_EXEC` personality upon loading any executable with the -`PT_GNU_STACK` program header and the `PF_X `flag set or with the absence of -this header, resulting in not only the stack, but also all readable virtual -memory mappings being executable. +executable, and the absence of this header indicates that the stack should be +executable. However, the Linux kernel currently sets the `READ_IMPLIES_EXEC` +personality upon loading any executable with the `PT_GNU_STACK` program header +and the `PF_X` flag set or with the absence of this header, resulting in not +only the stack, but also all readable virtual memory mappings being executable. An attempt to fix this [was made in 2012](https://lore.kernel.org/lkml/f298f914-2239-44e4-8aa1-a51282e7fac0@zmail15.collab.prod.int.phx2.redhat.com/), and another [was made in 2020](https://lore.kernel.org/kernel-hardening/20200327064820.12602-1-keescook@chromium.org/). The former never landed, and the latter partially fixed it, but introduced -other issues—the absence of the `PT_GNU_STACK` program header still causes -not only the stack, but also all readable virtual memory mappings to be -executable in some architectures, such as IA-32 and equivalent (or causes -the stack to be non-executable in some architectures, such as AMD64 and -equivalent, contradicting the LSB). +other issues—the absence of the `PT_GNU_STACK` program header still causes not +only the stack, but also all readable virtual memory mappings to be executable +in some architectures, such as IA-32 and equivalent (or causes the stack to be +non-executable in some architectures, such as AMD64 and equivalent, +contradicting the LSB). -The `READ_IMPLIES_EXEC` personality needs to be completely separated from -the `PT_GNU_STACK` program header by having a separate option for it (or -setarch -X could just be used whenever `READ_IMPLIES_EXEC` is needed), and -the absence of the `PT_GNU_STACK` program header needs to have more secure -defaults (unrelated to `READ_IMPLIES_EXEC`). +The `READ_IMPLIES_EXEC` personality needs to be completely separated from the +`PT_GNU_STACK` program header by having a separate option for it (or setarch -X +could just be used whenever `READ_IMPLIES_EXEC` is needed), and the absence of +the `PT_GNU_STACK` program header needs to have more secure defaults (unrelated +to `READ_IMPLIES_EXEC`). ## References @@ -576,19 +562,19 @@ defaults (unrelated to `READ_IMPLIES_EXEC`). 25. A. Clark. “Explicitly disable stack execution on linux and bsd #30859.” GitHub. . -26. “Replace stack overflow checking with stack probes #16012.” GitHub. +26. Zoxc. “Replace stack overflow checking with stack probes #16012.” GitHub. . -27. B. Striegel. “Extend stack probe support to non-tier-1 platforms, and - clarify policy for mitigating LLVM-dependent unsafety #43241.” GitHub. - . - -28. A. Crichton. “rustc: Implement stack probes for x86 #42816.” GitHub. +27. A. Crichton. “rustc: Implement stack probes for x86 #42816.” GitHub. . -29. A. Crichton. “Add \_\_rust\_probestack intrinsic #175.” GitHub. +28. A. Crichton. “Add \_\_rust\_probestack intrinsic #175.” GitHub. . +29. S. Guelton, S. Ledru, J. Stone. “Bringing Stack Clash Protection to Clang / + X86 — the Open Source Way.” The LLVM Project Blog. + . + 30. B. Anderson. “Consider applying -Wl,-z,relro or -Wl,-z,relro,-z,now by default #29877.” GitHub. . @@ -621,16 +607,16 @@ defaults (unrelated to `READ_IMPLIES_EXEC`). 39. A. Crichton. “Remove the alloc\_jemalloc crate #55238.” GitHub. . -40. R. de C Valle. “Tracking Issue for LLVM Control Flow Integrity (CFI) Support +40. bbjornse. “Add codegen option for using LLVM stack smash protection #84197.” + GitHub. + +41. R. de C. Valle. “Tracking Issue for LLVM Control Flow Integrity (CFI) Support for Rust #89653.” GitHub. . -41. “ControlFlowIntegrity.” The Rust Unstable Book. +42. “ControlFlowIntegrity.” The Rust Unstable Book. [https://doc.rust-lang.org/unstable-book/compiler-flags/sanitizer.html#controlflowintegrity](../unstable-book/compiler-flags/sanitizer.html#controlflowintegrity). -42. bbjornse. “add codegen option for using LLVM stack smash protection #84197.” - GitHub. - -43. ivanloz. “Add support for LLVM ShadowCallStack. #98208.” GitHub. +43. I. Lozano. “Add support for LLVM ShadowCallStack #98208.” GitHub. . 44. “ShadowCallStack.” The Rust Unstable Book. diff --git a/src/doc/rustc/src/images/image1.png b/src/doc/rustc/src/images/image1.png index ee2d3fd4f..0da45e566 100644 Binary files a/src/doc/rustc/src/images/image1.png and b/src/doc/rustc/src/images/image1.png differ diff --git a/src/doc/rustc/src/images/image2.png b/src/doc/rustc/src/images/image2.png index 03061e1f0..a9cf23f87 100644 Binary files a/src/doc/rustc/src/images/image2.png and b/src/doc/rustc/src/images/image2.png differ diff --git a/src/doc/rustc/src/images/image3.png b/src/doc/rustc/src/images/image3.png index ef02c605e..844a2fe67 100644 Binary files a/src/doc/rustc/src/images/image3.png and b/src/doc/rustc/src/images/image3.png differ diff --git a/src/doc/rustc/src/platform-support.md b/src/doc/rustc/src/platform-support.md index ff831a205..907e9c59f 100644 --- a/src/doc/rustc/src/platform-support.md +++ b/src/doc/rustc/src/platform-support.md @@ -33,9 +33,9 @@ All tier 1 targets with host tools support the full standard library. target | notes -------|------- `aarch64-unknown-linux-gnu` | ARM64 Linux (kernel 4.1, glibc 2.17+) [^missing-stack-probes] -`i686-pc-windows-gnu` | 32-bit MinGW (Windows 7+) [^windows-support] -`i686-pc-windows-msvc` | 32-bit MSVC (Windows 7+) [^windows-support] -`i686-unknown-linux-gnu` | 32-bit Linux (kernel 3.2+, glibc 2.17+) +`i686-pc-windows-gnu` | 32-bit MinGW (Windows 7+) [^windows-support] [^x86_32-floats-return-ABI] +`i686-pc-windows-msvc` | 32-bit MSVC (Windows 7+) [^windows-support] [^x86_32-floats-return-ABI] +`i686-unknown-linux-gnu` | 32-bit Linux (kernel 3.2+, glibc 2.17+) [^x86_32-floats-return-ABI] `x86_64-apple-darwin` | 64-bit macOS (10.12+, Sierra+) `x86_64-pc-windows-gnu` | 64-bit MinGW (Windows 7+) [^windows-support] `x86_64-pc-windows-msvc` | 64-bit MSVC (Windows 7+) [^windows-support] @@ -47,7 +47,10 @@ target | notes [^windows-support]: Only Windows 10 currently undergoes automated testing. Earlier versions of Windows rely on testing and support from the community. +[^x86_32-floats-return-ABI]: Due to limitations of the C ABI, floating-point support on `i686` targets is non-compliant: floating-point return values are passed via an x87 register, so NaN payload bits can be lost. See [issue #114479][x86-32-float-issue]. + [77071]: https://github.com/rust-lang/rust/issues/77071 +[x86-32-float-issue]: https://github.com/rust-lang/rust/issues/114479 ## Tier 1 @@ -90,10 +93,6 @@ target | notes `arm-unknown-linux-gnueabihf` | ARMv6 Linux, hardfloat (kernel 3.2, glibc 2.17) `armv7-unknown-linux-gnueabihf` | ARMv7-A Linux, hardfloat (kernel 3.2, glibc 2.17) [`loongarch64-unknown-linux-gnu`](platform-support/loongarch-linux.md) | LoongArch64 Linux, LP64D ABI (kernel 5.19, glibc 2.36) -`mips-unknown-linux-gnu` | MIPS Linux (kernel 4.4, glibc 2.23) -`mips64-unknown-linux-gnuabi64` | MIPS64 Linux, n64 ABI (kernel 4.4, glibc 2.23) -`mips64el-unknown-linux-gnuabi64` | MIPS64 (LE) Linux, n64 ABI (kernel 4.4, glibc 2.23) -`mipsel-unknown-linux-gnu` | MIPS (LE) Linux (kernel 4.4, glibc 2.23) `powerpc-unknown-linux-gnu` | PowerPC Linux (kernel 3.2, glibc 2.17) `powerpc64-unknown-linux-gnu` | PPC64 Linux (kernel 3.2, glibc 2.17) `powerpc64le-unknown-linux-gnu` | PPC64LE Linux (kernel 3.10, glibc 2.17) @@ -150,19 +149,16 @@ target | std | notes `armv7r-none-eabi` | * | Bare ARMv7-R `armv7r-none-eabihf` | * | Bare ARMv7-R, hardfloat `asmjs-unknown-emscripten` | ✓ | asm.js via Emscripten -`i586-pc-windows-msvc` | * | 32-bit Windows w/o SSE -`i586-unknown-linux-gnu` | ✓ | 32-bit Linux w/o SSE (kernel 3.2, glibc 2.17) -`i586-unknown-linux-musl` | ✓ | 32-bit Linux w/o SSE, MUSL -[`i686-linux-android`](platform-support/android.md) | ✓ | 32-bit x86 Android -`i686-unknown-freebsd` | ✓ | 32-bit FreeBSD -`i686-unknown-linux-musl` | ✓ | 32-bit Linux with MUSL +`i586-pc-windows-msvc` | * | 32-bit Windows w/o SSE [^x86_32-floats-x87] +`i586-unknown-linux-gnu` | ✓ | 32-bit Linux w/o SSE (kernel 3.2, glibc 2.17) [^x86_32-floats-x87] +`i586-unknown-linux-musl` | ✓ | 32-bit Linux w/o SSE, MUSL [^x86_32-floats-x87] +[`i586-unknown-netbsd`](platform-support/netbsd.md) | ✓ | 32-bit x86, restricted to Pentium +[`i686-linux-android`](platform-support/android.md) | ✓ | 32-bit x86 Android [^x86_32-floats-return-ABI] +`i686-unknown-freebsd` | ✓ | 32-bit FreeBSD [^x86_32-floats-return-ABI] +`i686-unknown-linux-musl` | ✓ | 32-bit Linux with MUSL [^x86_32-floats-return-ABI] [`i686-unknown-uefi`](platform-support/unknown-uefi.md) | * | 32-bit UEFI [`loongarch64-unknown-none`](platform-support/loongarch-none.md) | * | | LoongArch64 Bare-metal (LP64D ABI) [`loongarch64-unknown-none-softfloat`](platform-support/loongarch-none.md) | * | | LoongArch64 Bare-metal (LP64S ABI) -`mips-unknown-linux-musl` | ✓ | MIPS Linux with MUSL -`mips64-unknown-linux-muslabi64` | ✓ | MIPS64 Linux, n64 ABI, MUSL -`mips64el-unknown-linux-muslabi64` | ✓ | MIPS64 (LE) Linux, n64 ABI, MUSL -`mipsel-unknown-linux-musl` | ✓ | MIPS (LE) Linux with MUSL [`nvptx64-nvidia-cuda`](platform-support/nvptx64-nvidia-cuda.md) | * | --emit=asm generates PTX code that [runs on NVIDIA GPUs] `riscv32i-unknown-none-elf` | * | Bare RISC-V (RV32I ISA) `riscv32imac-unknown-none-elf` | * | Bare RISC-V (RV32IMAC ISA) @@ -195,6 +191,8 @@ target | std | notes `x86_64-unknown-redox` | ✓ | Redox OS [`x86_64-unknown-uefi`](platform-support/unknown-uefi.md) | * | 64-bit UEFI +[^x86_32-floats-x87]: Floating-point support on `i586` targets is non-compliant: the `x87` registers and instructions used for these targets do not provide IEEE-754-compliant behavior, in particular when it comes to rounding and NaN payload bits. See [issue #114479][x86-32-float-issue]. + [Fortanix ABI]: https://edp.fortanix.com/ ## Tier 3 @@ -220,6 +218,7 @@ target | std | host | notes -------|:---:|:----:|------- `aarch64-apple-ios-macabi` | ? | | Apple Catalyst on ARM64 [`aarch64-apple-tvos`](platform-support/apple-tvos.md) | ? | | ARM64 tvOS +[`aarch64-apple-tvos-sim`](platform-support/apple-tvos.md) | ? | | ARM64 tvOS Simulator [`aarch64-apple-watchos-sim`](platform-support/apple-watchos.md) | ✓ | | ARM64 Apple WatchOS Simulator [`aarch64-kmc-solid_asp3`](platform-support/kmc-solid.md) | ✓ | | ARM64 SOLID with TOPPERS/ASP3 [`aarch64-nintendo-switch-freestanding`](platform-support/aarch64-nintendo-switch-freestanding.md) | * | | ARM64 Nintendo Switch, Horizon @@ -247,7 +246,7 @@ target | std | host | notes `armv6-unknown-freebsd` | ✓ | ✓ | ARMv6 FreeBSD [`armv6-unknown-netbsd-eabihf`](platform-support/netbsd.md) | ✓ | ✓ | ARMv6 NetBSD w/hard-float [`armv6k-nintendo-3ds`](platform-support/armv6k-nintendo-3ds.md) | ? | | ARMv6K Nintendo 3DS, Horizon (Requires devkitARM toolchain) -[`armv7-sony-vita-newlibeabihf`](platform-support/armv7-sony-vita-newlibeabihf.md) | ? | | ARMv7-A Cortex-A9 Sony PlayStation Vita (requires VITASDK toolchain) +[`armv7-sony-vita-newlibeabihf`](platform-support/armv7-sony-vita-newlibeabihf.md) | ✓ | | ARMv7-A Cortex-A9 Sony PlayStation Vita (requires VITASDK toolchain) [`armv7-unknown-linux-ohos`](platform-support/openharmony.md) | ✓ | | ARMv7-A OpenHarmony | [`armv7-unknown-linux-uclibceabi`](platform-support/armv7-unknown-linux-uclibceabi.md) | ✓ | ✓ | ARMv7-A Linux with uClibc, softfloat [`armv7-unknown-linux-uclibceabihf`](platform-support/armv7-unknown-linux-uclibceabihf.md) | ✓ | ? | ARMv7-A Linux with uClibc, hardfloat @@ -262,23 +261,33 @@ target | std | host | notes `avr-unknown-gnu-atmega328` | * | | AVR. Requires `-Z build-std=core` `bpfeb-unknown-none` | * | | BPF (big endian) `bpfel-unknown-none` | * | | BPF (little endian) -`csky-unknown-linux-gnuabiv2` | ✓ | | C-SKY abiv2 Linux(little endian) +`csky-unknown-linux-gnuabiv2` | ✓ | | C-SKY abiv2 Linux (little endian) +`csky-unknown-linux-gnuabiv2hf` | ✓ | | C-SKY abiv2 Linux, hardfloat (little endian) `hexagon-unknown-linux-musl` | ? | | -`i386-apple-ios` | ✓ | | 32-bit x86 iOS -[`i586-pc-nto-qnx700`](platform-support/nto-qnx.md) | * | | 32-bit x86 QNX Neutrino 7.0 RTOS | -`i686-apple-darwin` | ✓ | ✓ | 32-bit macOS (10.12+, Sierra+) -`i686-pc-windows-msvc` | * | | 32-bit Windows XP support -[`i686-pc-windows-gnullvm`](platform-support/pc-windows-gnullvm.md) | ✓ | ✓ | -`i686-unknown-haiku` | ✓ | ✓ | 32-bit Haiku -[`i686-unknown-hurd-gnu`](platform-support/hurd.md) | ✓ | ✓ | 32-bit GNU/Hurd -[`i686-unknown-netbsd`](platform-support/netbsd.md) | ✓ | ✓ | NetBSD/i386 with SSE2 -[`i686-unknown-openbsd`](platform-support/openbsd.md) | ✓ | ✓ | 32-bit OpenBSD -`i686-uwp-windows-gnu` | ? | | -`i686-uwp-windows-msvc` | ? | | -`i686-wrs-vxworks` | ? | | +`i386-apple-ios` | ✓ | | 32-bit x86 iOS [^x86_32-floats-return-ABI] +[`i586-pc-nto-qnx700`](platform-support/nto-qnx.md) | * | | 32-bit x86 QNX Neutrino 7.0 RTOS [^x86_32-floats-return-ABI] +`i686-apple-darwin` | ✓ | ✓ | 32-bit macOS (10.12+, Sierra+) [^x86_32-floats-return-ABI] +`i686-pc-windows-msvc` | * | | 32-bit Windows XP support [^x86_32-floats-return-ABI] +[`i686-pc-windows-gnullvm`](platform-support/pc-windows-gnullvm.md) | ✓ | ✓ | [^x86_32-floats-return-ABI] +`i686-unknown-haiku` | ✓ | ✓ | 32-bit Haiku [^x86_32-floats-return-ABI] +[`i686-unknown-hurd-gnu`](platform-support/hurd.md) | ✓ | ✓ | 32-bit GNU/Hurd [^x86_32-floats-return-ABI] +[`i686-unknown-netbsd`](platform-support/netbsd.md) | ✓ | ✓ | NetBSD/i386 with SSE2 [^x86_32-floats-return-ABI] +[`i686-unknown-openbsd`](platform-support/openbsd.md) | ✓ | ✓ | 32-bit OpenBSD [^x86_32-floats-return-ABI] +`i686-uwp-windows-gnu` | ? | | [^x86_32-floats-return-ABI] +`i686-uwp-windows-msvc` | ? | | [^x86_32-floats-return-ABI] +`i686-wrs-vxworks` | ? | | [^x86_32-floats-return-ABI] [`m68k-unknown-linux-gnu`](platform-support/m68k-unknown-linux-gnu.md) | ? | | Motorola 680x0 Linux +`mips-unknown-linux-gnu` | ✓ | ✓ | MIPS Linux (kernel 4.4, glibc 2.23) +`mips-unknown-linux-musl` | ✓ | | MIPS Linux with musl libc `mips-unknown-linux-uclibc` | ✓ | | MIPS Linux with uClibc [`mips64-openwrt-linux-musl`](platform-support/mips64-openwrt-linux-musl.md) | ? | | MIPS64 for OpenWrt Linux MUSL +`mips64-unknown-linux-gnuabi64` | ✓ | ✓ | MIPS64 Linux, N64 ABI (kernel 4.4, glibc 2.23) +`mips64-unknown-linux-muslabi64` | ✓ | | MIPS64 Linux, N64 ABI, musl libc +`mips64el-unknown-linux-gnuabi64` | ✓ | ✓ | MIPS64 (little endian) Linux, N64 ABI (kernel 4.4, glibc 2.23) +`mips64el-unknown-linux-muslabi64` | ✓ | | MIPS64 (little endian) Linux, N64 ABI, musl libc +`mipsel-unknown-linux-gnu` | ✓ | ✓ | MIPS (little endian) Linux (kernel 4.4, glibc 2.23) +`mipsel-unknown-linux-musl` | ✓ | | MIPS (little endian) Linux with musl libc +[`mipsel-unknown-netbsd`](platform-support/netbsd.md) | ✓ | ✓ | 32-bit MIPS (LE), requires mips32 cpu support `mipsel-sony-psp` | * | | MIPS (LE) Sony PlayStation Portable (PSP) [`mipsel-sony-psx`](platform-support/mipsel-sony-psx.md) | * | | MIPS (LE) Sony PlayStation 1 (PSX) `mipsel-unknown-linux-uclibc` | ✓ | | MIPS (LE) Linux with uClibc @@ -301,7 +310,7 @@ target | std | host | notes `powerpc64-wrs-vxworks` | ? | | `powerpc64le-unknown-linux-musl` | ? | | [`powerpc64-unknown-openbsd`](platform-support/openbsd.md) | ✓ | ✓ | OpenBSD/powerpc64 -`powerpc64-ibm-aix` | ? | | 64-bit AIX (7.2 and newer) +[`powerpc64-ibm-aix`](platform-support/aix.md) | ? | | 64-bit AIX (7.2 and newer) `riscv32gc-unknown-linux-gnu` | | | RISC-V Linux (kernel 5.4, glibc 2.33) `riscv32gc-unknown-linux-musl` | | | RISC-V Linux (kernel 5.4, musl + RISCV32 support patches) `riscv32im-unknown-none-elf` | * | | Bare RISC-V (RV32IM ISA) diff --git a/src/doc/rustc/src/platform-support/aarch64-unknown-teeos.md b/src/doc/rustc/src/platform-support/aarch64-unknown-teeos.md index f8cd92f92..9233a36db 100644 --- a/src/doc/rustc/src/platform-support/aarch64-unknown-teeos.md +++ b/src/doc/rustc/src/platform-support/aarch64-unknown-teeos.md @@ -58,7 +58,7 @@ To build a rust toolchain, create a `config.toml` with the following contents: ```toml profile = "compiler" -changelog-seen = 2 +change-id = 115898 [build] sanitizers = true diff --git a/src/doc/rustc/src/platform-support/aix.md b/src/doc/rustc/src/platform-support/aix.md new file mode 100644 index 000000000..c3ce71a18 --- /dev/null +++ b/src/doc/rustc/src/platform-support/aix.md @@ -0,0 +1,26 @@ +# `powerpc64-ibm-aix` + +**Tier: 3** + +Rust for AIX operating system, currently only 64-bit PowerPC is supported. + +## Target maintainers + +- QIU Chaofan `qiucofan@cn.ibm.com`, https://github.com/ecnelises +- Kai LUO, `lkail@cn.ibm.com`, https://github.com/bzEq + +## Requirements + +This target supports host tools, std and alloc. This target cannot be cross-compiled as for now, mainly because of the unavailability of system linker on other platforms. + +Binary built for this target is expected to run on Power7 or newer CPU, and AIX 7.2 or newer version. + +Binary format of this platform is [XCOFF](https://www.ibm.com/docs/en/aix/7.2?topic=formats-xcoff-object-file-format). Archive file format is ['AIX big format'](https://www.ibm.com/docs/en/aix/7.2?topic=formats-ar-file-format-big). + +## Testing + +This target supports running test suites natively, but it's not available to cross-compile and execute in emulator. + +## Interoperability with C code + +This target supports C code. C code compiled by XL, Open XL and Clang are compatible with Rust. Typical triple of AIX on 64-bit PowerPC of these compilers are also `powerpc64-ibm-aix`. diff --git a/src/doc/rustc/src/platform-support/android.md b/src/doc/rustc/src/platform-support/android.md index 4ef74295e..9ddf00e3a 100644 --- a/src/doc/rustc/src/platform-support/android.md +++ b/src/doc/rustc/src/platform-support/android.md @@ -45,3 +45,19 @@ The riscv64-linux-android target is supported as a Tier 3 target. A list of all supported targets can be found [here](../platform-support.html) + +## Architecture Notes + +### riscv64-linux-android + +Currently the `riscv64-linux-android` target requires the following architecture features/extensions: + +* `a` (atomics) +* `d` (double-precision floating-point) +* `c` (compressed instruction set) +* `f` (single-precision floating-point) +* `m` (multiplication and division) +* `v` (vector) +* `Zba` (address calculation instructions) +* `Zbb` (base instructions) +* `Zbs` (single-bit instructions) diff --git a/src/doc/rustc/src/platform-support/apple-tvos.md b/src/doc/rustc/src/platform-support/apple-tvos.md index d87fd1959..e7ea109df 100644 --- a/src/doc/rustc/src/platform-support/apple-tvos.md +++ b/src/doc/rustc/src/platform-support/apple-tvos.md @@ -52,7 +52,7 @@ The targets can be built by enabling them for a `rustc` build in `config.toml`, ```toml [build] build-stage = 1 -target = ["aarch64-apple-tvos", "x86_64-apple-tvos"] +target = ["aarch64-apple-tvos", "x86_64-apple-tvos", "aarch64-apple-tvos-sim"] ``` It's possible that cargo under `-Zbuild-std` may also be used to target them. @@ -67,6 +67,8 @@ Rust programs can be built for these targets $ rustc --target aarch64-apple-tvos your-code.rs ... $ rustc --target x86_64-apple-tvos your-code.rs +... +$ rustc --target aarch64-apple-tvos-sim your-code.rs ``` ## Testing diff --git a/src/doc/rustc/src/platform-support/armv7-sony-vita-newlibeabihf.md b/src/doc/rustc/src/platform-support/armv7-sony-vita-newlibeabihf.md index 49eed366d..e1473bd96 100644 --- a/src/doc/rustc/src/platform-support/armv7-sony-vita-newlibeabihf.md +++ b/src/doc/rustc/src/platform-support/armv7-sony-vita-newlibeabihf.md @@ -2,15 +2,16 @@ **Tier: 3** -This tier supports the ARM Cortex A9 processor running on a PlayStation Vita console. `armv7-vita-newlibeabihf` aims to have support for `std` crate using `newlib` as a bridge. +This tier supports the ARM Cortex A9 processor running on a PlayStation Vita console. Rust support for this target is not affiliated with Sony, and is not derived from nor used with any official Sony SDK. ## Target maintainers -* [@amg98](https://github.com/amg98) * [@nikarh](https://github.com/nikarh) +* [@pheki](https://github.com/pheki) +* [@ZetaNumbers](https://github.com/ZetaNumbers) ## Requirements @@ -20,18 +21,16 @@ This target is cross-compiled, and requires installing [VITASDK](https://vitasdk `alloc`, and `panic_abort`. `std` is partially supported, but mostly works. Some APIs are unimplemented -and will simply return an error, such as `std::process`. An allocator is provided -by default. +and will simply return an error, such as `std::process`. -In order to support some APIs, binaries must be linked against `libc` written -for the target, using a linker for the target. These are provided by the -VITASDK toolchain. +This target generates binaries in the ELF format with thumb ISA by default. + +Binaries are linked with `arm-vita-eabi-gcc` provided by VITASDK toolchain. -This target generates binaries in the ELF format with thumb ISA. ## Building the target -Rust does not ship pre-compiled artifacts for this target. You can use `build-std` flag to build binaries with `std`: +Rust does not ship pre-compiled artifacts for this target. You can use `build-std` flag to build ELF binaries with `std`: ```sh cargo build -Z build-std=std,panic_abort --target=armv7-sony-vita-newlibeabihf --release @@ -39,113 +38,45 @@ cargo build -Z build-std=std,panic_abort --target=armv7-sony-vita-newlibeabihf - ## Building Rust programs -To test your developed rust programs on PlayStation Vita, first you must correctly package your elf. These steps can be preformed using tools available in VITASDK, and can be automated using a tool like `cargo-make`. +The recommended way to build artifacts that can be installed and run on PlayStation Vita is by using the [cargo-vita](https://github.com/vita-rust/cargo-vita) tool. This tool uses `build-std` and VITASDK toolchain to build artifacts runnable on Vita. + +To install the tool run: + +```sh +cargo install cargo-vita +``` -First, set up environment variables for `VITASDK`, and it's binaries: +[VITASDK](https://vitasdk.org/) toolchain must be installed, and the `VITASDK` environment variable must be set to its location, e.g.: ```sh export VITASDK=/opt/vitasdk -export PATH=$PATH:$VITASDK/bin ``` -Use the example below as a template for your project: +Add the following section to your project's `Cargo.toml`: + ```toml -[env] -TITLE = "Rust Hello World" -TITLEID = "RUST00001" - -# At least a "sce_sys" folder should be place there for app metadata (title, icons, description...) -# You can find sample assets for that on $VITASDK/share/gcc-arm-vita-eabi/samples/hello_world/sce_sys/ -STATIC_DIR = "static" # Folder where static assets should be placed (sce_sys folder is at $STATIC_DIR/sce_sys) -CARGO_TARGET_DIR = { script = ["echo ${CARGO_TARGET_DIR:=target}"] } -CARGO_OUT_DIR = "${CARGO_TARGET_DIR}/${RUST_TARGET}/release" - -[tasks.build] -description = "Build the project using `cargo`." -command = "cargo" -args = ["build", "-Z", "build-std=std,panic_abort", "--target=armv7-sony-vita-newlibeabihf", "--release"] - -[tasks.strip] -description = "Strip the produced ELF executable." -dependencies = ["build"] -command = "arm-vita-eabi-strip" -args = ["-g", '${CARGO_OUT_DIR}/${CARGO_MAKE_CRATE_FS_NAME}.elf'] - -[tasks.velf] -description = "Build an VELF executable from the obtained ELF file." -dependencies = ["strip"] -command = "vita-elf-create" -args = ['${CARGO_OUT_DIR}/${CARGO_MAKE_CRATE_NAME}.elf', '${CARGO_OUT_DIR}/${CARGO_MAKE_CRATE_NAME}.velf'] - -[tasks.eboot-bin] -description = "Build an `eboot.bin` file from the obtained VELF file." -dependencies = ["velf"] -command = "vita-make-fself" -args = ["-s", '${CARGO_OUT_DIR}/${CARGO_MAKE_CRATE_NAME}.velf', '${CARGO_OUT_DIR}/eboot.bin'] - -[tasks.param-sfo] -description = "Build the `param.sfo` manifest using with given TITLE and TITLEID." -command = "vita-mksfoex" -args = ["-s", 'TITLE_ID=${TITLEID}', '${TITLE}', '${CARGO_OUT_DIR}/param.sfo'] - -[tasks.manifest] -description = "List all static resources into a manifest file." -script = [ - 'mkdir -p "${CARGO_OUT_DIR}"', - ''' - if [ -d "${STATIC_DIR}" ]; then - find "${STATIC_DIR}" -type f > "${CARGO_OUT_DIR}/MANIFEST" - else - touch "${CARGO_OUT_DIR}/MANIFEST" - fi - ''' -] - -[tasks.vpk] -description = "Build a VPK distribution of the project executable and resources." -dependencies = ["eboot-bin", "param-sfo", "manifest"] -script_runner = "@rust" -script = [ - ''' - use std::io::BufRead; - use std::fs::File; - - fn main() { - - let crate_name = env!("CARGO_MAKE_CRATE_NAME"); - let static_dir = env!("STATIC_DIR"); - let out_dir = std::path::PathBuf::from(env!("CARGO_OUT_DIR")); - - let mut cmd = ::std::process::Command::new("vita-pack-vpk"); - cmd.arg("-s").arg(out_dir.join("param.sfo")); - cmd.arg("-b").arg(out_dir.join("eboot.bin")); - - // Add files from MANIFEST - if let Ok(file) = File::open(out_dir.join("MANIFEST")) { - let mut reader = ::std::io::BufReader::new(file); - let mut lines = reader.lines(); - while let Some(Ok(line)) = lines.next() { - let p1 = ::std::path::PathBuf::from(line); // path on FS - let p2 = p1.strip_prefix(static_dir).unwrap(); // path in VPK - cmd.arg("--add").arg(format!("{}={}", p1.display(), p2.display())); - } - } - - cmd.arg(out_dir.join(format!("{}.vpk", crate_name))) - .output() - .expect("command failed."); - } - ''' -] +[package.metadata.vita] +# A unique 9 character alphanumeric identifier of the app. +title_id = "RUSTAPP01" +# A title that will be used for the app. Optional, name will be used if not defined +title_name = "My application" ``` -After running the above script, you should be able to get a *.vpk file in the same folder your *.elf executable resides. Now you can pick it and install it on your own PlayStation Vita using, or you can use an [Vita3K](https://vita3k.org/) emulator. +To build a VPK with ELF in the release profile, run: + +```sh +cargo vita build vpk --release +``` + +After building a *.vpk file it can be uploaded to a PlayStation Vita and installed, or used with a [Vita3K](https://vita3k.org/) emulator. ## Testing -Currently there is no support to run the rustc test suite for this target. +The default Rust test runner is supported, and tests can be compiled to an elf and packed to a *.vpk file using `cargo-vita` tool. Filtering tests is not currently supported since passing command-line arguments to the executable is not supported on Vita, so the runner will always execute all tests. + +The Rust test suite for `library/std` is not yet supported. ## Cross-compilation -This target can be cross-compiled from `x86_64` on either Windows, MacOS or Linux systems. Other hosts are not supported for cross-compilation. +This target can be cross-compiled from `x86_64` on Windows, MacOS or Linux systems. Other hosts are not supported for cross-compilation. diff --git a/src/doc/rustc/src/platform-support/csky-unknown-linux-gnuabiv2.md b/src/doc/rustc/src/platform-support/csky-unknown-linux-gnuabiv2.md index e73598be0..a54abcb60 100644 --- a/src/doc/rustc/src/platform-support/csky-unknown-linux-gnuabiv2.md +++ b/src/doc/rustc/src/platform-support/csky-unknown-linux-gnuabiv2.md @@ -4,8 +4,21 @@ This target supports [C-SKY](https://github.com/c-sky) CPUs with `abi` v2 and `glibc`. -https://c-sky.github.io/ -https://gitlab.com/c-sky/ +target | std | host | notes +-------|:---:|:----:|------- +`csky-unknown-linux-gnuabiv2` | ✓ | | C-SKY abiv2 Linux (little endian) +`csky-unknown-linux-gnuabiv2hf` | ✓ | | C-SKY abiv2 Linux, hardfloat (little endian) + +Reference: + +- [CSKY ABI Manual](https://occ-oss-prod.oss-cn-hangzhou.aliyuncs.com/resource//1695027452256/T-HEAD_800_Series_ABI_Standards_Manual.pdf) +- [csky-linux-gnuabiv2-toolchain](https://occ-oss-prod.oss-cn-hangzhou.aliyuncs.com/resource/1356021/1619528643136/csky-linux-gnuabiv2-tools-x86_64-glibc-linux-4.9.56-20210423.tar.gz) +- [csky-linux-gnuabiv2-qemu](https://occ-oss-prod.oss-cn-hangzhou.aliyuncs.com/resource//1689324918932/xuantie-qemu-x86_64-Ubuntu-18.04-20230714-0202.tar.gz) + +other links: + +- https://c-sky.github.io/ +- https://gitlab.com/c-sky/ ## Target maintainers @@ -13,7 +26,6 @@ https://gitlab.com/c-sky/ ## Requirements - ## Building the target ### Get a C toolchain @@ -28,13 +40,17 @@ The target can be built by enabling it for a `rustc` build, by placing the follo ```toml [build] -target = ["x86_64-unknown-linux-gnu", "csky-unknown-linux-gnuabiv2"] +target = ["x86_64-unknown-linux-gnu", "csky-unknown-linux-gnuabiv2", "csky-unknown-linux-gnuabiv2hf"] stage = 2 [target.csky-unknown-linux-gnuabiv2] # ADJUST THIS PATH TO POINT AT YOUR TOOLCHAIN cc = "${TOOLCHAIN_PATH}/bin/csky-linux-gnuabiv2-gcc" +[target.csky-unknown-linux-gnuabiv2hf] +# ADJUST THIS PATH TO POINT AT YOUR TOOLCHAIN +cc = "${TOOLCHAIN_PATH}/bin/csky-linux-gnuabiv2-gcc" + ### Build ```sh diff --git a/src/doc/rustc/src/platform-support/fuchsia.md b/src/doc/rustc/src/platform-support/fuchsia.md index f7cce35b1..34ab3cdaf 100644 --- a/src/doc/rustc/src/platform-support/fuchsia.md +++ b/src/doc/rustc/src/platform-support/fuchsia.md @@ -692,10 +692,12 @@ Fuchsia's test runner interacts with the Fuchsia emulator and is located at test environment with: ```sh -src/ci/docker/scripts/fuchsia-test-runner.py start \ +( \ + src/ci/docker/scripts/fuchsia-test-runner.py start \ --rust-build ${RUST_SRC_PATH}/build \ --sdk ${SDK_PATH} \ --target {x86_64-unknown-fuchsia|aarch64-unknown-fuchsia} \ +) ``` Where `${RUST_SRC_PATH}/build` is the `build-dir` set in `config.toml` and diff --git a/src/doc/rustc/src/platform-support/mips-release-6.md b/src/doc/rustc/src/platform-support/mips-release-6.md index 3f1912fc6..9203a31e9 100644 --- a/src/doc/rustc/src/platform-support/mips-release-6.md +++ b/src/doc/rustc/src/platform-support/mips-release-6.md @@ -67,7 +67,7 @@ The following procedure outlines the build process for the MIPS64 R6 target with ### Prerequisite: Disable debuginfo -A LLVM bug makes rustc crash if debug or debug info generation is enabled. You need to edit `config.toml` to disable this: +An LLVM bug makes rustc crash if debug or debug info generation is enabled. You need to edit `config.toml` to disable this: ```toml [rust] diff --git a/src/doc/rustc/src/platform-support/nto-qnx.md b/src/doc/rustc/src/platform-support/nto-qnx.md index b376c4a84..9f0662783 100644 --- a/src/doc/rustc/src/platform-support/nto-qnx.md +++ b/src/doc/rustc/src/platform-support/nto-qnx.md @@ -98,7 +98,7 @@ Example content: ```toml profile = "compiler" -changelog-seen = 2 +change-id = 115898 ``` 2. Compile the Rust toolchain for an `x86_64-unknown-linux-gnu` host (for both `aarch64` and `x86_64` targets) diff --git a/src/doc/rustc/src/platform-support/openharmony.md b/src/doc/rustc/src/platform-support/openharmony.md index 89539f388..05fd407ed 100644 --- a/src/doc/rustc/src/platform-support/openharmony.md +++ b/src/doc/rustc/src/platform-support/openharmony.md @@ -101,7 +101,7 @@ To build a rust toolchain, create a `config.toml` with the following contents: ```toml profile = "compiler" -changelog-seen = 2 +change-id = 115898 [build] sanitizers = true diff --git a/src/doc/rustc/src/platform-support/unknown-uefi.md b/src/doc/rustc/src/platform-support/unknown-uefi.md index 68cd7fae3..1230ea22b 100644 --- a/src/doc/rustc/src/platform-support/unknown-uefi.md +++ b/src/doc/rustc/src/platform-support/unknown-uefi.md @@ -265,9 +265,14 @@ cargo build --target x86_64-unknown-uefi -Zbuild-std=std,panic_abort #### os_str - While the strings in UEFI should be valid UCS-2, in practice, many implementations just do not care and use UTF-16 strings. - Thus, the current implementation supports full UTF-16 strings. +#### stdio +- Uses `Simple Text Input Protocol` and `Simple Text Output Protocol`. +- Note: UEFI uses CRLF for new line. This means Enter key is registered as CR instead of LF. +#### args +- Uses `EFI_LOADED_IMAGE_PROTOCOL->LoadOptions` ## Example: Hello World With std -The following code features a valid UEFI application, including stdio and `alloc` (`OsString` and `Vec`): +The following code features a valid UEFI application, including `stdio` and `alloc` (`OsString` and `Vec`): This example can be compiled as binary crate via `cargo` using the toolchain compiled from the above source (named custom): @@ -286,6 +291,9 @@ use std::{ }; pub fn main() { + println!("Starting Rust Application..."); + + // Use System Table Directly let st = env::system_table().as_ptr() as *mut efi::SystemTable; let mut s: Vec = OsString::from("Hello World!\n").encode_wide().collect(); s.push(0); diff --git a/src/doc/rustc/src/profile-guided-optimization.md b/src/doc/rustc/src/profile-guided-optimization.md index d9cf7ce30..38b655b75 100644 --- a/src/doc/rustc/src/profile-guided-optimization.md +++ b/src/doc/rustc/src/profile-guided-optimization.md @@ -145,3 +145,26 @@ in Clang's documentation is therefore an interesting read for anyone who wants to use PGO with Rust. [clang-pgo]: https://clang.llvm.org/docs/UsersManual.html#profile-guided-optimization + +## Community Maintained Tools + +As an alternative to directly using the compiler for Profile-Guided Optimization, +you may choose to go with `cargo-pgo`, which has an intuitive command-line API +and saves you the trouble of doing all the manual work. You can read more about +it in their repository accessible from this link: https://github.com/Kobzol/cargo-pgo + +For the sake of completeness, here are the corresponding steps using `cargo-pgo`: + +```bash +# Install if you haven't already +cargo install cargo-pgo + +cargo pgo build +cargo pgo optimize +``` + +These steps will do the following just as before: + +1. Build an instrumented binary from the source code. +2. Run the instrumented binary to gather PGO profiles. +3. Use the gathered PGO profiles from the last step to build an optimized binary. diff --git a/src/doc/rustdoc/src/advanced-features.md b/src/doc/rustdoc/src/advanced-features.md index dbf0baec0..1733c8fc9 100644 --- a/src/doc/rustdoc/src/advanced-features.md +++ b/src/doc/rustdoc/src/advanced-features.md @@ -110,3 +110,23 @@ https://doc.rust-lang.org/stable/std/?search=%s&go_to_first=true This URL adds the `go_to_first=true` query parameter which can be appended to any `rustdoc` search URL to automatically go to the first result. + +## `#[repr(transparent)]`: Documenting the transparent representation + +You can read more about `#[repr(transparent)]` itself in the [Rust Reference][repr-trans-ref] and +in the [Rustonomicon][repr-trans-nomicon]. + +Since this representation is only considered part of the public ABI if the single field with non-trivial +size or alignment is public and if the documentation does not state otherwise, Rustdoc helpfully displays +the attribute if and only if the non-1-ZST field is public or at least one field is public in case all +fields are 1-ZST fields. The term *1-ZST* refers to types that are one-aligned and zero-sized. + +It would seem that one can manually hide the attribute with `#[cfg_attr(not(doc), repr(transparent))]` +if one wishes to declare the representation as private even if the non-1-ZST field is public. +However, due to [current limitations][cross-crate-cfg-doc], this method is not always guaranteed to work. +Therefore, if you would like to do so, you should always write it down in prose independently of whether +you use `cfg_attr` or not. + +[repr-trans-ref]: https://doc.rust-lang.org/reference/type-layout.html#the-transparent-representation +[repr-trans-nomicon]: https://doc.rust-lang.org/nomicon/other-reprs.html#reprtransparent +[cross-crate-cfg-doc]: https://github.com/rust-lang/rust/issues/114952 diff --git a/src/doc/rustdoc/src/unstable-features.md b/src/doc/rustdoc/src/unstable-features.md index 7473b0992..41602dec4 100644 --- a/src/doc/rustdoc/src/unstable-features.md +++ b/src/doc/rustdoc/src/unstable-features.md @@ -207,6 +207,21 @@ To do so, the `#[doc(keyword = "...")]` attribute is used. Example: mod empty_mod {} ``` +### Use the Rust logo as the crate logo + +This is for official Rust project use only. + +Internal Rustdoc pages like settings.html and scrape-examples-help.html show the Rust logo. +This logo is tracked as a static resource. The attribute `#![doc(rust_logo)]` makes this same +built-in resource act as the main logo. + +```rust +#![feature(rustdoc_internals)] +#![allow(internal_features)] +#![doc(rust_logo)] +//! This crate has the Rust(tm) branding on it. +``` + ## Effects of other nightly features These nightly-only features are not primarily related to Rustdoc, @@ -613,10 +628,10 @@ Using this flag looks like this: ```bash $ rustdoc src/lib.rs -Z unstable-options \ - --check-cfg='names()' --check-cfg='values(feature, "foo", "bar")' + --check-cfg='cfg(feature, values("foo", "bar"))' ``` -The example above check every well known names (`target_os`, `doc`, `test`, ... via `names()`) +The example above check every well known names and values (`target_os`, `doc`, `test`, ...) and check the values of `feature`: `foo` and `bar`. ### `--generate-link-to-definition`: Generate links on types in source code diff --git a/src/doc/rustdoc/src/write-documentation/what-to-include.md b/src/doc/rustdoc/src/write-documentation/what-to-include.md index 16457ed0f..75d3b7dae 100644 --- a/src/doc/rustdoc/src/write-documentation/what-to-include.md +++ b/src/doc/rustdoc/src/write-documentation/what-to-include.md @@ -73,7 +73,7 @@ and your test suite, this example needs some additional code: ``````text /// Example /// ```rust -/// # main() -> Result<(), std::num::ParseIntError> { +/// # fn main() -> Result<(), std::num::ParseIntError> { /// let fortytwo = "42".parse::()?; /// println!("{} + 10 = {}", fortytwo, fortytwo+10); /// # Ok(()) @@ -117,7 +117,7 @@ rustdoc --theme awesome.css src/lib.rs Here is an example of a new theme, [Ayu]. -[Ayu]: https://github.com/rust-lang/rust/blob/master/src/librustdoc/html/static/css/themes/ayu.css +[Ayu]: https://github.com/rust-lang/rust/blob/master/src/librustdoc/html/static/css/rustdoc.css#L2384-L2574 [API Guidelines]: https://rust-lang.github.io/api-guidelines/documentation.html#rustdoc-does-not-show-unhelpful-implementation-details-c-hidden [Documentation tests]: documentation-tests.md [on this blog]: https://blog.guillaume-gomez.fr/articles/2016-09-16+Generating+doc+with+rustdoc+and+a+custom+theme diff --git a/src/doc/unstable-book/src/compiler-flags/check-cfg.md b/src/doc/unstable-book/src/compiler-flags/check-cfg.md index 10f0fbc50..0e15c7907 100644 --- a/src/doc/unstable-book/src/compiler-flags/check-cfg.md +++ b/src/doc/unstable-book/src/compiler-flags/check-cfg.md @@ -10,97 +10,80 @@ This feature allows you to enable complete or partial checking of configuration. check them. The `--check-cfg` option takes a value, called the _check cfg specification_. The check cfg specification is parsed using the Rust metadata syntax, just as the `--cfg` option is. -`--check-cfg` option can take one of two forms: +`--check-cfg` option take one form: -1. `--check-cfg names(...)` enables checking condition names. -2. `--check-cfg values(...)` enables checking the values within list-valued conditions. - -These two options are independent. `names` checks only the namespace of condition names -while `values` checks only the namespace of the values of list-valued conditions. +1. `--check-cfg cfg(...)` enables checking the values within list-valued conditions. NOTE: No implicit expectation is added when using `--cfg` for both forms. Users are expected to -pass all expected names and values using `names(...)` and `values(...)`. +pass all expected names and values using `cfg(...)`. -## The `names(...)` form +## The `cfg(...)` form -The `names(...)` form enables checking the names. This form uses a named list: +The `cfg(...)` form enables checking the values within list-valued conditions. It has this +basic form: ```bash -rustc --check-cfg 'names(name1, name2, ... nameN)' +rustc --check-cfg 'cfg(name1, ..., nameN, values("value1", "value2", ... "valueN"))' ``` -where each `name` is a bare identifier (has no quotes). The order of the names is not significant. +where `name` is a bare identifier (has no quotes) and each `"value"` term is a quoted literal +string. `name` specifies the name of the condition, such as `feature` or `my_cfg`. -If `--check-cfg names(...)` is specified at least once, then `rustc` will check all references to -condition names. `rustc` will check every `#[cfg]` attribute, `#[cfg_attr]` attribute, `cfg` clause -inside `#[link]` attribute and `cfg!(...)` call against the provided list of expected condition -names. If a name is not present in this list, then `rustc` will report an `unexpected_cfgs` lint -diagnostic. The default diagnostic level for this lint is `Warn`. +When the `cfg(...)` option is specified, `rustc` will check every `#[cfg(name = "value")]` +attribute, `#[cfg_attr(name = "value")]` attribute, `#[link(name = "a", cfg(name = "value"))]` +and `cfg!(name = "value")` call. It will check that the `"value"` specified is present in the +list of expected values. If `"value"` is not in it, then `rustc` will report an `unexpected_cfgs` +lint diagnostic. The default diagnostic level for this lint is `Warn`. -If `--check-cfg names(...)` is not specified, then `rustc` will not check references to condition -names. +To enable checking of values, but to provide an empty set of expected values, use these forms: -`--check-cfg names(...)` may be specified more than once. The result is that the list of valid -condition names is merged across all options. It is legal for a condition name to be specified -more than once; redundantly specifying a condition name has no effect. +```bash +rustc --check-cfg 'cfg(name1, ..., nameN)' +rustc --check-cfg 'cfg(name1, ..., nameN, values())' +``` -To enable checking condition names with an empty set of valid condition names, use the following -form. The parentheses are required. +To enable checking of name but not values (i.e. unknown expected values), use this form: ```bash -rustc --check-cfg 'names()' +rustc --check-cfg 'cfg(name1, ..., nameN, values(any()))' ``` -Note that `--check-cfg 'names()'` is _not_ equivalent to omitting the option entirely. -The first form enables checking condition names, while specifying that there are no valid -condition names (outside of the set of well-known names defined by `rustc`). Omitting the -`--check-cfg 'names(...)'` option does not enable checking condition names. - -## The `values(...)` form +The `--check-cfg cfg(...)` option can be repeated, both for the same condition name and for +different names. If it is repeated for the same condition name, then the sets of values for that +condition are merged together (presedence is given to `any()`). -The `values(...)` form enables checking the values within list-valued conditions. It has this -form: +## Well known names and values -```bash -rustc --check-cfg `values(name, "value1", "value2", ... "valueN")' -``` +`rustc` has a internal list of well known names and their corresponding values. +Those well known names and values follows the same stability as what they refer to. -where `name` is a bare identifier (has no quotes) and each `"value"` term is a quoted literal -string. `name` specifies the name of the condition, such as `feature` or `target_os`. +Well known values checking is always enabled as long as a `--check-cfg` argument is present. -When the `values(...)` option is specified, `rustc` will check every `#[cfg(name = "value")]` -attribute, `#[cfg_attr(name = "value")]` attribute, `#[link(name = "a", cfg(name = "value"))]` -and `cfg!(name = "value")` call. It will check that the `"value"` specified is present in the -list of expected values. If `"value"` is not in it, then `rustc` will report an `unexpected_cfgs` -lint diagnostic. The default diagnostic level for this lint is `Warn`. +Well known names checking is always enable as long as a `--check-cfg` argument is present +**unless** any `cfg(any())` argument is passed. -To enable checking of values, but to provide an empty set of valid values, use this form: +To disable checking of well known names, use this form: ```bash -rustc --check-cfg `values(name)` +rustc --check-cfg 'cfg(any())' ``` -The `--check-cfg values(...)` option can be repeated, both for the same condition name and for -different names. If it is repeated for the same condition name, then the sets of values for that -condition are merged together. - -If `values()` is specified, then `rustc` will enable the checking of well-known values defined -by itself. Note that it's necessary to specify the `values()` form to enable the checking of -well known values, specifying the other forms doesn't implicitly enable it. +NOTE: If one want to enable values and names checking without having any cfg to declare, one +can use an empty `cfg()` argument. ## Examples Consider this command line: ```bash -rustc --check-cfg 'names(feature)' \ - --check-cfg 'values(feature, "lion", "zebra")' \ +rustc --check-cfg 'cfg(feature, values("lion", "zebra"))' \ --cfg 'feature="lion"' -Z unstable-options \ example.rs ``` This command line indicates that this crate has two features: `lion` and `zebra`. The `lion` -feature is enabled, while the `zebra` feature is disabled. Consider compiling this code: +feature is enabled, while the `zebra` feature is disabled. Exhaustive checking of names and +values are enabled by default. Consider compiling this code: ```rust // This is expected, and tame_lion() will be compiled @@ -119,35 +102,36 @@ fn poke_platypus() {} // and will cause a compiler warning (by default). #[cfg(feechure = "lion")] fn tame_lion() {} -``` -> Note: The `--check-cfg names(feature)` option is necessary only to enable checking the condition -> name, as in the last example. `feature` is a well-known (always-expected) condition name, and so -> it is not necessary to specify it in a `--check-cfg 'names(...)'` option. That option can be -> shortened to > `--check-cfg names()` in order to enable checking well-known condition names. +// This is UNEXPECTED, because 'windows' is a well known condition name, +// and because 'windows' doens't take any values, +// and will cause a compiler warning (by default). +#[cfg(windows = "unix")] +fn tame_windows() {} +``` ### Example: Checking condition names, but not values ```bash # This turns on checking for condition names, but not values, such as 'feature' values. -rustc --check-cfg 'names(is_embedded, has_feathers)' \ +rustc --check-cfg 'cfg(is_embedded, has_feathers, values(any()))' \ --cfg has_feathers -Z unstable-options ``` ```rust -#[cfg(is_embedded)] // This is expected as "is_embedded" was provided in names() -fn do_embedded() {} +#[cfg(is_embedded)] // This is expected as "is_embedded" was provided in cfg() +fn do_embedded() {} // and because names exhaustiveness was not disabled -#[cfg(has_feathers)] // This is expected as "has_feathers" was provided in names() -fn do_features() {} +#[cfg(has_feathers)] // This is expected as "has_feathers" was provided in cfg() +fn do_features() {} // and because names exhaustiveness was not disabled -#[cfg(has_feathers = "zapping")] // This is expected as "has_feathers" was provided in names() +#[cfg(has_feathers = "zapping")] // This is expected as "has_feathers" was provided in cfg() // and because no value checking was enable for "has_feathers" // no warning is emitted for the value "zapping" fn do_zapping() {} #[cfg(has_mumble_frotz)] // This is UNEXPECTED because names checking is enable and - // "has_mumble_frotz" was not provided in names() + // "has_mumble_frotz" was not provided in cfg() fn do_mumble_frotz() {} ``` @@ -155,25 +139,25 @@ fn do_mumble_frotz() {} ```bash # This turns on checking for feature values, but not for condition names. -rustc --check-cfg 'values(feature, "zapping", "lasers")' \ +rustc --check-cfg 'cfg(feature, values("zapping", "lasers"))' \ + --check-cfg 'cfg(any())' \ --cfg 'feature="zapping"' -Z unstable-options ``` ```rust -#[cfg(is_embedded)] // This is doesn't raise a warning, because names checking was not - // enable (ie not names()) +#[cfg(is_embedded)] // This is doesn't raise a warning, because names checking was + // disabled by 'cfg(any())' fn do_embedded() {} -#[cfg(has_feathers)] // Same as above, --check-cfg names(...) was never used so no name +#[cfg(has_feathers)] // Same as above, 'cfg(any())' was provided so no name // checking is performed fn do_features() {} - -#[cfg(feature = "lasers")] // This is expected, "lasers" is in the values(feature) list +#[cfg(feature = "lasers")] // This is expected, "lasers" is in the cfg(feature) list fn shoot_lasers() {} #[cfg(feature = "monkeys")] // This is UNEXPECTED, because "monkeys" is not in the - // --check-cfg values(feature) list + // cfg(feature) list fn write_shakespeare() {} ``` @@ -181,26 +165,92 @@ fn write_shakespeare() {} ```bash # This turns on checking for feature values and for condition names. -rustc --check-cfg 'names(is_embedded, has_feathers)' \ - --check-cfg 'values(feature, "zapping", "lasers")' \ +rustc --check-cfg 'cfg(is_embedded, has_feathers)' \ + --check-cfg 'cfg(feature, values("zapping", "lasers"))' \ --cfg has_feathers --cfg 'feature="zapping"' -Z unstable-options ``` ```rust -#[cfg(is_embedded)] // This is expected because "is_embedded" was provided in names() -fn do_embedded() {} +#[cfg(is_embedded)] // This is expected because "is_embedded" was provided in cfg() +fn do_embedded() {} // and doesn't take any value -#[cfg(has_feathers)] // This is expected because "has_feathers" was provided in names() -fn do_features() {} +#[cfg(has_feathers)] // This is expected because "has_feathers" was provided in cfg() +fn do_features() {} // and deosn't take any value -#[cfg(has_mumble_frotz)] // This is UNEXPECTED, because has_mumble_frotz is not in the - // --check-cfg names(...) list +#[cfg(has_mumble_frotz)] // This is UNEXPECTED, because "has_mumble_frotz" was never provided fn do_mumble_frotz() {} -#[cfg(feature = "lasers")] // This is expected, "lasers" is in the values(feature) list +#[cfg(feature = "lasers")] // This is expected, "lasers" is in the cfg(feature) list fn shoot_lasers() {} #[cfg(feature = "monkeys")] // This is UNEXPECTED, because "monkeys" is not in - // the values(feature) list + // the cfg(feature) list fn write_shakespeare() {} ``` + +## The deprecated `names(...)` form + +The `names(...)` form enables checking the names. This form uses a named list: + +```bash +rustc --check-cfg 'names(name1, name2, ... nameN)' +``` + +where each `name` is a bare identifier (has no quotes). The order of the names is not significant. + +If `--check-cfg names(...)` is specified at least once, then `rustc` will check all references to +condition names. `rustc` will check every `#[cfg]` attribute, `#[cfg_attr]` attribute, `cfg` clause +inside `#[link]` attribute and `cfg!(...)` call against the provided list of expected condition +names. If a name is not present in this list, then `rustc` will report an `unexpected_cfgs` lint +diagnostic. The default diagnostic level for this lint is `Warn`. + +If `--check-cfg names(...)` is not specified, then `rustc` will not check references to condition +names. + +`--check-cfg names(...)` may be specified more than once. The result is that the list of valid +condition names is merged across all options. It is legal for a condition name to be specified +more than once; redundantly specifying a condition name has no effect. + +To enable checking condition names with an empty set of valid condition names, use the following +form. The parentheses are required. + +```bash +rustc --check-cfg 'names()' +``` + +Note that `--check-cfg 'names()'` is _not_ equivalent to omitting the option entirely. +The first form enables checking condition names, while specifying that there are no valid +condition names (outside of the set of well-known names defined by `rustc`). Omitting the +`--check-cfg 'names(...)'` option does not enable checking condition names. + +## The deprecated `values(...)` form + +The `values(...)` form enables checking the values within list-valued conditions. It has this +form: + +```bash +rustc --check-cfg `values(name, "value1", "value2", ... "valueN")' +``` + +where `name` is a bare identifier (has no quotes) and each `"value"` term is a quoted literal +string. `name` specifies the name of the condition, such as `feature` or `target_os`. + +When the `values(...)` option is specified, `rustc` will check every `#[cfg(name = "value")]` +attribute, `#[cfg_attr(name = "value")]` attribute, `#[link(name = "a", cfg(name = "value"))]` +and `cfg!(name = "value")` call. It will check that the `"value"` specified is present in the +list of expected values. If `"value"` is not in it, then `rustc` will report an `unexpected_cfgs` +lint diagnostic. The default diagnostic level for this lint is `Warn`. + +To enable checking of values, but to provide an empty set of valid values, use this form: + +```bash +rustc --check-cfg `values(name)` +``` + +The `--check-cfg values(...)` option can be repeated, both for the same condition name and for +different names. If it is repeated for the same condition name, then the sets of values for that +condition are merged together. + +If `values()` is specified, then `rustc` will enable the checking of well-known values defined +by itself. Note that it's necessary to specify the `values()` form to enable the checking of +well known values, specifying the other forms doesn't implicitly enable it. diff --git a/src/doc/unstable-book/src/compiler-flags/no-jump-tables.md b/src/doc/unstable-book/src/compiler-flags/no-jump-tables.md new file mode 100644 index 000000000..f096c20f4 --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/no-jump-tables.md @@ -0,0 +1,19 @@ +# `no-jump-tables` + +The tracking issue for this feature is [#116592](https://github.com/rust-lang/rust/issues/116592) + +--- + +This option enables the `-fno-jump-tables` flag for LLVM, which makes the +codegen backend avoid generating jump tables when lowering switches. + +This option adds the LLVM `no-jump-tables=true` attribute to every function. + +The option can be used to help provide protection against +jump-oriented-programming (JOP) attacks, such as with the linux kernel's [IBT]. + +```sh +RUSTFLAGS="-Zno-jump-tables" cargo +nightly build -Z build-std +``` + +[IBT]: https://www.phoronix.com/news/Linux-IBT-By-Default-Tip diff --git a/src/doc/unstable-book/src/compiler-flags/remap-path-scope.md b/src/doc/unstable-book/src/compiler-flags/remap-path-scope.md new file mode 100644 index 000000000..13349ff6b --- /dev/null +++ b/src/doc/unstable-book/src/compiler-flags/remap-path-scope.md @@ -0,0 +1,24 @@ +# `remap-path-scope` + +The tracking issue for this feature is: [#111540](https://github.com/rust-lang/rust/issues/111540). + +------------------------ + +When the `--remap-path-prefix` option is passed to rustc, source path prefixes in all output will be affected by default. +The `--remap-path-scope` argument can be used in conjunction with `--remap-path-prefix` to determine paths in which output context should be affected. +This flag accepts a comma-separated list of values and may be specified multiple times, in which case the scopes are aggregated together. The valid scopes are: + +- `macro` - apply remappings to the expansion of `std::file!()` macro. This is where paths in embedded panic messages come from +- `diagnostics` - apply remappings to printed compiler diagnostics +- `unsplit-debuginfo` - apply remappings to debug information only when they are written to compiled executables or libraries, but not when they are in split debuginfo files +- `split-debuginfo` - apply remappings to debug information only when they are written to split debug information files, but not in compiled executables or libraries +- `split-debuginfo-path` - apply remappings to the paths pointing to split debug information files. Does nothing when these files are not generated. +- `object` - an alias for `macro,unsplit-debuginfo,split-debuginfo-path`. This ensures all paths in compiled executables or libraries are remapped, but not elsewhere. +- `all` and `true` - an alias for all of the above, also equivalent to supplying only `--remap-path-prefix` without `--remap-path-scope`. + +## Example +```sh +# This would produce an absolute path to main.rs in build outputs of +# "./main.rs". +rustc --remap-path-prefix=$(PWD)=/remapped -Zremap-path-prefix=object main.rs +``` diff --git a/src/doc/unstable-book/src/compiler-flags/sanitizer.md b/src/doc/unstable-book/src/compiler-flags/sanitizer.md index 49389b28c..502853f39 100644 --- a/src/doc/unstable-book/src/compiler-flags/sanitizer.md +++ b/src/doc/unstable-book/src/compiler-flags/sanitizer.md @@ -197,22 +197,26 @@ Shadow byte legend (one shadow byte represents 8 application bytes): # ControlFlowIntegrity -The LLVM Control Flow Integrity (CFI) support in the Rust compiler provides -forward-edge control flow protection for both Rust-compiled code only and for C -or C++ and Rust -compiled code mixed-language binaries, also known as “mixed -binaries” (i.e., for when C or C++ and Rust -compiled code share the same -virtual address space), by aggregating function pointers in groups identified by -their return and parameter types. - -LLVM CFI can be enabled with `-Zsanitizer=cfi` and requires LTO (i.e., `-Clto`). -Cross-language LLVM CFI can be enabled with `-Zsanitizer=cfi`, and requires the -`-Zsanitizer-cfi-normalize-integers` option to be used with Clang -`-fsanitize-cfi-icall-normalize-integers` for normalizing integer types, and -proper (i.e., non-rustc) LTO (i.e., `-Clinker-plugin-lto`). +The LLVM CFI support in the Rust compiler provides forward-edge control flow +protection for both Rust-compiled code only and for C or C++ and Rust -compiled +code mixed-language binaries, also known as “mixed binaries” (i.e., for when C +or C++ and Rust -compiled code share the same virtual address space), by +aggregating function pointers in groups identified by their return and parameter +types. + +LLVM CFI can be enabled with `-Zsanitizer=cfi` and requires LTO (i.e., +`-Clinker-plugin-lto` or `-Clto`). Cross-language LLVM CFI can be enabled with +`-Zsanitizer=cfi`, and requires the `-Zsanitizer-cfi-normalize-integers` option +to be used with Clang `-fsanitize-cfi-icall-experimental-normalize-integers` +option for cross-language LLVM CFI support, and proper (i.e., non-rustc) LTO +(i.e., `-Clinker-plugin-lto`). + +It is recommended to rebuild the standard library with CFI enabled by using the +Cargo build-std feature (i.e., `-Zbuild-std`) when enabling CFI. See the [Clang ControlFlowIntegrity documentation][clang-cfi] for more details. -## Example +## Example 1: Redirecting control flow using an indirect branch/call to an invalid destination ```rust,ignore (making doc tests pass cross-platform is hard) #![feature(naked_functions)] @@ -239,7 +243,7 @@ pub extern "C" fn add_two(x: i32) { nop nop nop - lea eax, [edi+2] + lea eax, [rdi+2] ret ", options(noreturn) @@ -258,8 +262,9 @@ fn main() { println!("With CFI enabled, you should not see the next answer"); let f: fn(i32) -> i32 = unsafe { - // Offsets 0-8 make it land in the landing pad/nop block, and offsets 1-8 are - // invalid branch/call destinations (i.e., within the body of the function). + // Offset 0 is a valid branch/call destination (i.e., the function entry + // point), but offsets 1-8 within the landing pad/nop block are invalid + // branch/call destinations (i.e., within the body of the function). mem::transmute::<*const u8, fn(i32) -> i32>((add_two as *const u8).offset(5)) }; let next_answer = do_twice(f, 5); @@ -267,38 +272,40 @@ fn main() { println!("The next answer is: {}", next_answer); } ``` -Fig. 1. Modified example from the [Advanced Functions and -Closures][rust-book-ch19-05] chapter of the [The Rust Programming -Language][rust-book] book. +Fig. 1. Redirecting control flow using an indirect branch/call to an invalid +destination (i.e., within the body of the function). ```shell $ cargo run --release Compiling rust-cfi-1 v0.1.0 (/home/rcvalle/rust-cfi-1) - Finished release [optimized] target(s) in 0.76s + Finished release [optimized] target(s) in 0.42s Running `target/release/rust-cfi-1` The answer is: 12 With CFI enabled, you should not see the next answer The next answer is: 14 $ ``` -Fig. 2. Build and execution of the modified example with LLVM CFI disabled. +Fig. 2. Build and execution of Fig. 1 with LLVM CFI disabled. ```shell -$ RUSTFLAGS="-Zsanitizer=cfi -Cembed-bitcode=yes -Clto" cargo run --release +$ RUSTFLAGS="-Clinker-plugin-lto -Clinker=clang -Clink-arg=-fuse-ld=lld -Zsanitizer=cfi" cargo run -Zbuild-std -Zbuild-std-features --release --target x86_64-unknown-linux-gnu + ... Compiling rust-cfi-1 v0.1.0 (/home/rcvalle/rust-cfi-1) - Finished release [optimized] target(s) in 3.39s - Running `target/release/rust-cfi-1` + Finished release [optimized] target(s) in 1m 08s + Running `target/x86_64-unknown-linux-gnu/release/rust-cfi-1` The answer is: 12 With CFI enabled, you should not see the next answer Illegal instruction $ ``` -Fig. 3. Build and execution of the modified example with LLVM CFI enabled. +Fig. 3. Build and execution of Fig. 1 with LLVM CFI enabled. When LLVM CFI is enabled, if there are any attempts to change/hijack control flow using an indirect branch/call to an invalid destination, the execution is terminated (see Fig. 3). +## Example 2: Redirecting control flow using an indirect branch/call to a function with a different number of parameters + ```rust use std::mem; @@ -327,39 +334,42 @@ fn main() { println!("The next answer is: {}", next_answer); } ``` -Fig. 4. Another modified example from the [Advanced Functions and -Closures][rust-book-ch19-05] chapter of the [The Rust Programming -Language][rust-book] book. +Fig. 4. Redirecting control flow using an indirect branch/call to a function +with a different number of parameters than arguments intended/passed in the +call/branch site. ```shell $ cargo run --release Compiling rust-cfi-2 v0.1.0 (/home/rcvalle/rust-cfi-2) - Finished release [optimized] target(s) in 0.76s + Finished release [optimized] target(s) in 0.43s Running `target/release/rust-cfi-2` The answer is: 12 With CFI enabled, you should not see the next answer The next answer is: 14 $ ``` -Fig. 5. Build and execution of the modified example with LLVM CFI disabled. +Fig. 5. Build and execution of Fig. 4 with LLVM CFI disabled. ```shell -$ RUSTFLAGS="-Cembed-bitcode=yes -Clto -Zsanitizer=cfi" cargo run --release +$ RUSTFLAGS="-Clinker-plugin-lto -Clinker=clang -Clink-arg=-fuse-ld=lld -Zsanitizer=cfi" cargo run -Zbuild-std -Zbuild-std-features --release --target x86_64-unknown-linux-gnu + ... Compiling rust-cfi-2 v0.1.0 (/home/rcvalle/rust-cfi-2) - Finished release [optimized] target(s) in 3.38s - Running `target/release/rust-cfi-2` + Finished release [optimized] target(s) in 1m 08s + Running `target/x86_64-unknown-linux-gnu/release/rust-cfi-2` The answer is: 12 With CFI enabled, you should not see the next answer Illegal instruction $ ``` -Fig. 6. Build and execution of the modified example with LLVM CFI enabled. +Fig. 6. Build and execution of Fig. 4 with LLVM CFI enabled. When LLVM CFI is enabled, if there are any attempts to change/hijack control flow using an indirect branch/call to a function with different number of parameters than arguments intended/passed in the call/branch site, the execution is also terminated (see Fig. 6). +## Example 3: Redirecting control flow using an indirect branch/call to a function with different return and parameter types + ```rust use std::mem; @@ -388,42 +398,46 @@ fn main() { println!("The next answer is: {}", next_answer); } ``` -Fig. 7. Another modified example from the [Advanced Functions and -Closures][rust-book-ch19-05] chapter of the [The Rust Programming -Language][rust-book] book. +Fig. 7. Redirecting control flow using an indirect branch/call to a function +with different return and parameter types than the return type expected and +arguments intended/passed at the call/branch site. ```shell $ cargo run --release Compiling rust-cfi-3 v0.1.0 (/home/rcvalle/rust-cfi-3) - Finished release [optimized] target(s) in 0.74s + Finished release [optimized] target(s) in 0.44s Running `target/release/rust-cfi-3` The answer is: 12 With CFI enabled, you should not see the next answer The next answer is: 14 $ ``` -Fig. 8. Build and execution of the modified example with LLVM CFI disabled. +Fig. 8. Build and execution of Fig. 7 with LLVM CFI disabled. ```shell -$ RUSTFLAGS="-Cembed-bitcode=yes -Clto -Zsanitizer=cfi" cargo run --release +$ RUSTFLAGS="-Clinker-plugin-lto -Clinker=clang -Clink-arg=-fuse-ld=lld -Zsanitizer=cfi" cargo run -Zbuild-std -Zbuild-std-features --release --target x86_64-unknown-linux-gnu + ... Compiling rust-cfi-3 v0.1.0 (/home/rcvalle/rust-cfi-3) - Finished release [optimized] target(s) in 3.40s - Running `target/release/rust-cfi-3` + Finished release [optimized] target(s) in 1m 07s + Running `target/x86_64-unknown-linux-gnu/release/rust-cfi-3` The answer is: 12 With CFI enabled, you should not see the next answer Illegal instruction $ ``` -Fig. 9. Build and execution of the modified example with LLVM CFI enabled. +Fig. 9. Build and execution of Fig. 7 with LLVM CFI enabled. When LLVM CFI is enabled, if there are any attempts to change/hijack control flow using an indirect branch/call to a function with different return and parameter types than the return type expected and arguments intended/passed in the call/branch site, the execution is also terminated (see Fig. 9). +## Example 4: Redirecting control flow using an indirect branch/call to a function with different return and parameter types across the FFI boundary + ```ignore (cannot-test-this-because-uses-custom-build) int -do_twice(int (*fn)(int), int arg) { +do_twice(int (*fn)(int), int arg) +{ return fn(arg) + fn(arg); } ``` @@ -459,54 +473,49 @@ fn main() { println!("The next answer is: {}", next_answer); } ``` -Fig. 11. Another modified example from the [Advanced Functions and -Closures][rust-book-ch19-05] chapter of the [The Rust Programming -Language][rust-book] book. +Fig. 11. Redirecting control flow using an indirect branch/call to a function +with different return and parameter types than the return type expected and +arguments intended/passed in the call/branch site, across the FFI boundary. ```shell $ make -mkdir -p target/debug -clang -I. -Isrc -Wall -flto -fvisibility=hidden -c -emit-llvm src/foo.c -o target/debug/libfoo.bc -llvm-ar rcs target/debug/libfoo.a target/debug/libfoo.bc -RUSTFLAGS="-L./target/debug -Clinker-plugin-lto -Clinker=clang -Clink-arg=-fuse-ld=lld" cargo build - Compiling main v0.1.0 (/home/rcvalle/rust-cross-cfi-1) - Finished dev [unoptimized + debuginfo] target(s) in 0.45s -$ ./target/debug/main +mkdir -p target/release +clang -I. -Isrc -Wall -c src/foo.c -o target/release/libfoo.o +llvm-ar rcs target/release/libfoo.a target/release/libfoo.o +RUSTFLAGS="-L./target/release -Clinker=clang -Clink-arg=-fuse-ld=lld" cargo build --release + Compiling rust-cfi-4 v0.1.0 (/home/rcvalle/rust-cfi-4) + Finished release [optimized] target(s) in 0.49s +$ ./target/release/rust-cfi-4 The answer is: 12 With CFI enabled, you should not see the next answer The next answer is: 14 $ ``` -Fig. 12. Build and execution of the modified example with LLVM CFI disabled. +Fig. 12. Build and execution of Figs. 10–11 with LLVM CFI disabled. ```shell $ make -mkdir -p target/debug -clang -I. -Isrc -Wall -flto -fvisibility=hidden -fsanitize=cfi -fsanitize-cfi-icall-normalize-integers -c -emit-llvm src/foo.c -o target/debug/libfoo.bc -llvm-ar rcs target/debug/libfoo.a target/debug/libfoo.bc -RUSTFLAGS="-L./target/debug -Clinker-plugin-lto -Clinker=clang -Clink-arg=-fuse-ld=lld -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers" cargo build - Compiling main v0.1.0 (/home/rcvalle/rust-cross-cfi-1) - Finished dev [unoptimized + debuginfo] target(s) in 0.45s -$ ./target/debug/main +mkdir -p target/release +clang -I. -Isrc -Wall -flto -fsanitize=cfi -fsanitize-cfi-icall-experimental-normalize-integers -fvisibility=hidden -c -emit-llvm src/foo.c -o target/release/libfoo.bc +llvm-ar rcs target/release/libfoo.a target/release/libfoo.bc +RUSTFLAGS="-L./target/release -Clinker-plugin-lto -Clinker=clang -Clink-arg=-fuse-ld=lld -Zsanitizer=cfi -Zsanitizer-cfi-normalize-integers" cargo build -Zbuild-std -Zbuild-std-features --release --target x86_64-unknown-linux-gnu + ... + Compiling rust-cfi-4 v0.1.0 (/home/rcvalle/rust-cfi-4) + Finished release [optimized] target(s) in 1m 06s +$ ./target/x86_64-unknown-linux-gnu/release/rust-cfi-4 The answer is: 12 With CFI enabled, you should not see the next answer Illegal instruction $ ``` -Fig. 13. Build and execution of the modified example with LLVM CFI enabled. - -When LLVM CFI is enabled, if there are any attempts to change/hijack control -flow using an indirect branch/call to a function with different return and -parameter types than the return type expected and arguments intended/passed in -the call/branch site, even across the FFI boundary and for extern "C" function -types indirectly called (i.e., callbacks/function pointers) across the FFI -boundary, in C or C++ and Rust -compiled code mixed-language binaries, also -known as “mixed binaries” (i.e., for when C or C++ and Rust -compiled code share -the same virtual address space), the execution is also terminated (see Fig. 13). - - -[rust-book-ch19-05]: https://doc.rust-lang.org/book/ch19-05-advanced-functions-and-closures.html -[rust-book]: https://doc.rust-lang.org/book/title-page.html +Fig. 13. Build and execution of FIgs. 10–11 with LLVM CFI enabled. + +When LLVM CFI is enabled, if there are any attempts to redirect control flow +using an indirect branch/call to a function with different return and parameter +types than the return type expected and arguments intended/passed in the +call/branch site, even across the FFI boundary and for extern "C" function types +indirectly called (i.e., callbacks/function pointers) across the FFI boundary, +the execution is also terminated (see Fig. 13). # HWAddressSanitizer diff --git a/src/doc/unstable-book/src/language-features/closure-track-caller.md b/src/doc/unstable-book/src/language-features/closure-track-caller.md index c948810d3..997452231 100644 --- a/src/doc/unstable-book/src/language-features/closure-track-caller.md +++ b/src/doc/unstable-book/src/language-features/closure-track-caller.md @@ -6,7 +6,7 @@ The tracking issue for this feature is: [#87417] ------------------------ -Allows using the `#[track_caller]` attribute on closures and generators. -Calls made to the closure or generator will have caller information +Allows using the `#[track_caller]` attribute on closures and coroutines. +Calls made to the closure or coroutine will have caller information available through `std::panic::Location::caller()`, just like using `#[track_caller]` on a function. diff --git a/src/doc/unstable-book/src/language-features/coroutines.md b/src/doc/unstable-book/src/language-features/coroutines.md new file mode 100644 index 000000000..f8e5a22fb --- /dev/null +++ b/src/doc/unstable-book/src/language-features/coroutines.md @@ -0,0 +1,246 @@ +# `coroutines` + +The tracking issue for this feature is: [#43122] + +[#43122]: https://github.com/rust-lang/rust/issues/43122 + +------------------------ + +The `coroutines` feature gate in Rust allows you to define coroutine or +coroutine literals. A coroutine is a "resumable function" that syntactically +resembles a closure but compiles to much different semantics in the compiler +itself. The primary feature of a coroutine is that it can be suspended during +execution to be resumed at a later date. Coroutines use the `yield` keyword to +"return", and then the caller can `resume` a coroutine to resume execution just +after the `yield` keyword. + +Coroutines are an extra-unstable feature in the compiler right now. Added in +[RFC 2033] they're mostly intended right now as a information/constraint +gathering phase. The intent is that experimentation can happen on the nightly +compiler before actual stabilization. A further RFC will be required to +stabilize coroutines and will likely contain at least a few small +tweaks to the overall design. + +[RFC 2033]: https://github.com/rust-lang/rfcs/pull/2033 + +A syntactical example of a coroutine is: + +```rust +#![feature(coroutines, coroutine_trait)] + +use std::ops::{Coroutine, CoroutineState}; +use std::pin::Pin; + +fn main() { + let mut coroutine = || { + yield 1; + return "foo" + }; + + match Pin::new(&mut coroutine).resume(()) { + CoroutineState::Yielded(1) => {} + _ => panic!("unexpected value from resume"), + } + match Pin::new(&mut coroutine).resume(()) { + CoroutineState::Complete("foo") => {} + _ => panic!("unexpected value from resume"), + } +} +``` + +Coroutines are closure-like literals which can contain a `yield` statement. The +`yield` statement takes an optional expression of a value to yield out of the +coroutine. All coroutine literals implement the `Coroutine` trait in the +`std::ops` module. The `Coroutine` trait has one main method, `resume`, which +resumes execution of the coroutine at the previous suspension point. + +An example of the control flow of coroutines is that the following example +prints all numbers in order: + +```rust +#![feature(coroutines, coroutine_trait)] + +use std::ops::Coroutine; +use std::pin::Pin; + +fn main() { + let mut coroutine = || { + println!("2"); + yield; + println!("4"); + }; + + println!("1"); + Pin::new(&mut coroutine).resume(()); + println!("3"); + Pin::new(&mut coroutine).resume(()); + println!("5"); +} +``` + +At this time the main intended use case of coroutines is an implementation +primitive for async/await syntax, but coroutines will likely be extended to +ergonomic implementations of iterators and other primitives in the future. +Feedback on the design and usage is always appreciated! + +### The `Coroutine` trait + +The `Coroutine` trait in `std::ops` currently looks like: + +```rust +# #![feature(arbitrary_self_types, coroutine_trait)] +# use std::ops::CoroutineState; +# use std::pin::Pin; + +pub trait Coroutine { + type Yield; + type Return; + fn resume(self: Pin<&mut Self>, resume: R) -> CoroutineState; +} +``` + +The `Coroutine::Yield` type is the type of values that can be yielded with the +`yield` statement. The `Coroutine::Return` type is the returned type of the +coroutine. This is typically the last expression in a coroutine's definition or +any value passed to `return` in a coroutine. The `resume` function is the entry +point for executing the `Coroutine` itself. + +The return value of `resume`, `CoroutineState`, looks like: + +```rust +pub enum CoroutineState { + Yielded(Y), + Complete(R), +} +``` + +The `Yielded` variant indicates that the coroutine can later be resumed. This +corresponds to a `yield` point in a coroutine. The `Complete` variant indicates +that the coroutine is complete and cannot be resumed again. Calling `resume` +after a coroutine has returned `Complete` will likely result in a panic of the +program. + +### Closure-like semantics + +The closure-like syntax for coroutines alludes to the fact that they also have +closure-like semantics. Namely: + +* When created, a coroutine executes no code. A closure literal does not + actually execute any of the closure's code on construction, and similarly a + coroutine literal does not execute any code inside the coroutine when + constructed. + +* Coroutines can capture outer variables by reference or by move, and this can + be tweaked with the `move` keyword at the beginning of the closure. Like + closures all coroutines will have an implicit environment which is inferred by + the compiler. Outer variables can be moved into a coroutine for use as the + coroutine progresses. + +* Coroutine literals produce a value with a unique type which implements the + `std::ops::Coroutine` trait. This allows actual execution of the coroutine + through the `Coroutine::resume` method as well as also naming it in return + types and such. + +* Traits like `Send` and `Sync` are automatically implemented for a `Coroutine` + depending on the captured variables of the environment. Unlike closures, + coroutines also depend on variables live across suspension points. This means + that although the ambient environment may be `Send` or `Sync`, the coroutine + itself may not be due to internal variables live across `yield` points being + not-`Send` or not-`Sync`. Note that coroutines do + not implement traits like `Copy` or `Clone` automatically. + +* Whenever a coroutine is dropped it will drop all captured environment + variables. + +### Coroutines as state machines + +In the compiler, coroutines are currently compiled as state machines. Each +`yield` expression will correspond to a different state that stores all live +variables over that suspension point. Resumption of a coroutine will dispatch on +the current state and then execute internally until a `yield` is reached, at +which point all state is saved off in the coroutine and a value is returned. + +Let's take a look at an example to see what's going on here: + +```rust +#![feature(coroutines, coroutine_trait)] + +use std::ops::Coroutine; +use std::pin::Pin; + +fn main() { + let ret = "foo"; + let mut coroutine = move || { + yield 1; + return ret + }; + + Pin::new(&mut coroutine).resume(()); + Pin::new(&mut coroutine).resume(()); +} +``` + +This coroutine literal will compile down to something similar to: + +```rust +#![feature(arbitrary_self_types, coroutines, coroutine_trait)] + +use std::ops::{Coroutine, CoroutineState}; +use std::pin::Pin; + +fn main() { + let ret = "foo"; + let mut coroutine = { + enum __Coroutine { + Start(&'static str), + Yield1(&'static str), + Done, + } + + impl Coroutine for __Coroutine { + type Yield = i32; + type Return = &'static str; + + fn resume(mut self: Pin<&mut Self>, resume: ()) -> CoroutineState { + use std::mem; + match mem::replace(&mut *self, __Coroutine::Done) { + __Coroutine::Start(s) => { + *self = __Coroutine::Yield1(s); + CoroutineState::Yielded(1) + } + + __Coroutine::Yield1(s) => { + *self = __Coroutine::Done; + CoroutineState::Complete(s) + } + + __Coroutine::Done => { + panic!("coroutine resumed after completion") + } + } + } + } + + __Coroutine::Start(ret) + }; + + Pin::new(&mut coroutine).resume(()); + Pin::new(&mut coroutine).resume(()); +} +``` + +Notably here we can see that the compiler is generating a fresh type, +`__Coroutine` in this case. This type has a number of states (represented here +as an `enum`) corresponding to each of the conceptual states of the coroutine. +At the beginning we're closing over our outer variable `foo` and then that +variable is also live over the `yield` point, so it's stored in both states. + +When the coroutine starts it'll immediately yield 1, but it saves off its state +just before it does so indicating that it has reached the yield point. Upon +resuming again we'll execute the `return ret` which returns the `Complete` +state. + +Here we can also note that the `Done` state, if resumed, panics immediately as +it's invalid to resume a completed coroutine. It's also worth noting that this +is just a rough desugaring, not a normative specification for what the compiler +does. diff --git a/src/doc/unstable-book/src/language-features/diagnostic-namespace.md b/src/doc/unstable-book/src/language-features/diagnostic-namespace.md new file mode 100644 index 000000000..7c46811a2 --- /dev/null +++ b/src/doc/unstable-book/src/language-features/diagnostic-namespace.md @@ -0,0 +1,84 @@ +# `diagnostic_namespace` + +The tracking issue for this feature is: [#111996] + +[#111996]: https://github.com/rust-lang/rust/issues/111996 + +------------------------ + +The `diagnostic_namespace` feature permits customization of compilation errors. + +## diagnostic::on_unimplemented + +With [#114452] support for `diagnostic::on_unimplemented` was added. + +When used on a trait declaration, the following options are available: + +* `message` to customize the primary error message +* `note` to add a customized note message to an error message +* `label` to customize the label part of the error message + +The attribute will hint to the compiler to use these in error messages: +```rust +// some library +#![feature(diagnostic_namespace)] + +#[diagnostic::on_unimplemented( + message = "cannot insert element", + label = "cannot be put into a table", + note = "see for more information about the Table api" +)] +pub trait Element { + // ... +} +``` + +```rust,compile_fail,E0277 +# #![feature(diagnostic_namespace)] +# +# #[diagnostic::on_unimplemented( +# message = "cannot insert element", +# label = "cannot be put into a table", +# note = "see for more information about the Table api" +# )] +# pub trait Element { +# // ... +# } +# struct Table; +# impl Table { +# fn insert(&self, element: T) { +# // .. +# } +# } +# fn main() { +# let table = Table; +# let element = (); +// user code +table.insert(element); +# } +``` + +```text +error[E0277]: cannot insert element + --> src/main.rs:24:18 + | +24 | table.insert(element); + | ------ ^^^^^^^ cannot be put into a table + | | + | required by a bound introduced by this call + | + = help: the trait `Element` is not implemented for `` + = note: see for more information about the Table api +note: required by a bound in `Table::insert` + --> src/main.rs:15:18 + | +15 | fn insert(&self, element: T) { + | ^^^^^^^ required by this bound in `Table::insert` + +For more information about this error, try `rustc --explain E0277`. +``` + +See [RFC 3368] for more information. + +[#114452]: https://github.com/rust-lang/rust/pull/114452 +[RFC 3368]: https://github.com/rust-lang/rfcs/blob/master/text/3368-diagnostic-attribute-namespace.md diff --git a/src/doc/unstable-book/src/language-features/generators.md b/src/doc/unstable-book/src/language-features/generators.md deleted file mode 100644 index 7b865c9c6..000000000 --- a/src/doc/unstable-book/src/language-features/generators.md +++ /dev/null @@ -1,246 +0,0 @@ -# `generators` - -The tracking issue for this feature is: [#43122] - -[#43122]: https://github.com/rust-lang/rust/issues/43122 - ------------------------- - -The `generators` feature gate in Rust allows you to define generator or -coroutine literals. A generator is a "resumable function" that syntactically -resembles a closure but compiles to much different semantics in the compiler -itself. The primary feature of a generator is that it can be suspended during -execution to be resumed at a later date. Generators use the `yield` keyword to -"return", and then the caller can `resume` a generator to resume execution just -after the `yield` keyword. - -Generators are an extra-unstable feature in the compiler right now. Added in -[RFC 2033] they're mostly intended right now as a information/constraint -gathering phase. The intent is that experimentation can happen on the nightly -compiler before actual stabilization. A further RFC will be required to -stabilize generators/coroutines and will likely contain at least a few small -tweaks to the overall design. - -[RFC 2033]: https://github.com/rust-lang/rfcs/pull/2033 - -A syntactical example of a generator is: - -```rust -#![feature(generators, generator_trait)] - -use std::ops::{Generator, GeneratorState}; -use std::pin::Pin; - -fn main() { - let mut generator = || { - yield 1; - return "foo" - }; - - match Pin::new(&mut generator).resume(()) { - GeneratorState::Yielded(1) => {} - _ => panic!("unexpected value from resume"), - } - match Pin::new(&mut generator).resume(()) { - GeneratorState::Complete("foo") => {} - _ => panic!("unexpected value from resume"), - } -} -``` - -Generators are closure-like literals which can contain a `yield` statement. The -`yield` statement takes an optional expression of a value to yield out of the -generator. All generator literals implement the `Generator` trait in the -`std::ops` module. The `Generator` trait has one main method, `resume`, which -resumes execution of the generator at the previous suspension point. - -An example of the control flow of generators is that the following example -prints all numbers in order: - -```rust -#![feature(generators, generator_trait)] - -use std::ops::Generator; -use std::pin::Pin; - -fn main() { - let mut generator = || { - println!("2"); - yield; - println!("4"); - }; - - println!("1"); - Pin::new(&mut generator).resume(()); - println!("3"); - Pin::new(&mut generator).resume(()); - println!("5"); -} -``` - -At this time the main intended use case of generators is an implementation -primitive for async/await syntax, but generators will likely be extended to -ergonomic implementations of iterators and other primitives in the future. -Feedback on the design and usage is always appreciated! - -### The `Generator` trait - -The `Generator` trait in `std::ops` currently looks like: - -```rust -# #![feature(arbitrary_self_types, generator_trait)] -# use std::ops::GeneratorState; -# use std::pin::Pin; - -pub trait Generator { - type Yield; - type Return; - fn resume(self: Pin<&mut Self>, resume: R) -> GeneratorState; -} -``` - -The `Generator::Yield` type is the type of values that can be yielded with the -`yield` statement. The `Generator::Return` type is the returned type of the -generator. This is typically the last expression in a generator's definition or -any value passed to `return` in a generator. The `resume` function is the entry -point for executing the `Generator` itself. - -The return value of `resume`, `GeneratorState`, looks like: - -```rust -pub enum GeneratorState { - Yielded(Y), - Complete(R), -} -``` - -The `Yielded` variant indicates that the generator can later be resumed. This -corresponds to a `yield` point in a generator. The `Complete` variant indicates -that the generator is complete and cannot be resumed again. Calling `resume` -after a generator has returned `Complete` will likely result in a panic of the -program. - -### Closure-like semantics - -The closure-like syntax for generators alludes to the fact that they also have -closure-like semantics. Namely: - -* When created, a generator executes no code. A closure literal does not - actually execute any of the closure's code on construction, and similarly a - generator literal does not execute any code inside the generator when - constructed. - -* Generators can capture outer variables by reference or by move, and this can - be tweaked with the `move` keyword at the beginning of the closure. Like - closures all generators will have an implicit environment which is inferred by - the compiler. Outer variables can be moved into a generator for use as the - generator progresses. - -* Generator literals produce a value with a unique type which implements the - `std::ops::Generator` trait. This allows actual execution of the generator - through the `Generator::resume` method as well as also naming it in return - types and such. - -* Traits like `Send` and `Sync` are automatically implemented for a `Generator` - depending on the captured variables of the environment. Unlike closures, - generators also depend on variables live across suspension points. This means - that although the ambient environment may be `Send` or `Sync`, the generator - itself may not be due to internal variables live across `yield` points being - not-`Send` or not-`Sync`. Note that generators do - not implement traits like `Copy` or `Clone` automatically. - -* Whenever a generator is dropped it will drop all captured environment - variables. - -### Generators as state machines - -In the compiler, generators are currently compiled as state machines. Each -`yield` expression will correspond to a different state that stores all live -variables over that suspension point. Resumption of a generator will dispatch on -the current state and then execute internally until a `yield` is reached, at -which point all state is saved off in the generator and a value is returned. - -Let's take a look at an example to see what's going on here: - -```rust -#![feature(generators, generator_trait)] - -use std::ops::Generator; -use std::pin::Pin; - -fn main() { - let ret = "foo"; - let mut generator = move || { - yield 1; - return ret - }; - - Pin::new(&mut generator).resume(()); - Pin::new(&mut generator).resume(()); -} -``` - -This generator literal will compile down to something similar to: - -```rust -#![feature(arbitrary_self_types, generators, generator_trait)] - -use std::ops::{Generator, GeneratorState}; -use std::pin::Pin; - -fn main() { - let ret = "foo"; - let mut generator = { - enum __Generator { - Start(&'static str), - Yield1(&'static str), - Done, - } - - impl Generator for __Generator { - type Yield = i32; - type Return = &'static str; - - fn resume(mut self: Pin<&mut Self>, resume: ()) -> GeneratorState { - use std::mem; - match mem::replace(&mut *self, __Generator::Done) { - __Generator::Start(s) => { - *self = __Generator::Yield1(s); - GeneratorState::Yielded(1) - } - - __Generator::Yield1(s) => { - *self = __Generator::Done; - GeneratorState::Complete(s) - } - - __Generator::Done => { - panic!("generator resumed after completion") - } - } - } - } - - __Generator::Start(ret) - }; - - Pin::new(&mut generator).resume(()); - Pin::new(&mut generator).resume(()); -} -``` - -Notably here we can see that the compiler is generating a fresh type, -`__Generator` in this case. This type has a number of states (represented here -as an `enum`) corresponding to each of the conceptual states of the generator. -At the beginning we're closing over our outer variable `foo` and then that -variable is also live over the `yield` point, so it's stored in both states. - -When the generator starts it'll immediately yield 1, but it saves off its state -just before it does so indicating that it has reached the yield point. Upon -resuming again we'll execute the `return ret` which returns the `Complete` -state. - -Here we can also note that the `Done` state, if resumed, panics immediately as -it's invalid to resume a completed generator. It's also worth noting that this -is just a rough desugaring, not a normative specification for what the compiler -does. diff --git a/src/doc/unstable-book/src/language-features/plugin.md b/src/doc/unstable-book/src/language-features/plugin.md deleted file mode 100644 index 189cc910a..000000000 --- a/src/doc/unstable-book/src/language-features/plugin.md +++ /dev/null @@ -1,114 +0,0 @@ -# `plugin` - -The tracking issue for this feature is: [#29597] - -[#29597]: https://github.com/rust-lang/rust/issues/29597 - - -This feature is part of "compiler plugins." It will often be used with the -`rustc_private` feature. - ------------------------- - -`rustc` can load compiler plugins, which are user-provided libraries that -extend the compiler's behavior with new lint checks, etc. - -A plugin is a dynamic library crate with a designated *registrar* function that -registers extensions with `rustc`. Other crates can load these extensions using -the crate attribute `#![plugin(...)]`. See the -`rustc_driver::plugin` documentation for more about the -mechanics of defining and loading a plugin. - -In the vast majority of cases, a plugin should *only* be used through -`#![plugin]` and not through an `extern crate` item. Linking a plugin would -pull in all of librustc_ast and librustc as dependencies of your crate. This is -generally unwanted unless you are building another plugin. - -The usual practice is to put compiler plugins in their own crate, separate from -any `macro_rules!` macros or ordinary Rust code meant to be used by consumers -of a library. - -# Lint plugins - -Plugins can extend [Rust's lint -infrastructure](../../reference/attributes/diagnostics.md#lint-check-attributes) with -additional checks for code style, safety, etc. Now let's write a plugin -[`lint-plugin-test.rs`](https://github.com/rust-lang/rust/blob/master/tests/ui-fulldeps/auxiliary/lint-plugin-test.rs) -that warns about any item named `lintme`. - -```rust,ignore (requires-stage-2) -#![feature(rustc_private)] - -extern crate rustc_ast; - -// Load rustc as a plugin to get macros -extern crate rustc_driver; -extern crate rustc_lint; -#[macro_use] -extern crate rustc_session; - -use rustc_ast::ast; -use rustc_driver::plugin::Registry; -use rustc_lint::{EarlyContext, EarlyLintPass, LintContext}; - -declare_lint!(TEST_LINT, Warn, "Warn about items named 'lintme'"); - -declare_lint_pass!(Pass => [TEST_LINT]); - -impl EarlyLintPass for Pass { - fn check_item(&mut self, cx: &EarlyContext, it: &ast::Item) { - if it.ident.name.as_str() == "lintme" { - cx.lint(TEST_LINT, "item is named 'lintme'", |lint| lint.set_span(it.span)); - } - } -} - -#[no_mangle] -fn __rustc_plugin_registrar(reg: &mut Registry) { - reg.lint_store.register_lints(&[&TEST_LINT]); - reg.lint_store.register_early_pass(|| Box::new(Pass)); -} -``` - -Then code like - -```rust,ignore (requires-plugin) -#![feature(plugin)] -#![plugin(lint_plugin_test)] - -fn lintme() { } -``` - -will produce a compiler warning: - -```txt -foo.rs:4:1: 4:16 warning: item is named 'lintme', #[warn(test_lint)] on by default -foo.rs:4 fn lintme() { } - ^~~~~~~~~~~~~~~ -``` - -The components of a lint plugin are: - -* one or more `declare_lint!` invocations, which define static `Lint` structs; - -* a struct holding any state needed by the lint pass (here, none); - -* a `LintPass` - implementation defining how to check each syntax element. A single - `LintPass` may call `span_lint` for several different `Lint`s, but should - register them all through the `get_lints` method. - -Lint passes are syntax traversals, but they run at a late stage of compilation -where type information is available. `rustc`'s [built-in -lints](https://github.com/rust-lang/rust/blob/master/compiler/rustc_lint_defs/src/builtin.rs) -mostly use the same infrastructure as lint plugins, and provide examples of how -to access type information. - -Lints defined by plugins are controlled by the usual [attributes and compiler -flags](../../reference/attributes/diagnostics.md#lint-check-attributes), e.g. -`#[allow(test_lint)]` or `-A test-lint`. These identifiers are derived from the -first argument to `declare_lint!`, with appropriate case and punctuation -conversion. - -You can run `rustc -W help foo.rs` to see a list of lints known to `rustc`, -including those provided by plugins loaded by `foo.rs`. diff --git a/src/doc/unstable-book/src/language-features/string-deref-patterns.md b/src/doc/unstable-book/src/language-features/string-deref-patterns.md new file mode 100644 index 000000000..372383075 --- /dev/null +++ b/src/doc/unstable-book/src/language-features/string-deref-patterns.md @@ -0,0 +1,45 @@ +# `string_deref_patterns` + +The tracking issue for this feature is: [#87121] + +[#87121]: https://github.com/rust-lang/rust/issues/87121 + +------------------------ + +This feature permits pattern matching `String` to `&str` through [its `Deref` implementation]. + +```rust +#![feature(string_deref_patterns)] + +pub enum Value { + String(String), + Number(u32), +} + +pub fn is_it_the_answer(value: Value) -> bool { + match value { + Value::String("42") => true, + Value::Number(42) => true, + _ => false, + } +} +``` + +Without this feature other constructs such as match guards have to be used. + +```rust +# pub enum Value { +# String(String), +# Number(u32), +# } +# +pub fn is_it_the_answer(value: Value) -> bool { + match value { + Value::String(s) if s == "42" => true, + Value::Number(42) => true, + _ => false, + } +} +``` + +[its `Deref` implementation]: https://doc.rust-lang.org/std/string/struct.String.html#impl-Deref-for-String diff --git a/src/doc/unstable-book/src/the-unstable-book.md b/src/doc/unstable-book/src/the-unstable-book.md index 9090b134d..0f4fb4056 100644 --- a/src/doc/unstable-book/src/the-unstable-book.md +++ b/src/doc/unstable-book/src/the-unstable-book.md @@ -5,31 +5,31 @@ each one organized by a "feature flag." That is, when using an unstable feature of Rust, you must use a flag, like this: ```rust -#![feature(generators, generator_trait)] +#![feature(coroutines, coroutine_trait)] -use std::ops::{Generator, GeneratorState}; +use std::ops::{Coroutine, CoroutineState}; use std::pin::Pin; fn main() { - let mut generator = || { + let mut coroutine = || { yield 1; return "foo" }; - match Pin::new(&mut generator).resume(()) { - GeneratorState::Yielded(1) => {} + match Pin::new(&mut coroutine).resume(()) { + CoroutineState::Yielded(1) => {} _ => panic!("unexpected value from resume"), } - match Pin::new(&mut generator).resume(()) { - GeneratorState::Complete("foo") => {} + match Pin::new(&mut coroutine).resume(()) { + CoroutineState::Complete("foo") => {} _ => panic!("unexpected value from resume"), } } ``` -The `generators` feature [has a chapter][generators] describing how to use it. +The `coroutines` feature [has a chapter][coroutines] describing how to use it. -[generators]: language-features/generators.md +[coroutines]: language-features/coroutines.md Because this documentation relates to unstable features, we make no guarantees that what is contained here is accurate or up to date. It's developed on a diff --git a/src/etc/completions/x.py.fish b/src/etc/completions/x.py.fish index 151c4120d..487969888 100644 --- a/src/etc/completions/x.py.fish +++ b/src/etc/completions/x.py.fish @@ -12,10 +12,10 @@ complete -c x.py -n "__fish_use_subcommand" -l keep-stage -d 'stage(s) to keep w complete -c x.py -n "__fish_use_subcommand" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_use_subcommand" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_use_subcommand" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_use_subcommand" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_use_subcommand" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_use_subcommand" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_use_subcommand" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_use_subcommand" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_use_subcommand" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_use_subcommand" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_use_subcommand" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_use_subcommand" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_use_subcommand" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -27,6 +27,8 @@ complete -c x.py -n "__fish_use_subcommand" -l include-default-paths -d 'include complete -c x.py -n "__fish_use_subcommand" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_use_subcommand" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_use_subcommand" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_use_subcommand" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_use_subcommand" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_use_subcommand" -s h -l help -d 'Print help' complete -c x.py -n "__fish_use_subcommand" -f -a "build" -d 'Compile either the compiler or libraries' complete -c x.py -n "__fish_use_subcommand" -f -a "check" -d 'Compile either the compiler or libraries, using cargo check' @@ -56,10 +58,10 @@ complete -c x.py -n "__fish_seen_subcommand_from build" -l keep-stage -d 'stage( complete -c x.py -n "__fish_seen_subcommand_from build" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from build" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from build" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from build" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from build" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from build" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from build" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from build" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from build" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from build" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from build" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from build" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from build" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -71,6 +73,8 @@ complete -c x.py -n "__fish_seen_subcommand_from build" -l include-default-paths complete -c x.py -n "__fish_seen_subcommand_from build" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from build" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from build" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from build" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from build" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from build" -s h -l help -d 'Print help (see more with \'--help\')' complete -c x.py -n "__fish_seen_subcommand_from check" -l config -d 'TOML configuration file for build' -r -F complete -c x.py -n "__fish_seen_subcommand_from check" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)" @@ -86,10 +90,10 @@ complete -c x.py -n "__fish_seen_subcommand_from check" -l keep-stage -d 'stage( complete -c x.py -n "__fish_seen_subcommand_from check" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from check" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from check" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from check" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from check" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from check" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from check" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from check" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from check" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from check" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from check" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from check" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from check" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -102,6 +106,8 @@ complete -c x.py -n "__fish_seen_subcommand_from check" -l include-default-paths complete -c x.py -n "__fish_seen_subcommand_from check" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from check" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from check" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from check" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from check" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from check" -s h -l help -d 'Print help (see more with \'--help\')' complete -c x.py -n "__fish_seen_subcommand_from clippy" -s A -d 'clippy lints to allow' -r complete -c x.py -n "__fish_seen_subcommand_from clippy" -s D -d 'clippy lints to deny' -r @@ -121,10 +127,10 @@ complete -c x.py -n "__fish_seen_subcommand_from clippy" -l keep-stage -d 'stage complete -c x.py -n "__fish_seen_subcommand_from clippy" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from clippy" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from clippy" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from clippy" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from clippy" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from clippy" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from clippy" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from clippy" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from clippy" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from clippy" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from clippy" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from clippy" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from clippy" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -137,6 +143,8 @@ complete -c x.py -n "__fish_seen_subcommand_from clippy" -l include-default-path complete -c x.py -n "__fish_seen_subcommand_from clippy" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from clippy" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from clippy" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from clippy" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from clippy" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from clippy" -s h -l help -d 'Print help (see more with \'--help\')' complete -c x.py -n "__fish_seen_subcommand_from fix" -l config -d 'TOML configuration file for build' -r -F complete -c x.py -n "__fish_seen_subcommand_from fix" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)" @@ -152,10 +160,10 @@ complete -c x.py -n "__fish_seen_subcommand_from fix" -l keep-stage -d 'stage(s) complete -c x.py -n "__fish_seen_subcommand_from fix" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from fix" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from fix" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from fix" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from fix" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from fix" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from fix" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from fix" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from fix" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from fix" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from fix" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from fix" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from fix" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -167,6 +175,8 @@ complete -c x.py -n "__fish_seen_subcommand_from fix" -l include-default-paths - complete -c x.py -n "__fish_seen_subcommand_from fix" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from fix" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from fix" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from fix" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from fix" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from fix" -s h -l help -d 'Print help (see more with \'--help\')' complete -c x.py -n "__fish_seen_subcommand_from fmt" -l config -d 'TOML configuration file for build' -r -F complete -c x.py -n "__fish_seen_subcommand_from fmt" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)" @@ -182,10 +192,10 @@ complete -c x.py -n "__fish_seen_subcommand_from fmt" -l keep-stage -d 'stage(s) complete -c x.py -n "__fish_seen_subcommand_from fmt" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from fmt" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from fmt" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from fmt" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from fmt" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from fmt" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from fmt" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from fmt" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from fmt" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from fmt" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from fmt" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from fmt" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from fmt" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -198,6 +208,8 @@ complete -c x.py -n "__fish_seen_subcommand_from fmt" -l include-default-paths - complete -c x.py -n "__fish_seen_subcommand_from fmt" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from fmt" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from fmt" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from fmt" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from fmt" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from fmt" -s h -l help -d 'Print help (see more with \'--help\')' complete -c x.py -n "__fish_seen_subcommand_from doc" -l config -d 'TOML configuration file for build' -r -F complete -c x.py -n "__fish_seen_subcommand_from doc" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)" @@ -213,10 +225,10 @@ complete -c x.py -n "__fish_seen_subcommand_from doc" -l keep-stage -d 'stage(s) complete -c x.py -n "__fish_seen_subcommand_from doc" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from doc" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from doc" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from doc" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from doc" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from doc" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from doc" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from doc" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from doc" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from doc" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from doc" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from doc" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from doc" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -230,6 +242,8 @@ complete -c x.py -n "__fish_seen_subcommand_from doc" -l include-default-paths - complete -c x.py -n "__fish_seen_subcommand_from doc" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from doc" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from doc" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from doc" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from doc" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from doc" -s h -l help -d 'Print help (see more with \'--help\')' complete -c x.py -n "__fish_seen_subcommand_from test" -l skip -d 'skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times' -r -F complete -c x.py -n "__fish_seen_subcommand_from test" -l test-args -d 'extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)' -r @@ -251,10 +265,10 @@ complete -c x.py -n "__fish_seen_subcommand_from test" -l keep-stage -d 'stage(s complete -c x.py -n "__fish_seen_subcommand_from test" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from test" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from test" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from test" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from test" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from test" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from test" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from test" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from test" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from test" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from test" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from test" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from test" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -273,6 +287,8 @@ complete -c x.py -n "__fish_seen_subcommand_from test" -l include-default-paths complete -c x.py -n "__fish_seen_subcommand_from test" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from test" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from test" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from test" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from test" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from test" -s h -l help -d 'Print help (see more with \'--help\')' complete -c x.py -n "__fish_seen_subcommand_from bench" -l test-args -r complete -c x.py -n "__fish_seen_subcommand_from bench" -l config -d 'TOML configuration file for build' -r -F @@ -289,10 +305,10 @@ complete -c x.py -n "__fish_seen_subcommand_from bench" -l keep-stage -d 'stage( complete -c x.py -n "__fish_seen_subcommand_from bench" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from bench" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from bench" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from bench" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from bench" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from bench" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from bench" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from bench" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from bench" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from bench" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from bench" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from bench" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from bench" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -304,6 +320,8 @@ complete -c x.py -n "__fish_seen_subcommand_from bench" -l include-default-paths complete -c x.py -n "__fish_seen_subcommand_from bench" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from bench" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from bench" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from bench" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from bench" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from bench" -s h -l help -d 'Print help' complete -c x.py -n "__fish_seen_subcommand_from clean" -l stage -d 'Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used' -r complete -c x.py -n "__fish_seen_subcommand_from clean" -l config -d 'TOML configuration file for build' -r -F @@ -319,10 +337,10 @@ complete -c x.py -n "__fish_seen_subcommand_from clean" -l keep-stage -d 'stage( complete -c x.py -n "__fish_seen_subcommand_from clean" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from clean" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from clean" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from clean" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from clean" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from clean" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from clean" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from clean" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from clean" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from clean" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from clean" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from clean" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from clean" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -335,6 +353,8 @@ complete -c x.py -n "__fish_seen_subcommand_from clean" -l include-default-paths complete -c x.py -n "__fish_seen_subcommand_from clean" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from clean" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from clean" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from clean" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from clean" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from clean" -s h -l help -d 'Print help' complete -c x.py -n "__fish_seen_subcommand_from dist" -l config -d 'TOML configuration file for build' -r -F complete -c x.py -n "__fish_seen_subcommand_from dist" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)" @@ -350,10 +370,10 @@ complete -c x.py -n "__fish_seen_subcommand_from dist" -l keep-stage -d 'stage(s complete -c x.py -n "__fish_seen_subcommand_from dist" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from dist" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from dist" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from dist" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from dist" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from dist" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from dist" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from dist" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from dist" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from dist" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from dist" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from dist" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from dist" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -365,6 +385,8 @@ complete -c x.py -n "__fish_seen_subcommand_from dist" -l include-default-paths complete -c x.py -n "__fish_seen_subcommand_from dist" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from dist" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from dist" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from dist" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from dist" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from dist" -s h -l help -d 'Print help' complete -c x.py -n "__fish_seen_subcommand_from install" -l config -d 'TOML configuration file for build' -r -F complete -c x.py -n "__fish_seen_subcommand_from install" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)" @@ -380,10 +402,10 @@ complete -c x.py -n "__fish_seen_subcommand_from install" -l keep-stage -d 'stag complete -c x.py -n "__fish_seen_subcommand_from install" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from install" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from install" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from install" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from install" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from install" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from install" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from install" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from install" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from install" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from install" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from install" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from install" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -395,6 +417,8 @@ complete -c x.py -n "__fish_seen_subcommand_from install" -l include-default-pat complete -c x.py -n "__fish_seen_subcommand_from install" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from install" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from install" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from install" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from install" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from install" -s h -l help -d 'Print help' complete -c x.py -n "__fish_seen_subcommand_from run" -l args -d 'arguments for the tool' -r complete -c x.py -n "__fish_seen_subcommand_from run" -l config -d 'TOML configuration file for build' -r -F @@ -411,10 +435,10 @@ complete -c x.py -n "__fish_seen_subcommand_from run" -l keep-stage -d 'stage(s) complete -c x.py -n "__fish_seen_subcommand_from run" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from run" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from run" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from run" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from run" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from run" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from run" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from run" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from run" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from run" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from run" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from run" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from run" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -426,6 +450,8 @@ complete -c x.py -n "__fish_seen_subcommand_from run" -l include-default-paths - complete -c x.py -n "__fish_seen_subcommand_from run" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from run" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from run" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from run" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from run" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from run" -s h -l help -d 'Print help (see more with \'--help\')' complete -c x.py -n "__fish_seen_subcommand_from setup" -l config -d 'TOML configuration file for build' -r -F complete -c x.py -n "__fish_seen_subcommand_from setup" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)" @@ -441,10 +467,10 @@ complete -c x.py -n "__fish_seen_subcommand_from setup" -l keep-stage -d 'stage( complete -c x.py -n "__fish_seen_subcommand_from setup" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from setup" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from setup" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from setup" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from setup" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from setup" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from setup" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from setup" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from setup" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from setup" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from setup" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from setup" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from setup" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -456,6 +482,8 @@ complete -c x.py -n "__fish_seen_subcommand_from setup" -l include-default-paths complete -c x.py -n "__fish_seen_subcommand_from setup" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from setup" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from setup" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from setup" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from setup" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from setup" -s h -l help -d 'Print help (see more with \'--help\')' complete -c x.py -n "__fish_seen_subcommand_from suggest" -l config -d 'TOML configuration file for build' -r -F complete -c x.py -n "__fish_seen_subcommand_from suggest" -l build-dir -d 'Build directory, overrides `build.build-dir` in `config.toml`' -r -f -a "(__fish_complete_directories)" @@ -471,10 +499,10 @@ complete -c x.py -n "__fish_seen_subcommand_from suggest" -l keep-stage -d 'stag complete -c x.py -n "__fish_seen_subcommand_from suggest" -l keep-stage-std -d 'stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)' -r -f complete -c x.py -n "__fish_seen_subcommand_from suggest" -l src -d 'path to the root of the rust checkout' -r -f -a "(__fish_complete_directories)" complete -c x.py -n "__fish_seen_subcommand_from suggest" -s j -l jobs -d 'number of jobs to run in parallel' -r -f -complete -c x.py -n "__fish_seen_subcommand_from suggest" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny ,warn ,default }" +complete -c x.py -n "__fish_seen_subcommand_from suggest" -l warnings -d 'if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour' -r -f -a "{deny '',warn '',default ''}" complete -c x.py -n "__fish_seen_subcommand_from suggest" -l error-format -d 'rustc error format' -r -f -complete -c x.py -n "__fish_seen_subcommand_from suggest" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always ,never ,auto }" -complete -c x.py -n "__fish_seen_subcommand_from suggest" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true ,false }" +complete -c x.py -n "__fish_seen_subcommand_from suggest" -l color -d 'whether to use color in cargo and rustc output' -r -f -a "{always '',never '',auto ''}" +complete -c x.py -n "__fish_seen_subcommand_from suggest" -l llvm-skip-rebuild -d 'whether rebuilding llvm should be skipped, overriding `skip-rebuld` in config.toml' -r -f -a "{true '',false ''}" complete -c x.py -n "__fish_seen_subcommand_from suggest" -l rust-profile-generate -d 'generate PGO profile with rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from suggest" -l rust-profile-use -d 'use PGO profile for rustc build' -r -F complete -c x.py -n "__fish_seen_subcommand_from suggest" -l llvm-profile-use -d 'use PGO profile for LLVM build' -r -F @@ -487,4 +515,6 @@ complete -c x.py -n "__fish_seen_subcommand_from suggest" -l include-default-pat complete -c x.py -n "__fish_seen_subcommand_from suggest" -l dry-run -d 'dry run; don\'t build anything' complete -c x.py -n "__fish_seen_subcommand_from suggest" -l json-output -d 'use message-format=json' complete -c x.py -n "__fish_seen_subcommand_from suggest" -l llvm-profile-generate -d 'generate PGO profile with llvm built for rustc' +complete -c x.py -n "__fish_seen_subcommand_from suggest" -l enable-bolt-settings -d 'Enable BOLT link flags' +complete -c x.py -n "__fish_seen_subcommand_from suggest" -l skip-stage0-validation -d 'Skip stage0 compiler validation' complete -c x.py -n "__fish_seen_subcommand_from suggest" -s h -l help -d 'Print help (see more with \'--help\')' diff --git a/src/etc/completions/x.py.ps1 b/src/etc/completions/x.py.ps1 index cafb8eed1..2fed1be72 100644 --- a/src/etc/completions/x.py.ps1 +++ b/src/etc/completions/x.py.ps1 @@ -53,6 +53,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help') [CompletionResult]::new('build', 'build', [CompletionResultType]::ParameterValue, 'Compile either the compiler or libraries') @@ -104,6 +106,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break @@ -142,15 +146,17 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break } 'x.py;clippy' { - [CompletionResult]::new('-A', 'A', [CompletionResultType]::ParameterName, 'clippy lints to allow') - [CompletionResult]::new('-D', 'D', [CompletionResultType]::ParameterName, 'clippy lints to deny') - [CompletionResult]::new('-W', 'W', [CompletionResultType]::ParameterName, 'clippy lints to warn on') - [CompletionResult]::new('-F', 'F', [CompletionResultType]::ParameterName, 'clippy lints to forbid') + [CompletionResult]::new('-A', 'A ', [CompletionResultType]::ParameterName, 'clippy lints to allow') + [CompletionResult]::new('-D', 'D ', [CompletionResultType]::ParameterName, 'clippy lints to deny') + [CompletionResult]::new('-W', 'W ', [CompletionResultType]::ParameterName, 'clippy lints to warn on') + [CompletionResult]::new('-F', 'F ', [CompletionResultType]::ParameterName, 'clippy lints to forbid') [CompletionResult]::new('--config', 'config', [CompletionResultType]::ParameterName, 'TOML configuration file for build') [CompletionResult]::new('--build-dir', 'build-dir', [CompletionResultType]::ParameterName, 'Build directory, overrides `build.build-dir` in `config.toml`') [CompletionResult]::new('--build', 'build', [CompletionResultType]::ParameterName, 'build target of the stage0 compiler') @@ -184,6 +190,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break @@ -221,6 +229,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break @@ -259,6 +269,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break @@ -298,6 +310,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break @@ -348,6 +362,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break @@ -386,6 +402,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help') break @@ -424,6 +442,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help') break @@ -461,6 +481,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help') break @@ -498,6 +520,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help') break @@ -536,6 +560,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break @@ -573,6 +599,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break @@ -611,6 +639,8 @@ Register-ArgumentCompleter -Native -CommandName 'x.py' -ScriptBlock { [CompletionResult]::new('--dry-run', 'dry-run', [CompletionResultType]::ParameterName, 'dry run; don''t build anything') [CompletionResult]::new('--json-output', 'json-output', [CompletionResultType]::ParameterName, 'use message-format=json') [CompletionResult]::new('--llvm-profile-generate', 'llvm-profile-generate', [CompletionResultType]::ParameterName, 'generate PGO profile with llvm built for rustc') + [CompletionResult]::new('--enable-bolt-settings', 'enable-bolt-settings', [CompletionResultType]::ParameterName, 'Enable BOLT link flags') + [CompletionResult]::new('--skip-stage0-validation', 'skip-stage0-validation', [CompletionResultType]::ParameterName, 'Skip stage0 compiler validation') [CompletionResult]::new('-h', 'h', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') [CompletionResult]::new('--help', 'help', [CompletionResultType]::ParameterName, 'Print help (see more with ''--help'')') break diff --git a/src/etc/completions/x.py.sh b/src/etc/completions/x.py.sh index 3c57e71bd..f22d7e3e1 100644 --- a/src/etc/completions/x.py.sh +++ b/src/etc/completions/x.py.sh @@ -61,7 +61,7 @@ _x.py() { case "${cmd}" in x.py) - opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]... build check clippy fix fmt doc test bench clean dist install run setup suggest" + opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]... build check clippy fix fmt doc test bench clean dist install run setup suggest" if [[ ${cur} == -* || ${COMP_CWORD} -eq 1 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -171,7 +171,7 @@ _x.py() { return 0 ;; x.py__bench) - opts="-v -i -j -h --test-args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --test-args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -285,7 +285,7 @@ _x.py() { return 0 ;; x.py__build) - opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -395,7 +395,7 @@ _x.py() { return 0 ;; x.py__check) - opts="-v -i -j -h --all-targets --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --all-targets --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -505,7 +505,7 @@ _x.py() { return 0 ;; x.py__clean) - opts="-v -i -j -h --all --stage --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --all --stage --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -615,7 +615,7 @@ _x.py() { return 0 ;; x.py__clippy) - opts="-A -D -W -F -v -i -j -h --fix --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-A -D -W -F -v -i -j -h --fix --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -741,7 +741,7 @@ _x.py() { return 0 ;; x.py__dist) - opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -851,7 +851,7 @@ _x.py() { return 0 ;; x.py__doc) - opts="-v -i -j -h --open --json --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --open --json --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -961,7 +961,7 @@ _x.py() { return 0 ;; x.py__fix) - opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -1071,7 +1071,7 @@ _x.py() { return 0 ;; x.py__fmt) - opts="-v -i -j -h --check --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --check --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -1181,7 +1181,7 @@ _x.py() { return 0 ;; x.py__install) - opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -1291,7 +1291,7 @@ _x.py() { return 0 ;; x.py__run) - opts="-v -i -j -h --args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --args --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -1405,7 +1405,7 @@ _x.py() { return 0 ;; x.py__setup) - opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [|hook|vscode|link] [PATHS]... [ARGS]..." + opts="-v -i -j -h --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [|hook|vscode|link] [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -1515,7 +1515,7 @@ _x.py() { return 0 ;; x.py__suggest) - opts="-v -i -j -h --run --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --run --verbose --incremental --config --build-dir --build --host --target --exclude --skip --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -1625,7 +1625,7 @@ _x.py() { return 0 ;; x.py__test) - opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --extra-checks --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --reproducible-artifact --set --help [PATHS]... [ARGS]..." + opts="-v -i -j -h --no-fail-fast --skip --test-args --rustc-args --no-doc --doc --bless --extra-checks --force-rerun --only-modified --compare-mode --pass --run --rustfix-coverage --verbose --incremental --config --build-dir --build --host --target --exclude --include-default-paths --rustc-error-format --on-fail --dry-run --stage --keep-stage --keep-stage-std --src --jobs --warnings --error-format --json-output --color --llvm-skip-rebuild --rust-profile-generate --rust-profile-use --llvm-profile-use --llvm-profile-generate --enable-bolt-settings --skip-stage0-validation --reproducible-artifact --set --help [PATHS]... [ARGS]..." if [[ ${cur} == -* || ${COMP_CWORD} -eq 2 ]] ; then COMPREPLY=( $(compgen -W "${opts}" -- "${cur}") ) return 0 @@ -1761,4 +1761,4 @@ _x.py() { esac } -complete -F _x.py -o bashdefault -o default x.py +complete -F _x.py -o nosort -o bashdefault -o default x.py diff --git a/src/etc/completions/x.py.zsh b/src/etc/completions/x.py.zsh new file mode 100644 index 000000000..1e5a7b5aa --- /dev/null +++ b/src/etc/completions/x.py.zsh @@ -0,0 +1,766 @@ +#compdef x.py + +autoload -U is-at-least + +_x.py() { + typeset -A opt_args + typeset -a _arguments_options + local ret=1 + + if is-at-least 5.2; then + _arguments_options=(-s -S -C) + else + _arguments_options=(-s -C) + fi + + local context curcontext="$curcontext" state line + _arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help]' \ +'--help[Print help]' \ +'::paths -- paths for the subcommand:_files' \ +'::free_args -- arguments passed to subcommands:' \ +":: :_x.py_commands" \ +"*::: :->bootstrap" \ +&& ret=0 + case $state in + (bootstrap) + words=($line[3] "${words[@]}") + (( CURRENT += 1 )) + curcontext="${curcontext%:*:*}:x.py-command-$line[3]:" + case $line[3] in + (build) +_arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(check) +_arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'--all-targets[Check all targets]' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(clippy) +_arguments "${_arguments_options[@]}" \ +'*-A+[clippy lints to allow]:LINT: ' \ +'*-D+[clippy lints to deny]:LINT: ' \ +'*-W+[clippy lints to warn on]:LINT: ' \ +'*-F+[clippy lints to forbid]:LINT: ' \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'--fix[]' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(fix) +_arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(fmt) +_arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'--check[check formatting instead of applying]' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(doc) +_arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'--open[open the docs in a browser]' \ +'--json[render the documentation in JSON format in addition to the usual HTML format]' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(test) +_arguments "${_arguments_options[@]}" \ +'*--skip=[skips tests matching SUBSTRING, if supported by test tool. May be passed multiple times]:SUBSTRING:_files' \ +'*--test-args=[extra arguments to be passed for the test tool being used (e.g. libtest, compiletest or rustdoc)]:ARGS: ' \ +'*--rustc-args=[extra options to pass the compiler when running tests]:ARGS: ' \ +'--extra-checks=[comma-separated list of other files types to check (accepts py, py\:lint, py\:fmt, shell)]:EXTRA_CHECKS: ' \ +'--compare-mode=[mode describing what file the actual ui output will be compared to]:COMPARE MODE: ' \ +'--pass=[force {check,build,run}-pass tests to this mode]:check | build | run: ' \ +'--run=[whether to execute run-* tests]:auto | always | never: ' \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'--no-fail-fast[run all tests regardless of failure]' \ +'--no-doc[do not run doc tests]' \ +'--doc[only run doc tests]' \ +'--bless[whether to automatically update stderr/stdout files]' \ +'--force-rerun[rerun tests even if the inputs are unchanged]' \ +'--only-modified[only run tests that result has been changed]' \ +'--rustfix-coverage[enable this to generate a Rustfix coverage file, which is saved in \`//rustfix_missing_coverage.txt\`]' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(bench) +_arguments "${_arguments_options[@]}" \ +'*--test-args=[]:TEST_ARGS: ' \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help]' \ +'--help[Print help]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(clean) +_arguments "${_arguments_options[@]}" \ +'--stage=[Clean a specific stage without touching other artifacts. By default, every stage is cleaned if this option is not used]:N: ' \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'--all[Clean the entire build directory (not used by default)]' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help]' \ +'--help[Print help]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(dist) +_arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help]' \ +'--help[Print help]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(install) +_arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help]' \ +'--help[Print help]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(run) +_arguments "${_arguments_options[@]}" \ +'*--args=[arguments for the tool]:ARGS: ' \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(setup) +_arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'::profile -- Either the profile for `config.toml` or another setup action. May be omitted to set up interactively:_files' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; +(suggest) +_arguments "${_arguments_options[@]}" \ +'--config=[TOML configuration file for build]:FILE:_files' \ +'--build-dir=[Build directory, overrides \`build.build-dir\` in \`config.toml\`]:DIR:_files -/' \ +'--build=[build target of the stage0 compiler]:BUILD:( )' \ +'--host=[host targets to build]:HOST:( )' \ +'--target=[target targets to build]:TARGET:( )' \ +'*--exclude=[build paths to exclude]:PATH:_files' \ +'*--skip=[build paths to skip]:PATH:_files' \ +'--rustc-error-format=[]:RUSTC_ERROR_FORMAT:( )' \ +'--on-fail=[command to run on failure]:CMD:_cmdstring' \ +'--stage=[stage to build (indicates compiler to use/test, e.g., stage 0 uses the bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)]:N:( )' \ +'*--keep-stage=[stage(s) to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'*--keep-stage-std=[stage(s) of the standard library to keep without recompiling (pass multiple times to keep e.g., both stages 0 and 1)]:N:( )' \ +'--src=[path to the root of the rust checkout]:DIR:_files -/' \ +'-j+[number of jobs to run in parallel]:JOBS:( )' \ +'--jobs=[number of jobs to run in parallel]:JOBS:( )' \ +'--warnings=[if value is deny, will deny warnings if value is warn, will emit warnings otherwise, use the default configured behaviour]:deny|warn:(deny warn default)' \ +'--error-format=[rustc error format]:FORMAT:( )' \ +'--color=[whether to use color in cargo and rustc output]:STYLE:(always never auto)' \ +'--llvm-skip-rebuild=[whether rebuilding llvm should be skipped, overriding \`skip-rebuld\` in config.toml]:VALUE:(true false)' \ +'--rust-profile-generate=[generate PGO profile with rustc build]:PROFILE:_files' \ +'--rust-profile-use=[use PGO profile for rustc build]:PROFILE:_files' \ +'--llvm-profile-use=[use PGO profile for LLVM build]:PROFILE:_files' \ +'*--reproducible-artifact=[Additional reproducible artifacts that should be added to the reproducible artifacts archive]:REPRODUCIBLE_ARTIFACT: ' \ +'*--set=[override options in config.toml]:section.option=value:( )' \ +'--run[run suggested tests]' \ +'*-v[use verbose output (-vv for very verbose)]' \ +'*--verbose[use verbose output (-vv for very verbose)]' \ +'-i[use incremental compilation]' \ +'--incremental[use incremental compilation]' \ +'--include-default-paths[include default paths in addition to the provided ones]' \ +'--dry-run[dry run; don'\''t build anything]' \ +'--json-output[use message-format=json]' \ +'--llvm-profile-generate[generate PGO profile with llvm built for rustc]' \ +'--enable-bolt-settings[Enable BOLT link flags]' \ +'--skip-stage0-validation[Skip stage0 compiler validation]' \ +'-h[Print help (see more with '\''--help'\'')]' \ +'--help[Print help (see more with '\''--help'\'')]' \ +'*::paths -- paths for the subcommand:_files' \ +&& ret=0 +;; + esac + ;; +esac +} + +(( $+functions[_x.py_commands] )) || +_x.py_commands() { + local commands; commands=( +'build:Compile either the compiler or libraries' \ +'check:Compile either the compiler or libraries, using cargo check' \ +'clippy:Run Clippy (uses rustup/cargo-installed clippy binary)' \ +'fix:Run cargo fix' \ +'fmt:Run rustfmt' \ +'doc:Build documentation' \ +'test:Build and run some test suites' \ +'bench:Build and run some benchmarks' \ +'clean:Clean out build directories' \ +'dist:Build distribution artifacts' \ +'install:Install distribution artifacts' \ +'run:Run tools contained in this repository' \ +'setup:Set up the environment for development' \ +'suggest:Suggest a subset of tests to run, based on modified files' \ + ) + _describe -t commands 'x.py commands' commands "$@" +} +(( $+functions[_x.py__bench_commands] )) || +_x.py__bench_commands() { + local commands; commands=() + _describe -t commands 'x.py bench commands' commands "$@" +} +(( $+functions[_x.py__build_commands] )) || +_x.py__build_commands() { + local commands; commands=() + _describe -t commands 'x.py build commands' commands "$@" +} +(( $+functions[_x.py__check_commands] )) || +_x.py__check_commands() { + local commands; commands=() + _describe -t commands 'x.py check commands' commands "$@" +} +(( $+functions[_x.py__clean_commands] )) || +_x.py__clean_commands() { + local commands; commands=() + _describe -t commands 'x.py clean commands' commands "$@" +} +(( $+functions[_x.py__clippy_commands] )) || +_x.py__clippy_commands() { + local commands; commands=() + _describe -t commands 'x.py clippy commands' commands "$@" +} +(( $+functions[_x.py__dist_commands] )) || +_x.py__dist_commands() { + local commands; commands=() + _describe -t commands 'x.py dist commands' commands "$@" +} +(( $+functions[_x.py__doc_commands] )) || +_x.py__doc_commands() { + local commands; commands=() + _describe -t commands 'x.py doc commands' commands "$@" +} +(( $+functions[_x.py__fix_commands] )) || +_x.py__fix_commands() { + local commands; commands=() + _describe -t commands 'x.py fix commands' commands "$@" +} +(( $+functions[_x.py__fmt_commands] )) || +_x.py__fmt_commands() { + local commands; commands=() + _describe -t commands 'x.py fmt commands' commands "$@" +} +(( $+functions[_x.py__install_commands] )) || +_x.py__install_commands() { + local commands; commands=() + _describe -t commands 'x.py install commands' commands "$@" +} +(( $+functions[_x.py__run_commands] )) || +_x.py__run_commands() { + local commands; commands=() + _describe -t commands 'x.py run commands' commands "$@" +} +(( $+functions[_x.py__setup_commands] )) || +_x.py__setup_commands() { + local commands; commands=() + _describe -t commands 'x.py setup commands' commands "$@" +} +(( $+functions[_x.py__suggest_commands] )) || +_x.py__suggest_commands() { + local commands; commands=() + _describe -t commands 'x.py suggest commands' commands "$@" +} +(( $+functions[_x.py__test_commands] )) || +_x.py__test_commands() { + local commands; commands=() + _describe -t commands 'x.py test commands' commands "$@" +} + +if [ "$funcstack[1]" = "_x.py" ]; then + _x.py "$@" +else + compdef _x.py x.py +fi diff --git a/src/etc/gdb_lookup.py b/src/etc/gdb_lookup.py index 8171cb4e9..f3ac9c109 100644 --- a/src/etc/gdb_lookup.py +++ b/src/etc/gdb_lookup.py @@ -1,4 +1,5 @@ import gdb +import gdb.printing import re from gdb_providers import * @@ -9,7 +10,7 @@ _gdb_version_matched = re.search('([0-9]+)\\.([0-9]+)', gdb.VERSION) gdb_version = [int(num) for num in _gdb_version_matched.groups()] if _gdb_version_matched else [] def register_printers(objfile): - objfile.pretty_printers.append(lookup) + objfile.pretty_printers.append(printer) # BACKCOMPAT: rust 1.35 @@ -38,58 +39,80 @@ def check_enum_discriminant(valobj): return True -def lookup(valobj): - rust_type = classify_rust_type(valobj.type) - - if rust_type == RustType.ENUM: - # use enum provider only for GDB <7.12 - if gdb_version[0] < 7 or (gdb_version[0] == 7 and gdb_version[1] < 12): - if check_enum_discriminant(valobj): - return EnumProvider(valobj) - - if rust_type == RustType.STD_STRING: - return StdStringProvider(valobj) - if rust_type == RustType.STD_OS_STRING: - return StdOsStringProvider(valobj) - if rust_type == RustType.STD_STR: - return StdStrProvider(valobj) - if rust_type == RustType.STD_SLICE: - return StdSliceProvider(valobj) - if rust_type == RustType.STD_VEC: - return StdVecProvider(valobj) - if rust_type == RustType.STD_VEC_DEQUE: - return StdVecDequeProvider(valobj) - if rust_type == RustType.STD_BTREE_SET: - return StdBTreeSetProvider(valobj) - if rust_type == RustType.STD_BTREE_MAP: - return StdBTreeMapProvider(valobj) - if rust_type == RustType.STD_HASH_MAP: - if is_hashbrown_hashmap(valobj): - return StdHashMapProvider(valobj) - else: - return StdOldHashMapProvider(valobj) - if rust_type == RustType.STD_HASH_SET: - hash_map = valobj[valobj.type.fields()[0]] - if is_hashbrown_hashmap(hash_map): - return StdHashMapProvider(valobj, show_values=False) - else: - return StdOldHashMapProvider(hash_map, show_values=False) - - if rust_type == RustType.STD_RC: - return StdRcProvider(valobj) - if rust_type == RustType.STD_ARC: - return StdRcProvider(valobj, is_atomic=True) - - if rust_type == RustType.STD_CELL: - return StdCellProvider(valobj) - if rust_type == RustType.STD_REF: - return StdRefProvider(valobj) - if rust_type == RustType.STD_REF_MUT: - return StdRefProvider(valobj) - if rust_type == RustType.STD_REF_CELL: - return StdRefCellProvider(valobj) - - if rust_type == RustType.STD_NONZERO_NUMBER: - return StdNonZeroNumberProvider(valobj) - +# Helper for enum printing that checks the discriminant. Only used in +# older gdb. +def enum_provider(valobj): + if check_enum_discriminant(valobj): + return EnumProvider(valobj) return None + + +# Helper to handle both old and new hash maps. +def hashmap_provider(valobj): + if is_hashbrown_hashmap(valobj): + return StdHashMapProvider(valobj) + else: + return StdOldHashMapProvider(valobj) + + +# Helper to handle both old and new hash sets. +def hashset_provider(valobj): + hash_map = valobj[valobj.type.fields()[0]] + if is_hashbrown_hashmap(hash_map): + return StdHashMapProvider(valobj, show_values=False) + else: + return StdOldHashMapProvider(hash_map, show_values=False) + + +class PrintByRustType(gdb.printing.SubPrettyPrinter): + def __init__(self, name, provider): + super(PrintByRustType, self).__init__(name) + self.provider = provider + + def __call__(self, val): + if self.enabled: + return self.provider(val) + return None + + +class RustPrettyPrinter(gdb.printing.PrettyPrinter): + def __init__(self, name): + super(RustPrettyPrinter, self).__init__(name, []) + self.type_map = {} + + def add(self, rust_type, provider): + # Just use the rust_type as the name. + printer = PrintByRustType(rust_type, provider) + self.type_map[rust_type] = printer + self.subprinters.append(printer) + + def __call__(self, valobj): + rust_type = classify_rust_type(valobj.type) + if rust_type in self.type_map: + return self.type_map[rust_type](valobj) + return None + + +printer = RustPrettyPrinter("rust") +# use enum provider only for GDB <7.12 +if gdb_version[0] < 7 or (gdb_version[0] == 7 and gdb_version[1] < 12): + printer.add(RustType.ENUM, enum_provider) +printer.add(RustType.STD_STRING, StdStringProvider) +printer.add(RustType.STD_OS_STRING, StdOsStringProvider) +printer.add(RustType.STD_STR, StdStrProvider) +printer.add(RustType.STD_SLICE, StdSliceProvider) +printer.add(RustType.STD_VEC, StdVecProvider) +printer.add(RustType.STD_VEC_DEQUE, StdVecDequeProvider) +printer.add(RustType.STD_BTREE_SET, StdBTreeSetProvider) +printer.add(RustType.STD_BTREE_MAP, StdBTreeMapProvider) +printer.add(RustType.STD_HASH_MAP, hashmap_provider) +printer.add(RustType.STD_HASH_SET, hashset_provider) +printer.add(RustType.STD_RC, StdRcProvider) +printer.add(RustType.STD_ARC, lambda valobj: StdRcProvider(valobj, is_atomic=True)) + +printer.add(RustType.STD_CELL, StdCellProvider) +printer.add(RustType.STD_REF, StdRefProvider) +printer.add(RustType.STD_REF_MUT, StdRefProvider) +printer.add(RustType.STD_REF_CELL, StdRefCellProvider) + +printer.add(RustType.STD_NONZERO_NUMBER, StdNonZeroNumberProvider) diff --git a/src/etc/gdb_providers.py b/src/etc/gdb_providers.py index 32b8d8e24..e851aa626 100644 --- a/src/etc/gdb_providers.py +++ b/src/etc/gdb_providers.py @@ -18,70 +18,79 @@ def unwrap_unique_or_non_null(unique_or_nonnull): return ptr if ptr.type.code == gdb.TYPE_CODE_PTR else ptr[ptr.type.fields()[0]] -class EnumProvider: +# GDB 14 has a tag class that indicates that extension methods are ok +# to call. Use of this tag only requires that printers hide local +# attributes and methods by prefixing them with "_". +if hasattr(gdb, 'ValuePrinter'): + printer_base = gdb.ValuePrinter +else: + printer_base = object + + +class EnumProvider(printer_base): def __init__(self, valobj): content = valobj[valobj.type.fields()[0]] fields = content.type.fields() - self.empty = len(fields) == 0 - if not self.empty: + self._empty = len(fields) == 0 + if not self._empty: if len(fields) == 1: discriminant = 0 else: discriminant = int(content[fields[0]]) + 1 - self.active_variant = content[fields[discriminant]] - self.name = fields[discriminant].name - self.full_name = "{}::{}".format(valobj.type.name, self.name) + self._active_variant = content[fields[discriminant]] + self._name = fields[discriminant].name + self._full_name = "{}::{}".format(valobj.type.name, self._name) else: - self.full_name = valobj.type.name + self._full_name = valobj.type.name def to_string(self): - return self.full_name + return self._full_name def children(self): - if not self.empty: - yield self.name, self.active_variant + if not self._empty: + yield self._name, self._active_variant -class StdStringProvider: +class StdStringProvider(printer_base): def __init__(self, valobj): - self.valobj = valobj + self._valobj = valobj vec = valobj["vec"] - self.length = int(vec["len"]) - self.data_ptr = unwrap_unique_or_non_null(vec["buf"]["ptr"]) + self._length = int(vec["len"]) + self._data_ptr = unwrap_unique_or_non_null(vec["buf"]["ptr"]) def to_string(self): - return self.data_ptr.lazy_string(encoding="utf-8", length=self.length) + return self._data_ptr.lazy_string(encoding="utf-8", length=self._length) @staticmethod def display_hint(): return "string" -class StdOsStringProvider: +class StdOsStringProvider(printer_base): def __init__(self, valobj): - self.valobj = valobj - buf = self.valobj["inner"]["inner"] + self._valobj = valobj + buf = self._valobj["inner"]["inner"] is_windows = "Wtf8Buf" in buf.type.name vec = buf[ZERO_FIELD] if is_windows else buf - self.length = int(vec["len"]) - self.data_ptr = unwrap_unique_or_non_null(vec["buf"]["ptr"]) + self._length = int(vec["len"]) + self._data_ptr = unwrap_unique_or_non_null(vec["buf"]["ptr"]) def to_string(self): - return self.data_ptr.lazy_string(encoding="utf-8", length=self.length) + return self._data_ptr.lazy_string(encoding="utf-8", length=self._length) def display_hint(self): return "string" -class StdStrProvider: +class StdStrProvider(printer_base): def __init__(self, valobj): - self.valobj = valobj - self.length = int(valobj["length"]) - self.data_ptr = valobj["data_ptr"] + self._valobj = valobj + self._length = int(valobj["length"]) + self._data_ptr = valobj["data_ptr"] def to_string(self): - return self.data_ptr.lazy_string(encoding="utf-8", length=self.length) + return self._data_ptr.lazy_string(encoding="utf-8", length=self._length) @staticmethod def display_hint(): @@ -103,36 +112,36 @@ def _enumerate_array_elements(element_ptrs): yield key, element -class StdSliceProvider: +class StdSliceProvider(printer_base): def __init__(self, valobj): - self.valobj = valobj - self.length = int(valobj["length"]) - self.data_ptr = valobj["data_ptr"] + self._valobj = valobj + self._length = int(valobj["length"]) + self._data_ptr = valobj["data_ptr"] def to_string(self): - return "{}(size={})".format(self.valobj.type, self.length) + return "{}(size={})".format(self._valobj.type, self._length) def children(self): return _enumerate_array_elements( - self.data_ptr + index for index in xrange(self.length) + self._data_ptr + index for index in xrange(self._length) ) @staticmethod def display_hint(): return "array" -class StdVecProvider: +class StdVecProvider(printer_base): def __init__(self, valobj): - self.valobj = valobj - self.length = int(valobj["len"]) - self.data_ptr = unwrap_unique_or_non_null(valobj["buf"]["ptr"]) + self._valobj = valobj + self._length = int(valobj["len"]) + self._data_ptr = unwrap_unique_or_non_null(valobj["buf"]["ptr"]) def to_string(self): - return "Vec(size={})".format(self.length) + return "Vec(size={})".format(self._length) def children(self): return _enumerate_array_elements( - self.data_ptr + index for index in xrange(self.length) + self._data_ptr + index for index in xrange(self._length) ) @staticmethod @@ -140,20 +149,20 @@ class StdVecProvider: return "array" -class StdVecDequeProvider: +class StdVecDequeProvider(printer_base): def __init__(self, valobj): - self.valobj = valobj - self.head = int(valobj["head"]) - self.size = int(valobj["len"]) - self.cap = int(valobj["buf"]["cap"]) - self.data_ptr = unwrap_unique_or_non_null(valobj["buf"]["ptr"]) + self._valobj = valobj + self._head = int(valobj["head"]) + self._size = int(valobj["len"]) + self._cap = int(valobj["buf"]["cap"]) + self._data_ptr = unwrap_unique_or_non_null(valobj["buf"]["ptr"]) def to_string(self): - return "VecDeque(size={})".format(self.size) + return "VecDeque(size={})".format(self._size) def children(self): return _enumerate_array_elements( - (self.data_ptr + ((self.head + index) % self.cap)) for index in xrange(self.size) + (self._data_ptr + ((self._head + index) % self._cap)) for index in xrange(self._size) ) @staticmethod @@ -161,81 +170,81 @@ class StdVecDequeProvider: return "array" -class StdRcProvider: +class StdRcProvider(printer_base): def __init__(self, valobj, is_atomic=False): - self.valobj = valobj - self.is_atomic = is_atomic - self.ptr = unwrap_unique_or_non_null(valobj["ptr"]) - self.value = self.ptr["data" if is_atomic else "value"] - self.strong = self.ptr["strong"]["v" if is_atomic else "value"]["value"] - self.weak = self.ptr["weak"]["v" if is_atomic else "value"]["value"] - 1 + self._valobj = valobj + self._is_atomic = is_atomic + self._ptr = unwrap_unique_or_non_null(valobj["ptr"]) + self._value = self._ptr["data" if is_atomic else "value"] + self._strong = self._ptr["strong"]["v" if is_atomic else "value"]["value"] + self._weak = self._ptr["weak"]["v" if is_atomic else "value"]["value"] - 1 def to_string(self): - if self.is_atomic: - return "Arc(strong={}, weak={})".format(int(self.strong), int(self.weak)) + if self._is_atomic: + return "Arc(strong={}, weak={})".format(int(self._strong), int(self._weak)) else: - return "Rc(strong={}, weak={})".format(int(self.strong), int(self.weak)) + return "Rc(strong={}, weak={})".format(int(self._strong), int(self._weak)) def children(self): - yield "value", self.value - yield "strong", self.strong - yield "weak", self.weak + yield "value", self._value + yield "strong", self._strong + yield "weak", self._weak -class StdCellProvider: +class StdCellProvider(printer_base): def __init__(self, valobj): - self.value = valobj["value"]["value"] + self._value = valobj["value"]["value"] def to_string(self): return "Cell" def children(self): - yield "value", self.value + yield "value", self._value -class StdRefProvider: +class StdRefProvider(printer_base): def __init__(self, valobj): - self.value = valobj["value"].dereference() - self.borrow = valobj["borrow"]["borrow"]["value"]["value"] + self._value = valobj["value"].dereference() + self._borrow = valobj["borrow"]["borrow"]["value"]["value"] def to_string(self): - borrow = int(self.borrow) + borrow = int(self._borrow) if borrow >= 0: return "Ref(borrow={})".format(borrow) else: return "Ref(borrow_mut={})".format(-borrow) def children(self): - yield "*value", self.value - yield "borrow", self.borrow + yield "*value", self._value + yield "borrow", self._borrow -class StdRefCellProvider: +class StdRefCellProvider(printer_base): def __init__(self, valobj): - self.value = valobj["value"]["value"] - self.borrow = valobj["borrow"]["value"]["value"] + self._value = valobj["value"]["value"] + self._borrow = valobj["borrow"]["value"]["value"] def to_string(self): - borrow = int(self.borrow) + borrow = int(self._borrow) if borrow >= 0: return "RefCell(borrow={})".format(borrow) else: return "RefCell(borrow_mut={})".format(-borrow) def children(self): - yield "value", self.value - yield "borrow", self.borrow + yield "value", self._value + yield "borrow", self._borrow -class StdNonZeroNumberProvider: +class StdNonZeroNumberProvider(printer_base): def __init__(self, valobj): fields = valobj.type.fields() assert len(fields) == 1 field = list(fields)[0] - self.value = str(valobj[field.name]) + self._value = str(valobj[field.name]) def to_string(self): - return self.value + return self._value # Yields children (in a provider's sense of the word) for a BTreeMap. @@ -280,15 +289,15 @@ def children_of_btree_map(map): yield child -class StdBTreeSetProvider: +class StdBTreeSetProvider(printer_base): def __init__(self, valobj): - self.valobj = valobj + self._valobj = valobj def to_string(self): - return "BTreeSet(size={})".format(self.valobj["map"]["length"]) + return "BTreeSet(size={})".format(self._valobj["map"]["length"]) def children(self): - inner_map = self.valobj["map"] + inner_map = self._valobj["map"] for i, (child, _) in enumerate(children_of_btree_map(inner_map)): yield "[{}]".format(i), child @@ -297,15 +306,15 @@ class StdBTreeSetProvider: return "array" -class StdBTreeMapProvider: +class StdBTreeMapProvider(printer_base): def __init__(self, valobj): - self.valobj = valobj + self._valobj = valobj def to_string(self): - return "BTreeMap(size={})".format(self.valobj["length"]) + return "BTreeMap(size={})".format(self._valobj["length"]) def children(self): - for i, (key, val) in enumerate(children_of_btree_map(self.valobj)): + for i, (key, val) in enumerate(children_of_btree_map(self._valobj)): yield "key{}".format(i), key yield "val{}".format(i), val @@ -315,124 +324,124 @@ class StdBTreeMapProvider: # BACKCOMPAT: rust 1.35 -class StdOldHashMapProvider: +class StdOldHashMapProvider(printer_base): def __init__(self, valobj, show_values=True): - self.valobj = valobj - self.show_values = show_values - - self.table = self.valobj["table"] - self.size = int(self.table["size"]) - self.hashes = self.table["hashes"] - self.hash_uint_type = self.hashes.type - self.hash_uint_size = self.hashes.type.sizeof - self.modulo = 2 ** self.hash_uint_size - self.data_ptr = self.hashes[ZERO_FIELD]["pointer"] - - self.capacity_mask = int(self.table["capacity_mask"]) - self.capacity = (self.capacity_mask + 1) % self.modulo - - marker = self.table["marker"].type - self.pair_type = marker.template_argument(0) - self.pair_type_size = self.pair_type.sizeof - - self.valid_indices = [] - for idx in range(self.capacity): - data_ptr = self.data_ptr.cast(self.hash_uint_type.pointer()) + self._valobj = valobj + self._show_values = show_values + + self._table = self._valobj["table"] + self._size = int(self._table["size"]) + self._hashes = self._table["hashes"] + self._hash_uint_type = self._hashes.type + self._hash_uint_size = self._hashes.type.sizeof + self._modulo = 2 ** self._hash_uint_size + self._data_ptr = self._hashes[ZERO_FIELD]["pointer"] + + self._capacity_mask = int(self._table["capacity_mask"]) + self._capacity = (self._capacity_mask + 1) % self._modulo + + marker = self._table["marker"].type + self._pair_type = marker.template_argument(0) + self._pair_type_size = self._pair_type.sizeof + + self._valid_indices = [] + for idx in range(self._capacity): + data_ptr = self._data_ptr.cast(self._hash_uint_type.pointer()) address = data_ptr + idx hash_uint = address.dereference() hash_ptr = hash_uint[ZERO_FIELD]["pointer"] if int(hash_ptr) != 0: - self.valid_indices.append(idx) + self._valid_indices.append(idx) def to_string(self): - if self.show_values: - return "HashMap(size={})".format(self.size) + if self._show_values: + return "HashMap(size={})".format(self._size) else: - return "HashSet(size={})".format(self.size) + return "HashSet(size={})".format(self._size) def children(self): - start = int(self.data_ptr) & ~1 + start = int(self._data_ptr) & ~1 - hashes = self.hash_uint_size * self.capacity - align = self.pair_type_size - len_rounded_up = (((((hashes + align) % self.modulo - 1) % self.modulo) & ~( - (align - 1) % self.modulo)) % self.modulo - hashes) % self.modulo + hashes = self._hash_uint_size * self._capacity + align = self._pair_type_size + len_rounded_up = (((((hashes + align) % self._modulo - 1) % self._modulo) & ~( + (align - 1) % self._modulo)) % self._modulo - hashes) % self._modulo pairs_offset = hashes + len_rounded_up - pairs_start = gdb.Value(start + pairs_offset).cast(self.pair_type.pointer()) + pairs_start = gdb.Value(start + pairs_offset).cast(self._pair_type.pointer()) - for index in range(self.size): - table_index = self.valid_indices[index] - idx = table_index & self.capacity_mask + for index in range(self._size): + table_index = self._valid_indices[index] + idx = table_index & self._capacity_mask element = (pairs_start + idx).dereference() - if self.show_values: + if self._show_values: yield "key{}".format(index), element[ZERO_FIELD] yield "val{}".format(index), element[FIRST_FIELD] else: yield "[{}]".format(index), element[ZERO_FIELD] def display_hint(self): - return "map" if self.show_values else "array" + return "map" if self._show_values else "array" -class StdHashMapProvider: +class StdHashMapProvider(printer_base): def __init__(self, valobj, show_values=True): - self.valobj = valobj - self.show_values = show_values + self._valobj = valobj + self._show_values = show_values - table = self.table() + table = self._table() table_inner = table["table"] capacity = int(table_inner["bucket_mask"]) + 1 ctrl = table_inner["ctrl"]["pointer"] - self.size = int(table_inner["items"]) - self.pair_type = table.type.template_argument(0).strip_typedefs() + self._size = int(table_inner["items"]) + self._pair_type = table.type.template_argument(0).strip_typedefs() - self.new_layout = not table_inner.type.has_key("data") - if self.new_layout: - self.data_ptr = ctrl.cast(self.pair_type.pointer()) + self._new_layout = not table_inner.type.has_key("data") + if self._new_layout: + self._data_ptr = ctrl.cast(self._pair_type.pointer()) else: - self.data_ptr = table_inner["data"]["pointer"] + self._data_ptr = table_inner["data"]["pointer"] - self.valid_indices = [] + self._valid_indices = [] for idx in range(capacity): address = ctrl + idx value = address.dereference() is_presented = value & 128 == 0 if is_presented: - self.valid_indices.append(idx) + self._valid_indices.append(idx) - def table(self): - if self.show_values: - hashbrown_hashmap = self.valobj["base"] - elif self.valobj.type.fields()[0].name == "map": + def _table(self): + if self._show_values: + hashbrown_hashmap = self._valobj["base"] + elif self._valobj.type.fields()[0].name == "map": # BACKCOMPAT: rust 1.47 # HashSet wraps std::collections::HashMap, which wraps hashbrown::HashMap - hashbrown_hashmap = self.valobj["map"]["base"] + hashbrown_hashmap = self._valobj["map"]["base"] else: # HashSet wraps hashbrown::HashSet, which wraps hashbrown::HashMap - hashbrown_hashmap = self.valobj["base"]["map"] + hashbrown_hashmap = self._valobj["base"]["map"] return hashbrown_hashmap["table"] def to_string(self): - if self.show_values: - return "HashMap(size={})".format(self.size) + if self._show_values: + return "HashMap(size={})".format(self._size) else: - return "HashSet(size={})".format(self.size) + return "HashSet(size={})".format(self._size) def children(self): - pairs_start = self.data_ptr + pairs_start = self._data_ptr - for index in range(self.size): - idx = self.valid_indices[index] - if self.new_layout: + for index in range(self._size): + idx = self._valid_indices[index] + if self._new_layout: idx = -(idx + 1) element = (pairs_start + idx).dereference() - if self.show_values: + if self._show_values: yield "key{}".format(index), element[ZERO_FIELD] yield "val{}".format(index), element[FIRST_FIELD] else: yield "[{}]".format(index), element[ZERO_FIELD] def display_hint(self): - return "map" if self.show_values else "array" + return "map" if self._show_values else "array" diff --git a/src/etc/test-float-parse/Cargo.lock b/src/etc/test-float-parse/Cargo.lock new file mode 100644 index 000000000..3f60423fe --- /dev/null +++ b/src/etc/test-float-parse/Cargo.lock @@ -0,0 +1,75 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "getrandom" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4136b2a15dd319360be1c07d9933517ccf0be8f16bf62a3bee4f0d618df427" +dependencies = [ + "cfg-if", + "libc", + "wasi", +] + +[[package]] +name = "libc" +version = "0.2.147" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" + +[[package]] +name = "ppv-lite86" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" + +[[package]] +name = "rand" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" +dependencies = [ + "libc", + "rand_chacha", + "rand_core", +] + +[[package]] +name = "rand_chacha" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" +dependencies = [ + "getrandom", +] + +[[package]] +name = "test-float-parse" +version = "0.1.0" +dependencies = [ + "rand", +] + +[[package]] +name = "wasi" +version = "0.11.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" diff --git a/src/etc/test-float-parse/Cargo.toml b/src/etc/test-float-parse/Cargo.toml index 6d7b227d0..a045be956 100644 --- a/src/etc/test-float-parse/Cargo.toml +++ b/src/etc/test-float-parse/Cargo.toml @@ -8,7 +8,7 @@ publish = false resolver = "1" [dependencies] -rand = "0.4" +rand = "0.8" [lib] name = "test_float_parse" diff --git a/src/librustdoc/Cargo.toml b/src/librustdoc/Cargo.toml index 29912b957..f3917b978 100644 --- a/src/librustdoc/Cargo.toml +++ b/src/librustdoc/Cargo.toml @@ -10,7 +10,8 @@ path = "lib.rs" arrayvec = { version = "0.7", default-features = false } askama = { version = "0.12", default-features = false, features = ["config"] } itertools = "0.10.1" -minifier = "0.2.2" +indexmap = "2" +minifier = "0.3.0" once_cell = "1.10.0" regex = "1" rustdoc-json-types = { path = "../rustdoc-json-types" } diff --git a/src/librustdoc/clean/auto_trait.rs b/src/librustdoc/clean/auto_trait.rs index a06f31a93..bdf6a0f6b 100644 --- a/src/librustdoc/clean/auto_trait.rs +++ b/src/librustdoc/clean/auto_trait.rs @@ -551,8 +551,8 @@ where WherePredicate::RegionPredicate { lifetime, bounds } => { lifetime_to_bounds.entry(lifetime).or_default().extend(bounds); } - WherePredicate::EqPredicate { lhs, rhs, bound_params } => { - match *lhs { + WherePredicate::EqPredicate { lhs, rhs } => { + match lhs { Type::QPath(box QPathData { ref assoc, ref self_type, @@ -590,14 +590,13 @@ where GenericArgs::AngleBracketed { ref mut bindings, .. } => { bindings.push(TypeBinding { assoc: assoc.clone(), - kind: TypeBindingKind::Equality { term: *rhs }, + kind: TypeBindingKind::Equality { term: rhs }, }); } GenericArgs::Parenthesized { .. } => { existing_predicates.push(WherePredicate::EqPredicate { lhs: lhs.clone(), rhs, - bound_params, }); continue; // If something other than a Fn ends up // with parentheses, leave it alone diff --git a/src/librustdoc/clean/inline.rs b/src/librustdoc/clean/inline.rs index fcbcfbf5c..974ba1e3b 100644 --- a/src/librustdoc/clean/inline.rs +++ b/src/librustdoc/clean/inline.rs @@ -18,14 +18,16 @@ use rustc_span::hygiene::MacroKind; use rustc_span::symbol::{kw, sym, Symbol}; use crate::clean::{ - self, clean_fn_decl_from_did_and_sig, clean_generics, clean_impl_item, clean_middle_assoc_item, - clean_middle_field, clean_middle_ty, clean_trait_ref_with_bindings, clean_ty, - clean_ty_alias_inner_type, clean_ty_generics, clean_variant_def, utils, Attributes, + self, clean_bound_vars, clean_fn_decl_from_did_and_sig, clean_generics, clean_impl_item, + clean_middle_assoc_item, clean_middle_field, clean_middle_ty, clean_trait_ref_with_bindings, + clean_ty, clean_ty_alias_inner_type, clean_ty_generics, clean_variant_def, utils, Attributes, AttributesExt, ImplKind, ItemId, Type, }; use crate::core::DocContext; use crate::formats::item_type::ItemType; +use super::Item; + /// Attempt to inline a definition into this AST. /// /// This function will fetch the definition specified, and if it is @@ -83,7 +85,7 @@ pub(crate) fn try_inline( Res::Def(DefKind::TyAlias, did) => { record_extern_fqn(cx, did, ItemType::TypeAlias); build_impls(cx, did, attrs_without_docs, &mut ret); - clean::TypeAliasItem(build_type_alias(cx, did)) + clean::TypeAliasItem(build_type_alias(cx, did, &mut ret)) } Res::Def(DefKind::Enum, did) => { record_extern_fqn(cx, did, ItemType::Enum); @@ -239,20 +241,13 @@ pub(crate) fn build_external_trait(cx: &mut DocContext<'_>, did: DefId) -> clean fn build_external_function<'tcx>(cx: &mut DocContext<'tcx>, did: DefId) -> Box { let sig = cx.tcx.fn_sig(did).instantiate_identity(); - - let late_bound_regions = sig.bound_vars().into_iter().filter_map(|var| match var { - ty::BoundVariableKind::Region(ty::BrNamed(_, name)) if name != kw::UnderscoreLifetime => { - Some(clean::GenericParamDef::lifetime(name)) - } - _ => None, - }); - let predicates = cx.tcx.explicit_predicates_of(did); + let (generics, decl) = clean::enter_impl_trait(cx, |cx| { // NOTE: generics need to be cleaned before the decl! let mut generics = clean_ty_generics(cx, cx.tcx.generics_of(did), predicates); // FIXME: This does not place parameters in source order (late-bound ones come last) - generics.params.extend(late_bound_regions); + generics.params.extend(clean_bound_vars(sig.bound_vars())); let decl = clean_fn_decl_from_did_and_sig(cx, Some(did), sig); (generics, decl) }); @@ -288,11 +283,15 @@ fn build_union(cx: &mut DocContext<'_>, did: DefId) -> clean::Union { clean::Union { generics, fields } } -fn build_type_alias(cx: &mut DocContext<'_>, did: DefId) -> Box { +fn build_type_alias( + cx: &mut DocContext<'_>, + did: DefId, + ret: &mut Vec, +) -> Box { let predicates = cx.tcx.explicit_predicates_of(did); let ty = cx.tcx.type_of(did).instantiate_identity(); let type_ = clean_middle_ty(ty::Binder::dummy(ty), cx, Some(did), None); - let inner_type = clean_ty_alias_inner_type(ty, cx); + let inner_type = clean_ty_alias_inner_type(ty, cx, ret); Box::new(clean::TypeAlias { type_, @@ -600,7 +599,7 @@ fn build_module_items( let prim_ty = clean::PrimitiveType::from(p); items.push(clean::Item { name: None, - attrs: Box::new(clean::Attributes::default()), + attrs: Box::default(), // We can use the item's `DefId` directly since the only information ever used // from it is `DefId.krate`. item_id: ItemId::DefId(did), @@ -648,13 +647,13 @@ fn build_const(cx: &mut DocContext<'_>, def_id: DefId) -> clean::Constant { clean::simplify::move_bounds_to_generic_parameters(&mut generics); clean::Constant { - type_: clean_middle_ty( + type_: Box::new(clean_middle_ty( ty::Binder::dummy(cx.tcx.type_of(def_id).instantiate_identity()), cx, Some(def_id), None, - ), - generics: Box::new(generics), + )), + generics, kind: clean::ConstantKind::Extern { def_id }, } } diff --git a/src/librustdoc/clean/mod.rs b/src/librustdoc/clean/mod.rs index 0caa92e44..1b7ca7bf7 100644 --- a/src/librustdoc/clean/mod.rs +++ b/src/librustdoc/clean/mod.rs @@ -232,20 +232,11 @@ fn clean_poly_trait_ref_with_bindings<'tcx>( poly_trait_ref: ty::PolyTraitRef<'tcx>, bindings: ThinVec, ) -> GenericBound { - // collect any late bound regions - let late_bound_regions: Vec<_> = cx - .tcx - .collect_referenced_late_bound_regions(&poly_trait_ref) - .into_iter() - .filter_map(|br| match br { - ty::BrNamed(_, name) if br.is_named() => Some(GenericParamDef::lifetime(name)), - _ => None, - }) - .collect(); - - let trait_ = clean_trait_ref_with_bindings(cx, poly_trait_ref, bindings); GenericBound::TraitBound( - PolyTrait { trait_, generic_params: late_bound_regions }, + PolyTrait { + trait_: clean_trait_ref_with_bindings(cx, poly_trait_ref, bindings), + generic_params: clean_bound_vars(poly_trait_ref.bound_vars()), + }, hir::TraitBoundModifier::None, ) } @@ -268,13 +259,13 @@ fn clean_lifetime<'tcx>(lifetime: &hir::Lifetime, cx: &mut DocContext<'tcx>) -> pub(crate) fn clean_const<'tcx>(constant: &hir::ConstArg, cx: &mut DocContext<'tcx>) -> Constant { let def_id = cx.tcx.hir().body_owner_def_id(constant.value.body).to_def_id(); Constant { - type_: clean_middle_ty( + type_: Box::new(clean_middle_ty( ty::Binder::dummy(cx.tcx.type_of(def_id).instantiate_identity()), cx, Some(def_id), None, - ), - generics: Box::new(Generics::default()), + )), + generics: Generics::default(), kind: ConstantKind::Anonymous { body: constant.value.body }, } } @@ -285,8 +276,8 @@ pub(crate) fn clean_middle_const<'tcx>( ) -> Constant { // FIXME: instead of storing the stringified expression, store `self` directly instead. Constant { - type_: clean_middle_ty(constant.map_bound(|c| c.ty()), cx, None, None), - generics: Box::new(Generics::default()), + type_: Box::new(clean_middle_ty(constant.map_bound(|c| c.ty()), cx, None, None)), + generics: Generics::default(), kind: ConstantKind::TyConst { expr: constant.skip_binder().to_string().into() }, } } @@ -338,9 +329,8 @@ fn clean_where_predicate<'tcx>( }, hir::WherePredicate::EqPredicate(ref wrp) => WherePredicate::EqPredicate { - lhs: Box::new(clean_ty(wrp.lhs_ty, cx)), - rhs: Box::new(clean_ty(wrp.rhs_ty, cx).into()), - bound_params: Vec::new(), + lhs: clean_ty(wrp.lhs_ty, cx), + rhs: clean_ty(wrp.rhs_ty, cx).into(), }, }) } @@ -436,20 +426,9 @@ fn clean_projection_predicate<'tcx>( pred: ty::Binder<'tcx, ty::ProjectionPredicate<'tcx>>, cx: &mut DocContext<'tcx>, ) -> WherePredicate { - let late_bound_regions = cx - .tcx - .collect_referenced_late_bound_regions(&pred) - .into_iter() - .filter_map(|br| match br { - ty::BrNamed(_, name) if br.is_named() => Some(GenericParamDef::lifetime(name)), - _ => None, - }) - .collect(); - WherePredicate::EqPredicate { - lhs: Box::new(clean_projection(pred.map_bound(|p| p.projection_ty), cx, None)), - rhs: Box::new(clean_middle_term(pred.map_bound(|p| p.term), cx)), - bound_params: late_bound_regions, + lhs: clean_projection(pred.map_bound(|p| p.projection_ty), cx, None), + rhs: clean_middle_term(pred.map_bound(|p| p.term), cx), } } @@ -496,8 +475,9 @@ fn projection_to_path_segment<'tcx>( ty: ty::Binder<'tcx, ty::AliasTy<'tcx>>, cx: &mut DocContext<'tcx>, ) -> PathSegment { - let item = cx.tcx.associated_item(ty.skip_binder().def_id); - let generics = cx.tcx.generics_of(ty.skip_binder().def_id); + let def_id = ty.skip_binder().def_id; + let item = cx.tcx.associated_item(def_id); + let generics = cx.tcx.generics_of(def_id); PathSegment { name: item.name, args: GenericArgs::AngleBracketed { @@ -505,7 +485,7 @@ fn projection_to_path_segment<'tcx>( cx, ty.map_bound(|ty| &ty.args[generics.parent_count..]), false, - None, + def_id, ) .into(), bindings: Default::default(), @@ -519,7 +499,7 @@ fn clean_generic_param_def<'tcx>( ) -> GenericParamDef { let (name, kind) = match def.kind { ty::GenericParamDefKind::Lifetime => { - (def.name, GenericParamDefKind::Lifetime { outlives: vec![] }) + (def.name, GenericParamDefKind::Lifetime { outlives: ThinVec::new() }) } ty::GenericParamDefKind::Type { has_default, synthetic, .. } => { let default = if has_default { @@ -536,7 +516,7 @@ fn clean_generic_param_def<'tcx>( def.name, GenericParamDefKind::Type { did: def.def_id, - bounds: vec![], // These are filled in from the where-clauses. + bounds: ThinVec::new(), // These are filled in from the where-clauses. default: default.map(Box::new), synthetic, }, @@ -588,7 +568,7 @@ fn clean_generic_param<'tcx>( }) .collect() } else { - Vec::new() + ThinVec::new() }; (param.name.ident().name, GenericParamDefKind::Lifetime { outlives }) } @@ -601,7 +581,7 @@ fn clean_generic_param<'tcx>( .filter_map(|x| clean_generic_bound(x, cx)) .collect() } else { - Vec::new() + ThinVec::new() }; ( param.name.ident().name, @@ -657,7 +637,7 @@ pub(crate) fn clean_generics<'tcx>( match param.kind { GenericParamDefKind::Lifetime { .. } => unreachable!(), GenericParamDefKind::Type { did, ref bounds, .. } => { - cx.impl_trait_bounds.insert(did.into(), bounds.clone()); + cx.impl_trait_bounds.insert(did.into(), bounds.to_vec()); } GenericParamDefKind::Const { .. } => unreachable!(), } @@ -705,8 +685,8 @@ pub(crate) fn clean_generics<'tcx>( } } } - WherePredicate::EqPredicate { lhs, rhs, bound_params } => { - eq_predicates.push(WherePredicate::EqPredicate { lhs, rhs, bound_params }); + WherePredicate::EqPredicate { lhs, rhs } => { + eq_predicates.push(WherePredicate::EqPredicate { lhs, rhs }); } } } @@ -800,11 +780,9 @@ fn clean_ty_generics<'tcx>( }) .collect::>(); - // param index -> [(trait DefId, associated type name & generics, term, higher-ranked params)] - let mut impl_trait_proj = FxHashMap::< - u32, - Vec<(DefId, PathSegment, ty::Binder<'_, ty::Term<'_>>, Vec)>, - >::default(); + // param index -> [(trait DefId, associated type name & generics, term)] + let mut impl_trait_proj = + FxHashMap::>)>>::default(); let where_predicates = preds .predicates @@ -856,11 +834,6 @@ fn clean_ty_generics<'tcx>( trait_did, name, proj.map_bound(|p| p.term), - pred.get_bound_params() - .into_iter() - .flatten() - .cloned() - .collect(), )); } @@ -896,9 +869,9 @@ fn clean_ty_generics<'tcx>( let crate::core::ImplTraitParam::ParamIndex(idx) = param else { unreachable!() }; if let Some(proj) = impl_trait_proj.remove(&idx) { - for (trait_did, name, rhs, bound_params) in proj { + for (trait_did, name, rhs) in proj { let rhs = clean_middle_term(rhs, cx); - simplify::merge_bounds(cx, &mut bounds, bound_params, trait_did, name, &rhs); + simplify::merge_bounds(cx, &mut bounds, trait_did, name, &rhs); } } @@ -962,11 +935,16 @@ fn clean_ty_generics<'tcx>( fn clean_ty_alias_inner_type<'tcx>( ty: Ty<'tcx>, cx: &mut DocContext<'tcx>, + ret: &mut Vec, ) -> Option { let ty::Adt(adt_def, args) = ty.kind() else { return None; }; + if !adt_def.did().is_local() { + inline::build_impls(cx, adt_def.did(), None, ret); + } + Some(if adt_def.is_enum() { let variants: rustc_index::IndexVec<_, _> = adt_def .variants() @@ -974,6 +952,10 @@ fn clean_ty_alias_inner_type<'tcx>( .map(|variant| clean_variant_def_with_args(variant, args, cx)) .collect(); + if !adt_def.did().is_local() { + inline::record_extern_fqn(cx, adt_def.did(), ItemType::Enum); + } + TypeAliasInnerType::Enum { variants, is_non_exhaustive: adt_def.is_variant_list_non_exhaustive(), @@ -989,8 +971,14 @@ fn clean_ty_alias_inner_type<'tcx>( clean_variant_def_with_args(variant, args, cx).kind.inner_items().cloned().collect(); if adt_def.is_struct() { + if !adt_def.did().is_local() { + inline::record_extern_fqn(cx, adt_def.did(), ItemType::Struct); + } TypeAliasInnerType::Struct { ctor_kind: variant.ctor_kind(), fields } } else { + if !adt_def.did().is_local() { + inline::record_extern_fqn(cx, adt_def.did(), ItemType::Union); + } TypeAliasInnerType::Union { fields } } }) @@ -1244,14 +1232,14 @@ fn clean_trait_item<'tcx>(trait_item: &hir::TraitItem<'tcx>, cx: &mut DocContext hir::TraitItemKind::Const(ty, Some(default)) => { let generics = enter_impl_trait(cx, |cx| clean_generics(trait_item.generics, cx)); AssocConstItem( - Box::new(generics), - clean_ty(ty, cx), + generics, + Box::new(clean_ty(ty, cx)), ConstantKind::Local { def_id: local_did, body: default }, ) } hir::TraitItemKind::Const(ty, None) => { let generics = enter_impl_trait(cx, |cx| clean_generics(trait_item.generics, cx)); - TyAssocConstItem(Box::new(generics), clean_ty(ty, cx)) + TyAssocConstItem(generics, Box::new(clean_ty(ty, cx))) } hir::TraitItemKind::Fn(ref sig, hir::TraitFn::Provided(body)) => { let m = clean_function(cx, sig, trait_item.generics, FunctionArgs::Body(body)); @@ -1300,7 +1288,7 @@ pub(crate) fn clean_impl_item<'tcx>( hir::ImplItemKind::Const(ty, expr) => { let generics = clean_generics(impl_.generics, cx); let default = ConstantKind::Local { def_id: local_did, body: expr }; - AssocConstItem(Box::new(generics), clean_ty(ty, cx), default) + AssocConstItem(generics, Box::new(clean_ty(ty, cx)), default) } hir::ImplItemKind::Fn(ref sig, body) => { let m = clean_function(cx, sig, impl_.generics, FunctionArgs::Body(body)); @@ -1339,18 +1327,18 @@ pub(crate) fn clean_middle_assoc_item<'tcx>( let tcx = cx.tcx; let kind = match assoc_item.kind { ty::AssocKind::Const => { - let ty = clean_middle_ty( + let ty = Box::new(clean_middle_ty( ty::Binder::dummy(tcx.type_of(assoc_item.def_id).instantiate_identity()), cx, Some(assoc_item.def_id), None, - ); + )); - let mut generics = Box::new(clean_ty_generics( + let mut generics = clean_ty_generics( cx, tcx.generics_of(assoc_item.def_id), tcx.explicit_predicates_of(assoc_item.def_id), - )); + ); simplify::move_bounds_to_generic_parameters(&mut generics); let provided = match assoc_item.container { @@ -1365,23 +1353,13 @@ pub(crate) fn clean_middle_assoc_item<'tcx>( } ty::AssocKind::Fn => { let sig = tcx.fn_sig(assoc_item.def_id).instantiate_identity(); - - let late_bound_regions = sig.bound_vars().into_iter().filter_map(|var| match var { - ty::BoundVariableKind::Region(ty::BrNamed(_, name)) - if name != kw::UnderscoreLifetime => - { - Some(GenericParamDef::lifetime(name)) - } - _ => None, - }); - let mut generics = clean_ty_generics( cx, tcx.generics_of(assoc_item.def_id), tcx.explicit_predicates_of(assoc_item.def_id), ); // FIXME: This does not place parameters in source order (late-bound ones come last) - generics.params.extend(late_bound_regions); + generics.params.extend(clean_bound_vars(sig.bound_vars())); let mut decl = clean_fn_decl_from_did_and_sig(cx, Some(assoc_item.def_id), sig); @@ -2117,9 +2095,11 @@ pub(crate) fn clean_middle_ty<'tcx>( // FIXME: should we merge the outer and inner binders somehow? let sig = bound_ty.skip_binder().fn_sig(cx.tcx); let decl = clean_fn_decl_from_did_and_sig(cx, None, sig); + let generic_params = clean_bound_vars(sig.bound_vars()); + BareFunction(Box::new(BareFunctionDecl { unsafety: sig.unsafety(), - generic_params: Vec::new(), + generic_params, decl, abi: sig.abi(), })) @@ -2195,8 +2175,8 @@ pub(crate) fn clean_middle_ty<'tcx>( let late_bound_regions: FxIndexSet<_> = obj .iter() - .flat_map(|pb| pb.bound_vars()) - .filter_map(|br| match br { + .flat_map(|pred| pred.bound_vars()) + .filter_map(|var| match var { ty::BoundVariableKind::Region(ty::BrNamed(_, name)) if name != kw::UnderscoreLifetime => { @@ -2221,18 +2201,19 @@ pub(crate) fn clean_middle_ty<'tcx>( } ty::Alias(ty::Inherent, alias_ty) => { + let def_id = alias_ty.def_id; let alias_ty = bound_ty.rebind(alias_ty); let self_type = clean_middle_ty(alias_ty.map_bound(|ty| ty.self_ty()), cx, None, None); Type::QPath(Box::new(QPathData { assoc: PathSegment { - name: cx.tcx.associated_item(alias_ty.skip_binder().def_id).name, + name: cx.tcx.associated_item(def_id).name, args: GenericArgs::AngleBracketed { args: ty_args_to_args( cx, alias_ty.map_bound(|ty| ty.args.as_slice()), true, - None, + def_id, ) .into(), bindings: Default::default(), @@ -2270,6 +2251,11 @@ pub(crate) fn clean_middle_ty<'tcx>( } } + ty::Bound(_, ref ty) => match ty.kind { + ty::BoundTyKind::Param(_, name) => Generic(name), + ty::BoundTyKind::Anon => panic!("unexpected anonymous bound type variable"), + }, + ty::Alias(ty::Opaque, ty::AliasTy { def_id, args, .. }) => { // If it's already in the same alias, don't get an infinite loop. if cx.current_type_aliases.contains_key(&def_id) { @@ -2297,10 +2283,9 @@ pub(crate) fn clean_middle_ty<'tcx>( } ty::Closure(..) => panic!("Closure"), - ty::Generator(..) => panic!("Generator"), - ty::Bound(..) => panic!("Bound"), + ty::Coroutine(..) => panic!("Coroutine"), ty::Placeholder(..) => panic!("Placeholder"), - ty::GeneratorWitness(..) => panic!("GeneratorWitness"), + ty::CoroutineWitness(..) => panic!("CoroutineWitness"), ty::Infer(..) => panic!("Infer"), ty::Error(_) => rustc_errors::FatalError.raise(), } @@ -2549,7 +2534,8 @@ fn clean_generic_args<'tcx>( } hir::GenericArg::Lifetime(_) => GenericArg::Lifetime(Lifetime::elided()), hir::GenericArg::Type(ty) => GenericArg::Type(clean_ty(ty, cx)), - // FIXME(effects): This will still emit `` for non-const impls of const traits + // Checking for `#[rustc_host]` on the `AnonConst` not only accounts for the case + // where the argument is `host` but for all possible cases (e.g., `true`, `false`). hir::GenericArg::Const(ct) if cx.tcx.has_attr(ct.value.def_id, sym::rustc_host) => { @@ -2750,8 +2736,8 @@ fn clean_maybe_renamed_item<'tcx>( StaticItem(Static { type_: clean_ty(ty, cx), mutability, expr: Some(body_id) }) } ItemKind::Const(ty, generics, body_id) => ConstantItem(Constant { - type_: clean_ty(ty, cx), - generics: Box::new(clean_generics(generics, cx)), + type_: Box::new(clean_ty(ty, cx)), + generics: clean_generics(generics, cx), kind: ConstantKind::Local { body: body_id, def_id }, }), ItemKind::OpaqueTy(ref ty) => OpaqueTyItem(OpaqueTy { @@ -2776,14 +2762,24 @@ fn clean_maybe_renamed_item<'tcx>( } let ty = cx.tcx.type_of(def_id).instantiate_identity(); - let inner_type = clean_ty_alias_inner_type(ty, cx); - TypeAliasItem(Box::new(TypeAlias { - generics, - inner_type, - type_: rustdoc_ty, - item_type: Some(type_), - })) + let mut ret = Vec::new(); + let inner_type = clean_ty_alias_inner_type(ty, cx, &mut ret); + + ret.push(generate_item_with_correct_attrs( + cx, + TypeAliasItem(Box::new(TypeAlias { + generics, + inner_type, + type_: rustdoc_ty, + item_type: Some(type_), + })), + item.owner_id.def_id.to_def_id(), + name, + import_id, + renamed, + )); + return ret; } ItemKind::Enum(ref def, generics) => EnumItem(Enum { variants: def.variants.iter().map(|v| clean_variant(v, cx)).collect(), @@ -3137,3 +3133,30 @@ fn clean_type_binding<'tcx>( }, } } + +fn clean_bound_vars<'tcx>( + bound_vars: &'tcx ty::List, +) -> Vec { + bound_vars + .into_iter() + .filter_map(|var| match var { + ty::BoundVariableKind::Region(ty::BrNamed(_, name)) + if name != kw::UnderscoreLifetime => + { + Some(GenericParamDef::lifetime(name)) + } + ty::BoundVariableKind::Ty(ty::BoundTyKind::Param(did, name)) => Some(GenericParamDef { + name, + kind: GenericParamDefKind::Type { + did, + bounds: ThinVec::new(), + default: None, + synthetic: false, + }, + }), + // FIXME(non_lifetime_binders): Support higher-ranked const parameters. + ty::BoundVariableKind::Const => None, + _ => None, + }) + .collect() +} diff --git a/src/librustdoc/clean/simplify.rs b/src/librustdoc/clean/simplify.rs index 7b8f20326..627f15e67 100644 --- a/src/librustdoc/clean/simplify.rs +++ b/src/librustdoc/clean/simplify.rs @@ -40,18 +40,18 @@ pub(crate) fn where_clauses(cx: &DocContext<'_>, clauses: Vec) -> ThinVec { lifetimes.push((lifetime, bounds)); } - WP::EqPredicate { lhs, rhs, bound_params } => equalities.push((lhs, rhs, bound_params)), + WP::EqPredicate { lhs, rhs } => equalities.push((lhs, rhs)), } } // Look for equality predicates on associated types that can be merged into // general bound predicates. - equalities.retain(|(lhs, rhs, bound_params)| { + equalities.retain(|(lhs, rhs)| { let Some((ty, trait_did, name)) = lhs.projection() else { return true; }; let Some((bounds, _)) = tybounds.get_mut(ty) else { return true }; - merge_bounds(cx, bounds, bound_params.clone(), trait_did, name, rhs) + merge_bounds(cx, bounds, trait_did, name, rhs) }); // And finally, let's reassemble everything @@ -64,18 +64,13 @@ pub(crate) fn where_clauses(cx: &DocContext<'_>, clauses: Vec) -> ThinVec, bounds: &mut Vec, - mut bound_params: Vec, trait_did: DefId, assoc: clean::PathSegment, rhs: &clean::Term, @@ -93,12 +88,6 @@ pub(crate) fn merge_bounds( } let last = trait_ref.trait_.segments.last_mut().expect("segments were empty"); - trait_ref.generic_params.append(&mut bound_params); - // Sort parameters (likely) originating from a hashset alphabetically to - // produce predictable output (and to allow for full deduplication). - trait_ref.generic_params.sort_unstable_by(|p, q| p.name.as_str().cmp(q.name.as_str())); - trait_ref.generic_params.dedup_by_key(|p| p.name); - match last.args { PP::AngleBracketed { ref mut bindings, .. } => { bindings.push(clean::TypeBinding { @@ -156,7 +145,7 @@ pub(crate) fn move_bounds_to_generic_parameters(generics: &mut clean::Generics) .. }) = generics.params.iter_mut().find(|param| ¶m.name == arg) { - param_bounds.append(bounds); + param_bounds.extend(bounds.drain(..)); } else if let WherePredicate::RegionPredicate { lifetime: Lifetime(arg), bounds } = &mut pred && let Some(GenericParamDef { kind: GenericParamDefKind::Lifetime { outlives: param_bounds }, diff --git a/src/librustdoc/clean/types.rs b/src/librustdoc/clean/types.rs index 4256e7b51..88ee4e3a2 100644 --- a/src/librustdoc/clean/types.rs +++ b/src/librustdoc/clean/types.rs @@ -12,7 +12,7 @@ use thin_vec::ThinVec; use rustc_ast as ast; use rustc_ast_pretty::pprust; -use rustc_attr::{ConstStability, Deprecation, Stability, StabilityLevel}; +use rustc_attr::{ConstStability, Deprecation, Stability, StabilityLevel, StableSince}; use rustc_const_eval::const_eval::is_unstable_const_fn; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir as hir; @@ -585,14 +585,14 @@ impl Item { }) } - pub(crate) fn stable_since(&self, tcx: TyCtxt<'_>) -> Option { + pub(crate) fn stable_since(&self, tcx: TyCtxt<'_>) -> Option { match self.stability(tcx)?.level { StabilityLevel::Stable { since, .. } => Some(since), StabilityLevel::Unstable { .. } => None, } } - pub(crate) fn const_stable_since(&self, tcx: TyCtxt<'_>) -> Option { + pub(crate) fn const_stable_since(&self, tcx: TyCtxt<'_>) -> Option { match self.const_stability(tcx)?.level { StabilityLevel::Stable { since, .. } => Some(since), StabilityLevel::Unstable { .. } => None, @@ -713,12 +713,16 @@ impl Item { Some(tcx.visibility(def_id)) } - pub(crate) fn attributes(&self, tcx: TyCtxt<'_>, keep_as_is: bool) -> Vec { + pub(crate) fn attributes( + &self, + tcx: TyCtxt<'_>, + cache: &Cache, + keep_as_is: bool, + ) -> Vec { const ALLOWED_ATTRIBUTES: &[Symbol] = - &[sym::export_name, sym::link_section, sym::no_mangle, sym::repr, sym::non_exhaustive]; + &[sym::export_name, sym::link_section, sym::no_mangle, sym::non_exhaustive]; use rustc_abi::IntegerType; - use rustc_middle::ty::ReprFlags; let mut attrs: Vec = self .attrs @@ -739,20 +743,38 @@ impl Item { } }) .collect(); - if let Some(def_id) = self.def_id() && - !def_id.is_local() && - // This check is needed because `adt_def` will panic if not a compatible type otherwise... - matches!(self.type_(), ItemType::Struct | ItemType::Enum | ItemType::Union) + if !keep_as_is + && let Some(def_id) = self.def_id() + && let ItemType::Struct | ItemType::Enum | ItemType::Union = self.type_() { - let repr = tcx.adt_def(def_id).repr(); + let adt = tcx.adt_def(def_id); + let repr = adt.repr(); let mut out = Vec::new(); - if repr.flags.contains(ReprFlags::IS_C) { + if repr.c() { out.push("C"); } - if repr.flags.contains(ReprFlags::IS_TRANSPARENT) { - out.push("transparent"); + if repr.transparent() { + // Render `repr(transparent)` iff the non-1-ZST field is public or at least one + // field is public in case all fields are 1-ZST fields. + let render_transparent = cache.document_private + || adt + .all_fields() + .find(|field| { + let ty = + field.ty(tcx, ty::GenericArgs::identity_for_item(tcx, field.did)); + tcx.layout_of(tcx.param_env(field.did).and(ty)) + .is_ok_and(|layout| !layout.is_1zst()) + }) + .map_or_else( + || adt.all_fields().any(|field| field.vis.is_public()), + |field| field.vis.is_public(), + ); + + if render_transparent { + out.push("transparent"); + } } - if repr.flags.contains(ReprFlags::IS_SIMD) { + if repr.simd() { out.push("simd"); } let pack_s; @@ -777,10 +799,9 @@ impl Item { }; out.push(&int_s); } - if out.is_empty() { - return Vec::new(); + if !out.is_empty() { + attrs.push(format!("#[repr({})]", out.join(", "))); } - attrs.push(format!("#[repr({})]", out.join(", "))); } attrs } @@ -831,9 +852,9 @@ pub(crate) enum ItemKind { ProcMacroItem(ProcMacro), PrimitiveItem(PrimitiveType), /// A required associated constant in a trait declaration. - TyAssocConstItem(Box, Type), + TyAssocConstItem(Generics, Box), /// An associated constant in a trait impl or a provided one in a trait declaration. - AssocConstItem(Box, Type, ConstantKind), + AssocConstItem(Generics, Box, ConstantKind), /// A required associated type in a trait declaration. /// /// The bounds may be non-empty if there is a `where` clause. @@ -1289,7 +1310,7 @@ impl Lifetime { pub(crate) enum WherePredicate { BoundPredicate { ty: Type, bounds: Vec, bound_params: Vec }, RegionPredicate { lifetime: Lifetime, bounds: Vec }, - EqPredicate { lhs: Box, rhs: Box, bound_params: Vec }, + EqPredicate { lhs: Type, rhs: Term }, } impl WherePredicate { @@ -1300,21 +1321,13 @@ impl WherePredicate { _ => None, } } - - pub(crate) fn get_bound_params(&self) -> Option<&[GenericParamDef]> { - match self { - Self::BoundPredicate { bound_params, .. } | Self::EqPredicate { bound_params, .. } => { - Some(bound_params) - } - _ => None, - } - } } #[derive(Clone, PartialEq, Eq, Debug, Hash)] pub(crate) enum GenericParamDefKind { - Lifetime { outlives: Vec }, - Type { did: DefId, bounds: Vec, default: Option>, synthetic: bool }, + Lifetime { outlives: ThinVec }, + Type { did: DefId, bounds: ThinVec, default: Option>, synthetic: bool }, + // Option> makes this type smaller than `Option` would. Const { ty: Box, default: Option>, is_host_effect: bool }, } @@ -1332,7 +1345,7 @@ pub(crate) struct GenericParamDef { impl GenericParamDef { pub(crate) fn lifetime(name: Symbol) -> Self { - Self { name, kind: GenericParamDefKind::Lifetime { outlives: Vec::new() } } + Self { name, kind: GenericParamDefKind::Lifetime { outlives: ThinVec::new() } } } pub(crate) fn is_synthetic_param(&self) -> bool { @@ -1443,6 +1456,9 @@ impl Trait { pub(crate) fn unsafety(&self, tcx: TyCtxt<'_>) -> hir::Unsafety { tcx.trait_def(self.def_id).unsafety } + pub(crate) fn is_object_safe(&self, tcx: TyCtxt<'_>) -> bool { + tcx.check_is_object_safe(self.def_id) + } } #[derive(Clone, Debug)] @@ -2094,9 +2110,8 @@ impl Discriminant { pub(crate) fn expr(&self, tcx: TyCtxt<'_>) -> Option { self.expr.map(|body| rendered_const(tcx, body)) } - /// Will always be a machine readable number, without underscores or suffixes. - pub(crate) fn value(&self, tcx: TyCtxt<'_>) -> String { - print_evaluated_const(tcx, self.value, false).unwrap() + pub(crate) fn value(&self, tcx: TyCtxt<'_>, with_underscores: bool) -> String { + print_evaluated_const(tcx, self.value, with_underscores, false).unwrap() } } @@ -2271,8 +2286,8 @@ pub(crate) struct Static { #[derive(Clone, PartialEq, Eq, Hash, Debug)] pub(crate) struct Constant { - pub(crate) type_: Type, - pub(crate) generics: Box, + pub(crate) type_: Box, + pub(crate) generics: Generics, pub(crate) kind: ConstantKind, } @@ -2341,7 +2356,7 @@ impl ConstantKind { match *self { ConstantKind::TyConst { .. } | ConstantKind::Anonymous { .. } => None, ConstantKind::Extern { def_id } | ConstantKind::Local { def_id, .. } => { - print_evaluated_const(tcx, def_id, true) + print_evaluated_const(tcx, def_id, true, true) } } } @@ -2510,11 +2525,10 @@ mod size_asserts { static_assert_size!(DocFragment, 32); static_assert_size!(GenericArg, 32); static_assert_size!(GenericArgs, 32); - static_assert_size!(GenericParamDef, 56); + static_assert_size!(GenericParamDef, 40); static_assert_size!(Generics, 16); static_assert_size!(Item, 56); - // FIXME(generic_const_items): Further reduce the size. - static_assert_size!(ItemKind, 72); + static_assert_size!(ItemKind, 56); static_assert_size!(PathSegment, 40); static_assert_size!(Type, 32); // tidy-alphabetical-end diff --git a/src/librustdoc/clean/types/tests.rs b/src/librustdoc/clean/types/tests.rs index 4907a5527..ee7c0068e 100644 --- a/src/librustdoc/clean/types/tests.rs +++ b/src/librustdoc/clean/types/tests.rs @@ -1,9 +1,8 @@ use super::*; use rustc_resolve::rustdoc::{unindent_doc_fragments, DocFragment, DocFragmentKind}; -use rustc_span::create_default_session_globals_then; -use rustc_span::source_map::DUMMY_SP; use rustc_span::symbol::Symbol; +use rustc_span::{create_default_session_globals_then, DUMMY_SP}; fn create_doc_fragment(s: &str) -> Vec { vec![DocFragment { diff --git a/src/librustdoc/clean/utils.rs b/src/librustdoc/clean/utils.rs index 61e653423..9ff00c194 100644 --- a/src/librustdoc/clean/utils.rs +++ b/src/librustdoc/clean/utils.rs @@ -17,6 +17,7 @@ use rustc_hir::def_id::{DefId, LocalDefId, LOCAL_CRATE}; use rustc_metadata::rendered_const; use rustc_middle::mir; use rustc_middle::ty::{self, GenericArgKind, GenericArgsRef, TyCtxt}; +use rustc_middle::ty::{TypeVisitable, TypeVisitableExt}; use rustc_span::symbol::{kw, sym, Symbol}; use std::fmt::Write as _; use std::mem; @@ -76,44 +77,119 @@ pub(crate) fn krate(cx: &mut DocContext<'_>) -> Crate { pub(crate) fn ty_args_to_args<'tcx>( cx: &mut DocContext<'tcx>, - args: ty::Binder<'tcx, &'tcx [ty::GenericArg<'tcx>]>, + ty_args: ty::Binder<'tcx, &'tcx [ty::GenericArg<'tcx>]>, has_self: bool, - container: Option, + owner: DefId, ) -> Vec { - let mut skip_first = has_self; - let mut ret_val = - Vec::with_capacity(args.skip_binder().len().saturating_sub(if skip_first { 1 } else { 0 })); - - ret_val.extend(args.iter().enumerate().filter_map(|(index, kind)| { - match kind.skip_binder().unpack() { - GenericArgKind::Lifetime(lt) => { - Some(GenericArg::Lifetime(clean_middle_region(lt).unwrap_or(Lifetime::elided()))) - } - GenericArgKind::Type(_) if skip_first => { - skip_first = false; - None + if ty_args.skip_binder().is_empty() { + // Fast path which avoids executing the query `generics_of`. + return Vec::new(); + } + + let params = &cx.tcx.generics_of(owner).params; + let mut elision_has_failed_once_before = false; + + let offset = if has_self { 1 } else { 0 }; + let mut args = Vec::with_capacity(ty_args.skip_binder().len().saturating_sub(offset)); + + let ty_arg_to_arg = |(index, arg): (usize, &ty::GenericArg<'tcx>)| match arg.unpack() { + GenericArgKind::Lifetime(lt) => { + Some(GenericArg::Lifetime(clean_middle_region(lt).unwrap_or(Lifetime::elided()))) + } + GenericArgKind::Type(_) if has_self && index == 0 => None, + GenericArgKind::Type(ty) => { + if !elision_has_failed_once_before + && let Some(default) = params[index].default_value(cx.tcx) + { + let default = + ty_args.map_bound(|args| default.instantiate(cx.tcx, args).expect_ty()); + + if can_elide_generic_arg(ty_args.rebind(ty), default) { + return None; + } + + elision_has_failed_once_before = true; } - GenericArgKind::Type(ty) => Some(GenericArg::Type(clean_middle_ty( - kind.rebind(ty), + + Some(GenericArg::Type(clean_middle_ty( + ty_args.rebind(ty), cx, None, - container.map(|container| crate::clean::ContainerTy::Regular { - ty: container, - args, + Some(crate::clean::ContainerTy::Regular { + ty: owner, + args: ty_args, has_self, arg: index, }), - ))), - // FIXME(effects): this relies on the host effect being called `host`, which users could also name - // their const generics. - // FIXME(effects): this causes `host = true` and `host = false` generics to also be emitted. - GenericArgKind::Const(ct) if let ty::ConstKind::Param(p) = ct.kind() && p.name == sym::host => None, - GenericArgKind::Const(ct) => { - Some(GenericArg::Const(Box::new(clean_middle_const(kind.rebind(ct), cx)))) + ))) + } + GenericArgKind::Const(ct) => { + if let ty::GenericParamDefKind::Const { is_host_effect: true, .. } = params[index].kind + { + return None; + } + + if !elision_has_failed_once_before + && let Some(default) = params[index].default_value(cx.tcx) + { + let default = + ty_args.map_bound(|args| default.instantiate(cx.tcx, args).expect_const()); + + if can_elide_generic_arg(ty_args.rebind(ct), default) { + return None; + } + + elision_has_failed_once_before = true; } + + Some(GenericArg::Const(Box::new(clean_middle_const(ty_args.rebind(ct), cx)))) } - })); - ret_val + }; + + args.extend(ty_args.skip_binder().iter().enumerate().rev().filter_map(ty_arg_to_arg)); + args.reverse(); + args +} + +/// Check if the generic argument `actual` coincides with the `default` and can therefore be elided. +/// +/// This uses a very conservative approach for performance and correctness reasons, meaning for +/// several classes of terms it claims that they cannot be elided even if they theoretically could. +/// This is absolutely fine since it mostly concerns edge cases. +fn can_elide_generic_arg<'tcx, Term>( + actual: ty::Binder<'tcx, Term>, + default: ty::Binder<'tcx, Term>, +) -> bool +where + Term: Eq + TypeVisitable>, +{ + // In practice, we shouldn't have any inference variables at this point. + // However to be safe, we bail out if we do happen to stumble upon them. + if actual.has_infer() || default.has_infer() { + return false; + } + + // Since we don't properly keep track of bound variables in rustdoc (yet), we don't attempt to + // make any sense out of escaping bound variables. We simply don't have enough context and it + // would be incorrect to try to do so anyway. + if actual.has_escaping_bound_vars() || default.has_escaping_bound_vars() { + return false; + } + + // Theoretically we could now check if either term contains (non-escaping) late-bound regions or + // projections, relate the two using an `InferCtxt` and check if the resulting obligations hold. + // Having projections means that the terms can potentially be further normalized thereby possibly + // revealing that they are equal after all. Regarding late-bound regions, they could to be + // liberated allowing us to consider more types to be equal by ignoring the names of binders + // (e.g., `for<'a> TYPE<'a>` and `for<'b> TYPE<'b>`). + // + // However, we are mostly interested in “reeliding” generic args, i.e., eliding generic args that + // were originally elided by the user and later filled in by the compiler contrary to eliding + // arbitrary generic arguments if they happen to semantically coincide with the default (of course, + // we cannot possibly distinguish these two cases). Therefore and for performance reasons, it + // suffices to only perform a syntactic / structural check by comparing the memory addresses of + // the interned arguments. + actual.skip_binder() == default.skip_binder() } fn external_generic_args<'tcx>( @@ -123,7 +199,7 @@ fn external_generic_args<'tcx>( bindings: ThinVec, ty_args: ty::Binder<'tcx, GenericArgsRef<'tcx>>, ) -> GenericArgs { - let args = ty_args_to_args(cx, ty_args.map_bound(|args| &args[..]), has_self, Some(did)); + let args = ty_args_to_args(cx, ty_args.map_bound(|args| &args[..]), has_self, did); if cx.tcx.fn_trait_kind_from_def_id(did).is_some() { let ty = ty_args @@ -279,7 +355,8 @@ pub(crate) fn print_const(cx: &DocContext<'_>, n: ty::Const<'_>) -> String { pub(crate) fn print_evaluated_const( tcx: TyCtxt<'_>, def_id: DefId, - underscores_and_type: bool, + with_underscores: bool, + with_type: bool, ) -> Option { tcx.const_eval_poly(def_id).ok().and_then(|val| { let ty = tcx.type_of(def_id).instantiate_identity(); @@ -288,7 +365,7 @@ pub(crate) fn print_evaluated_const( (mir::ConstValue::Scalar(_), &ty::Adt(_, _)) => None, (mir::ConstValue::Scalar(_), _) => { let const_ = mir::Const::from_value(val, ty); - Some(print_const_with_custom_print_scalar(tcx, const_, underscores_and_type)) + Some(print_const_with_custom_print_scalar(tcx, const_, with_underscores, with_type)) } _ => None, } @@ -324,32 +401,37 @@ fn format_integer_with_underscore_sep(num: &str) -> String { fn print_const_with_custom_print_scalar<'tcx>( tcx: TyCtxt<'tcx>, ct: mir::Const<'tcx>, - underscores_and_type: bool, + with_underscores: bool, + with_type: bool, ) -> String { // Use a slightly different format for integer types which always shows the actual value. // For all other types, fallback to the original `pretty_print_const`. match (ct, ct.ty().kind()) { (mir::Const::Val(mir::ConstValue::Scalar(int), _), ty::Uint(ui)) => { - if underscores_and_type { - format!("{}{}", format_integer_with_underscore_sep(&int.to_string()), ui.name_str()) + let mut output = if with_underscores { + format_integer_with_underscore_sep(&int.to_string()) } else { int.to_string() + }; + if with_type { + output += ui.name_str(); } + output } (mir::Const::Val(mir::ConstValue::Scalar(int), _), ty::Int(i)) => { let ty = ct.ty(); let size = tcx.layout_of(ty::ParamEnv::empty().and(ty)).unwrap().size; let data = int.assert_bits(size); let sign_extended_data = size.sign_extend(data) as i128; - if underscores_and_type { - format!( - "{}{}", - format_integer_with_underscore_sep(&sign_extended_data.to_string()), - i.name_str() - ) + let mut output = if with_underscores { + format_integer_with_underscore_sep(&sign_extended_data.to_string()) } else { sign_extended_data.to_string() + }; + if with_type { + output += i.name_str(); } + output } _ => ct.to_string(), } @@ -502,7 +584,7 @@ pub(crate) fn has_doc_flag(tcx: TyCtxt<'_>, did: DefId, flag: Symbol) -> bool { /// Set by `bootstrap::Builder::doc_rust_lang_org_channel` in order to keep tests passing on beta/stable. pub(crate) const DOC_RUST_LANG_ORG_CHANNEL: &str = env!("DOC_RUST_LANG_ORG_CHANNEL"); pub(crate) static DOC_CHANNEL: Lazy<&'static str> = - Lazy::new(|| DOC_RUST_LANG_ORG_CHANNEL.rsplit("/").filter(|c| !c.is_empty()).next().unwrap()); + Lazy::new(|| DOC_RUST_LANG_ORG_CHANNEL.rsplit('/').filter(|c| !c.is_empty()).next().unwrap()); /// Render a sequence of macro arms in a format suitable for displaying to the user /// as part of an item declaration. diff --git a/src/librustdoc/core.rs b/src/librustdoc/core.rs index 3e6066c78..6d9f8b820 100644 --- a/src/librustdoc/core.rs +++ b/src/librustdoc/core.rs @@ -14,8 +14,8 @@ use rustc_lint::{late_lint_mod, MissingDoc}; use rustc_middle::hir::nested_filter; use rustc_middle::ty::{ParamEnv, Ty, TyCtxt}; use rustc_session::config::{self, CrateType, ErrorOutputType, ResolveDocLinks}; +use rustc_session::lint; use rustc_session::Session; -use rustc_session::{lint, EarlyErrorHandler}; use rustc_span::symbol::sym; use rustc_span::{source_map, Span}; @@ -23,6 +23,7 @@ use std::cell::RefCell; use std::mem; use std::rc::Rc; use std::sync::LazyLock; +use std::sync::{atomic::AtomicBool, Arc}; use crate::clean::inline::build_external_trait; use crate::clean::{self, ItemId}; @@ -174,7 +175,6 @@ pub(crate) fn new_handler( /// Parse, resolve, and typecheck the given crate. pub(crate) fn create_config( - handler: &EarlyErrorHandler, RustdocOptions { input, crate_name, @@ -198,6 +198,7 @@ pub(crate) fn create_config( .. }: RustdocOptions, RenderOptions { document_private, .. }: &RenderOptions, + using_internal_features: Arc, ) -> rustc_interface::Config { // Add the doc cfg into the doc build. cfgs.push("doc".to_string()); @@ -253,8 +254,8 @@ pub(crate) fn create_config( interface::Config { opts: sessopts, - crate_cfg: interface::parse_cfgspecs(handler, cfgs), - crate_check_cfg: interface::parse_check_cfg(handler, check_cfgs), + crate_cfg: cfgs, + crate_check_cfg: check_cfgs, input, output_file: None, output_dir: None, @@ -262,6 +263,7 @@ pub(crate) fn create_config( locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES, lint_caps, parse_sess_created: None, + hash_untracked_state: None, register_lints: Some(Box::new(crate::lint::register_lints)), override_queries: Some(|_sess, providers| { // We do not register late module lints, so this only runs `MissingDoc`. @@ -292,6 +294,7 @@ pub(crate) fn create_config( make_codegen_backend: None, registry: rustc_driver::diagnostics_registry(), ice_file: None, + using_internal_features, expanded_args, } } @@ -316,10 +319,14 @@ pub(crate) fn run_global_ctxt( tcx.hir().for_each_module(|module| tcx.ensure().collect_mod_item_types(module)) }); - // NOTE: This is copy/pasted from typeck/lib.rs and should be kept in sync with those changes. + // NOTE: These are copy/pasted from typeck/lib.rs and should be kept in sync with those changes. + let _ = tcx.sess.time("wf_checking", || { + tcx.hir().try_par_for_each_module(|module| tcx.ensure().check_mod_type_wf(module)) + }); tcx.sess.time("item_types_checking", || { tcx.hir().for_each_module(|module| tcx.ensure().check_mod_item_types(module)) }); + tcx.sess.abort_if_errors(); tcx.sess.time("missing_docs", || rustc_lint::check_crate(tcx)); tcx.sess.time("check_mod_attrs", || { diff --git a/src/librustdoc/doctest.rs b/src/librustdoc/doctest.rs index 741d329fb..241286580 100644 --- a/src/librustdoc/doctest.rs +++ b/src/librustdoc/doctest.rs @@ -13,7 +13,7 @@ use rustc_parse::parser::attr::InnerAttrPolicy; use rustc_resolve::rustdoc::span_of_fragments; use rustc_session::config::{self, CrateType, ErrorOutputType}; use rustc_session::parse::ParseSess; -use rustc_session::{lint, EarlyErrorHandler, Session}; +use rustc_session::{lint, Session}; use rustc_span::edition::Edition; use rustc_span::source_map::SourceMap; use rustc_span::symbol::sym; @@ -85,18 +85,13 @@ pub(crate) fn run(options: RustdocOptions) -> Result<(), ErrorGuaranteed> { ..config::Options::default() }; - let early_error_handler = EarlyErrorHandler::new(ErrorOutputType::default()); - let mut cfgs = options.cfgs.clone(); cfgs.push("doc".to_owned()); cfgs.push("doctest".to_owned()); let config = interface::Config { opts: sessopts, - crate_cfg: interface::parse_cfgspecs(&early_error_handler, cfgs), - crate_check_cfg: interface::parse_check_cfg( - &early_error_handler, - options.check_cfgs.clone(), - ), + crate_cfg: cfgs, + crate_check_cfg: options.check_cfgs.clone(), input, output_file: None, output_dir: None, @@ -104,11 +99,13 @@ pub(crate) fn run(options: RustdocOptions) -> Result<(), ErrorGuaranteed> { locale_resources: rustc_driver::DEFAULT_LOCALE_RESOURCES, lint_caps, parse_sess_created: None, + hash_untracked_state: None, register_lints: Some(Box::new(crate::lint::register_lints)), override_queries: None, make_codegen_backend: None, registry: rustc_driver::diagnostics_registry(), ice_file: None, + using_internal_features: Arc::default(), expanded_args: options.expanded_args.clone(), }; diff --git a/src/librustdoc/formats/cache.rs b/src/librustdoc/formats/cache.rs index 4c6e7dfb9..abff77253 100644 --- a/src/librustdoc/formats/cache.rs +++ b/src/librustdoc/formats/cache.rs @@ -50,8 +50,8 @@ pub(crate) struct Cache { /// Unlike 'paths', this mapping ignores any renames that occur /// due to 'use' statements. /// - /// This map is used when writing out the special 'implementors' - /// javascript file. By using the exact path that the type + /// This map is used when writing out the `impl.trait` and `impl.type` + /// javascript files. By using the exact path that the type /// is declared with, we ensure that each path will be identical /// to the path used if the corresponding type is inlined. By /// doing this, we can detect duplicate impls on a trait page, and only display @@ -221,19 +221,25 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { _ => self.cache.stripped_mod, }; + #[inline] + fn is_from_private_dep(tcx: TyCtxt<'_>, cache: &Cache, def_id: DefId) -> bool { + let krate = def_id.krate; + + cache.masked_crates.contains(&krate) || tcx.is_private_dep(krate) + } + // If the impl is from a masked crate or references something from a // masked crate then remove it completely. - if let clean::ImplItem(ref i) = *item.kind { - if self.cache.masked_crates.contains(&item.item_id.krate()) + if let clean::ImplItem(ref i) = *item.kind && + (self.cache.masked_crates.contains(&item.item_id.krate()) || i.trait_ .as_ref() - .map_or(false, |t| self.cache.masked_crates.contains(&t.def_id().krate)) + .map_or(false, |t| is_from_private_dep(self.tcx, self.cache, t.def_id())) || i.for_ .def_id(self.cache) - .map_or(false, |d| self.cache.masked_crates.contains(&d.krate)) - { - return None; - } + .map_or(false, |d| is_from_private_dep(self.tcx, self.cache, d))) + { + return None; } // Propagate a trait method's documentation to all implementors of the @@ -334,33 +340,37 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { // A crate has a module at its root, containing all items, // which should not be indexed. The crate-item itself is // inserted later on when serializing the search-index. - if item.item_id.as_def_id().map_or(false, |idx| !idx.is_crate_root()) { + if item.item_id.as_def_id().map_or(false, |idx| !idx.is_crate_root()) + && let ty = item.type_() + && (ty != ItemType::StructField + || u16::from_str_radix(s.as_str(), 10).is_err()) + { let desc = short_markdown_summary(&item.doc_value(), &item.link_names(self.cache)); - let ty = item.type_(); - if ty != ItemType::StructField - || u16::from_str_radix(s.as_str(), 10).is_err() - { - // In case this is a field from a tuple struct, we don't add it into - // the search index because its name is something like "0", which is - // not useful for rustdoc search. - self.cache.search_index.push(IndexItem { - ty, - name: s, - path: join_with_double_colon(path), - desc, - parent, - parent_idx: None, - search_type: get_function_type_for_search( - &item, - self.tcx, - clean_impl_generics(self.cache.parent_stack.last()).as_ref(), - self.cache, - ), - aliases: item.attrs.get_doc_aliases(), - deprecation: item.deprecation(self.tcx), - }); - } + // In case this is a field from a tuple struct, we don't add it into + // the search index because its name is something like "0", which is + // not useful for rustdoc search. + self.cache.search_index.push(IndexItem { + ty, + name: s, + path: join_with_double_colon(path), + desc, + parent, + parent_idx: None, + impl_id: if let Some(ParentStackItem::Impl { item_id, .. }) = self.cache.parent_stack.last() { + item_id.as_def_id() + } else { + None + }, + search_type: get_function_type_for_search( + &item, + self.tcx, + clean_impl_generics(self.cache.parent_stack.last()).as_ref(), + self.cache, + ), + aliases: item.attrs.get_doc_aliases(), + deprecation: item.deprecation(self.tcx), + }); } } (Some(parent), None) if is_inherent_impl_item => { @@ -371,6 +381,13 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { parent, item: item.clone(), impl_generics, + impl_id: if let Some(ParentStackItem::Impl { item_id, .. }) = + self.cache.parent_stack.last() + { + item_id.as_def_id() + } else { + None + }, }); } _ => {} @@ -541,6 +558,7 @@ impl<'a, 'tcx> DocFolder for CacheBuilder<'a, 'tcx> { pub(crate) struct OrphanImplItem { pub(crate) parent: DefId, + pub(crate) impl_id: Option, pub(crate) item: clean::Item, pub(crate) impl_generics: Option<(clean::Type, clean::Generics)>, } diff --git a/src/librustdoc/formats/item_type.rs b/src/librustdoc/formats/item_type.rs index be2ee7915..def3a90c8 100644 --- a/src/librustdoc/formats/item_type.rs +++ b/src/librustdoc/formats/item_type.rs @@ -141,7 +141,7 @@ impl From for ItemType { | DefKind::GlobalAsm | DefKind::Impl { .. } | DefKind::Closure - | DefKind::Generator => Self::ForeignType, + | DefKind::Coroutine => Self::ForeignType, } } } @@ -180,6 +180,9 @@ impl ItemType { pub(crate) fn is_method(&self) -> bool { matches!(*self, ItemType::Method | ItemType::TyMethod) } + pub(crate) fn is_adt(&self) -> bool { + matches!(*self, ItemType::Struct | ItemType::Union | ItemType::Enum) + } } impl fmt::Display for ItemType { diff --git a/src/librustdoc/html/format.rs b/src/librustdoc/html/format.rs index 2751b6613..29fd880af 100644 --- a/src/librustdoc/html/format.rs +++ b/src/librustdoc/html/format.rs @@ -325,8 +325,7 @@ pub(crate) fn print_where_clause<'a, 'tcx: 'a>( bounds_display.truncate(bounds_display.len() - " + ".len()); write!(f, "{}: {bounds_display}", lifetime.print()) } - // FIXME(fmease): Render bound params. - clean::WherePredicate::EqPredicate { lhs, rhs, bound_params: _ } => { + clean::WherePredicate::EqPredicate { lhs, rhs } => { if f.alternate() { write!(f, "{:#} == {:#}", lhs.print(cx), rhs.print(cx)) } else { @@ -848,7 +847,7 @@ fn resolved_path<'cx>( fn primitive_link( f: &mut fmt::Formatter<'_>, prim: clean::PrimitiveType, - name: &str, + name: fmt::Arguments<'_>, cx: &Context<'_>, ) -> fmt::Result { primitive_link_fragment(f, prim, name, "", cx) @@ -857,7 +856,7 @@ fn primitive_link( fn primitive_link_fragment( f: &mut fmt::Formatter<'_>, prim: clean::PrimitiveType, - name: &str, + name: fmt::Arguments<'_>, fragment: &str, cx: &Context<'_>, ) -> fmt::Result { @@ -908,7 +907,7 @@ fn primitive_link_fragment( None => {} } } - f.write_str(name)?; + std::fmt::Display::fmt(&name, f)?; if needs_termination { write!(f, "")?; } @@ -978,9 +977,11 @@ fn fmt_type<'cx>( } clean::Infer => write!(f, "_"), clean::Primitive(clean::PrimitiveType::Never) => { - primitive_link(f, PrimitiveType::Never, "!", cx) + primitive_link(f, PrimitiveType::Never, format_args!("!"), cx) + } + clean::Primitive(prim) => { + primitive_link(f, prim, format_args!("{}", prim.as_sym().as_str()), cx) } - clean::Primitive(prim) => primitive_link(f, prim, prim.as_sym().as_str(), cx), clean::BareFunction(ref decl) => { if f.alternate() { write!( @@ -999,16 +1000,16 @@ fn fmt_type<'cx>( decl.unsafety.print_with_space(), print_abi_with_space(decl.abi) )?; - primitive_link(f, PrimitiveType::Fn, "fn", cx)?; + primitive_link(f, PrimitiveType::Fn, format_args!("fn"), cx)?; write!(f, "{}", decl.decl.print(cx)) } } clean::Tuple(ref typs) => { match &typs[..] { - &[] => primitive_link(f, PrimitiveType::Unit, "()", cx), + &[] => primitive_link(f, PrimitiveType::Unit, format_args!("()"), cx), [one] => { if let clean::Generic(name) = one { - primitive_link(f, PrimitiveType::Tuple, &format!("({name},)"), cx) + primitive_link(f, PrimitiveType::Tuple, format_args!("({name},)"), cx) } else { write!(f, "(")?; // Carry `f.alternate()` into this display w/o branching manually. @@ -1029,7 +1030,10 @@ fn fmt_type<'cx>( primitive_link( f, PrimitiveType::Tuple, - &format!("({})", generic_names.iter().map(|s| s.as_str()).join(", ")), + format_args!( + "({})", + generic_names.iter().map(|s| s.as_str()).join(", ") + ), cx, ) } else { @@ -1048,7 +1052,7 @@ fn fmt_type<'cx>( } clean::Slice(ref t) => match **t { clean::Generic(name) => { - primitive_link(f, PrimitiveType::Slice, &format!("[{name}]"), cx) + primitive_link(f, PrimitiveType::Slice, format_args!("[{name}]"), cx) } _ => { write!(f, "[")?; @@ -1060,7 +1064,7 @@ fn fmt_type<'cx>( clean::Generic(name) if !f.alternate() => primitive_link( f, PrimitiveType::Array, - &format!("[{name}; {n}]", n = Escape(n)), + format_args!("[{name}; {n}]", n = Escape(n)), cx, ), _ => { @@ -1070,7 +1074,12 @@ fn fmt_type<'cx>( write!(f, "; {n}")?; } else { write!(f, "; ")?; - primitive_link(f, PrimitiveType::Array, &format!("{n}", n = Escape(n)), cx)?; + primitive_link( + f, + PrimitiveType::Array, + format_args!("{n}", n = Escape(n)), + cx, + )?; } write!(f, "]") } @@ -1082,22 +1091,32 @@ fn fmt_type<'cx>( }; if matches!(**t, clean::Generic(_)) || t.is_assoc_ty() { - let text = if f.alternate() { - format!("*{m} {ty:#}", ty = t.print(cx)) + let ty = t.print(cx); + if f.alternate() { + primitive_link( + f, + clean::PrimitiveType::RawPointer, + format_args!("*{m} {ty:#}"), + cx, + ) } else { - format!("*{m} {ty}", ty = t.print(cx)) - }; - primitive_link(f, clean::PrimitiveType::RawPointer, &text, cx) + primitive_link( + f, + clean::PrimitiveType::RawPointer, + format_args!("*{m} {ty}"), + cx, + ) + } } else { - primitive_link(f, clean::PrimitiveType::RawPointer, &format!("*{m} "), cx)?; + primitive_link(f, clean::PrimitiveType::RawPointer, format_args!("*{m} "), cx)?; fmt::Display::fmt(&t.print(cx), f) } } clean::BorrowedRef { lifetime: ref l, mutability, type_: ref ty } => { - let lt = match l { - Some(l) => format!("{} ", l.print()), - _ => String::new(), - }; + let lt = display_fn(|f| match l { + Some(l) => write!(f, "{} ", l.print()), + _ => Ok(()), + }); let m = mutability.print_with_space(); let amp = if f.alternate() { "&" } else { "&" }; @@ -1105,7 +1124,7 @@ fn fmt_type<'cx>( return primitive_link( f, PrimitiveType::Reference, - &format!("{amp}{lt}{m}{name}"), + format_args!("{amp}{lt}{m}{name}"), cx, ); } @@ -1255,7 +1274,7 @@ impl clean::Impl { { // Hardcoded anchor library/core/src/primitive_docs.rs // Link should match `# Trait implementations` - primitive_link_fragment(f, PrimitiveType::Tuple, &format!("({name}₁, {name}₂, …, {name}ₙ)"), "#trait-implementations-1", cx)?; + primitive_link_fragment(f, PrimitiveType::Tuple, format_args!("({name}₁, {name}₂, …, {name}ₙ)"), "#trait-implementations-1", cx)?; } else if let clean::BareFunction(bare_fn) = &self.for_ && let [clean::Argument { type_: clean::Type::Generic(name), .. }] = &bare_fn.decl.inputs.values[..] && (self.kind.is_fake_variadic() || self.kind.is_auto()) @@ -1282,7 +1301,7 @@ impl clean::Impl { } else { "" }; - primitive_link_fragment(f, PrimitiveType::Tuple, &format!("fn ({name}₁, {name}₂, …, {name}ₙ{ellipsis})"), "#trait-implementations-1", cx)?; + primitive_link_fragment(f, PrimitiveType::Tuple, format_args!("fn ({name}₁, {name}₂, …, {name}ₙ{ellipsis})"), "#trait-implementations-1", cx)?; // Write output. if !bare_fn.decl.output.is_unit() { write!(f, " -> ")?; @@ -1666,7 +1685,12 @@ impl clean::ImportSource { } let name = self.path.last(); if let hir::def::Res::PrimTy(p) = self.path.res { - primitive_link(f, PrimitiveType::from(p), name.as_str(), cx)?; + primitive_link( + f, + PrimitiveType::from(p), + format_args!("{}", name.as_str()), + cx, + )?; } else { f.write_str(name.as_str())?; } diff --git a/src/librustdoc/html/layout.rs b/src/librustdoc/html/layout.rs index 8c5871d91..d4b4db0f3 100644 --- a/src/librustdoc/html/layout.rs +++ b/src/librustdoc/html/layout.rs @@ -17,6 +17,7 @@ pub(crate) struct Layout { pub(crate) external_html: ExternalHtml, pub(crate) default_settings: FxHashMap, pub(crate) krate: String, + pub(crate) krate_version: String, /// The given user css file which allow to customize the generated /// documentation theme. pub(crate) css_file_extension: Option, @@ -31,6 +32,7 @@ pub(crate) struct Page<'a> { pub(crate) static_root_path: Option<&'a str>, pub(crate) description: &'a str, pub(crate) resource_suffix: &'a str, + pub(crate) rust_logo: bool, } impl<'a> Page<'a> { @@ -54,9 +56,19 @@ struct PageLayout<'a> { themes: Vec, sidebar: String, content: String, - krate_with_trailing_slash: String, rust_channel: &'static str, pub(crate) rustdoc_version: &'a str, + // same as layout.krate, except on top-level pages like + // Settings, Help, All Crates, and About Scraped Examples, + // where these things instead give Rustdoc name and version. + // + // These are separate from the variables used for the search + // engine, because "Rustdoc" isn't necessarily a crate in + // the current workspace. + display_krate: &'a str, + display_krate_with_trailing_slash: String, + display_krate_version_number: &'a str, + display_krate_version_extra: &'a str, } pub(crate) fn render( @@ -66,12 +78,26 @@ pub(crate) fn render( t: T, style_files: &[StylePath], ) -> String { + let rustdoc_version = rustc_interface::util::version_str!().unwrap_or("unknown version"); + + let (display_krate, display_krate_version, display_krate_with_trailing_slash) = + if page.root_path == "./" { + // top level pages use Rust branding + ("Rustdoc", rustdoc_version, String::new()) + } else { + let display_krate_with_trailing_slash = + ensure_trailing_slash(&layout.krate).to_string(); + (&layout.krate[..], &layout.krate_version[..], display_krate_with_trailing_slash) + }; let static_root_path = page.get_static_root_path(); - let krate_with_trailing_slash = ensure_trailing_slash(&layout.krate).to_string(); + + // bootstrap passes in parts of the version separated by tabs, but other stuff might use spaces + let (display_krate_version_number, display_krate_version_extra) = + display_krate_version.split_once([' ', '\t']).unwrap_or((display_krate_version, "")); + let mut themes: Vec = style_files.iter().map(|s| s.basename().unwrap()).collect(); themes.sort(); - let rustdoc_version = rustc_interface::util::version_str!().unwrap_or("unknown version"); let content = Buffer::html().to_display(t); // Note: This must happen before making the sidebar. let sidebar = Buffer::html().to_display(sidebar); PageLayout { @@ -82,7 +108,10 @@ pub(crate) fn render( themes, sidebar, content, - krate_with_trailing_slash, + display_krate, + display_krate_with_trailing_slash, + display_krate_version_number, + display_krate_version_extra, rust_channel: *crate::clean::utils::DOC_CHANNEL, rustdoc_version, } diff --git a/src/librustdoc/html/markdown.rs b/src/librustdoc/html/markdown.rs index d24e6e5fa..2807dfed0 100644 --- a/src/librustdoc/html/markdown.rs +++ b/src/librustdoc/html/markdown.rs @@ -1750,7 +1750,7 @@ pub(crate) fn markdown_links<'md, R>( } // do not actually include braces in the span let range = (open_brace + 1)..close_brace; - MarkdownLinkRange::Destination(range.clone()) + MarkdownLinkRange::Destination(range) }; let span_for_offset_forward = |span: Range, open: u8, close: u8| { @@ -1786,7 +1786,7 @@ pub(crate) fn markdown_links<'md, R>( } // do not actually include braces in the span let range = (open_brace + 1)..close_brace; - MarkdownLinkRange::Destination(range.clone()) + MarkdownLinkRange::Destination(range) }; let mut broken_link_callback = |link: BrokenLink<'md>| Some((link.reference, "".into())); @@ -2024,6 +2024,7 @@ fn init_id_map() -> FxHashMap, usize> { map.insert("required-associated-consts".into(), 1); map.insert("required-methods".into(), 1); map.insert("provided-methods".into(), 1); + map.insert("object-safety".into(), 1); map.insert("implementors".into(), 1); map.insert("synthetic-implementors".into(), 1); map.insert("implementations-list".into(), 1); diff --git a/src/librustdoc/html/render/context.rs b/src/librustdoc/html/render/context.rs index 97714afaa..50777134d 100644 --- a/src/librustdoc/html/render/context.rs +++ b/src/librustdoc/html/render/context.rs @@ -7,13 +7,10 @@ use std::sync::mpsc::{channel, Receiver}; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir::def_id::{DefIdMap, LOCAL_CRATE}; -use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams}; use rustc_middle::ty::TyCtxt; use rustc_session::Session; -use rustc_span::def_id::DefId; use rustc_span::edition::Edition; -use rustc_span::source_map::FileName; -use rustc_span::{sym, Symbol}; +use rustc_span::{sym, FileName, Symbol}; use super::print_item::{full_path, item_path, print_item}; use super::search_index::build_index; @@ -24,13 +21,14 @@ use super::{ sidebar::{sidebar_module_like, Sidebar}, AllTypes, LinkFromSrc, StylePath, }; -use crate::clean::{self, types::ExternalLocation, ExternalCrate, TypeAliasItem}; +use crate::clean::utils::has_doc_flag; +use crate::clean::{self, types::ExternalLocation, ExternalCrate}; use crate::config::{ModuleSorting, RenderOptions}; use crate::docfs::{DocFS, PathError}; use crate::error::Error; use crate::formats::cache::Cache; use crate::formats::item_type::ItemType; -use crate::formats::{self, FormatRenderer}; +use crate::formats::FormatRenderer; use crate::html::escape::Escape; use crate::html::format::{join_with_double_colon, Buffer}; use crate::html::markdown::{self, plain_text_summary, ErrorCodes, IdMap}; @@ -149,53 +147,6 @@ impl SharedContext<'_> { pub(crate) fn edition(&self) -> Edition { self.tcx.sess.edition() } - - /// Returns a list of impls on the given type, and, if it's a type alias, - /// other types that it aliases. - pub(crate) fn all_impls_for_item<'a>( - &'a self, - it: &clean::Item, - did: DefId, - ) -> Vec<&'a formats::Impl> { - let tcx = self.tcx; - let cache = &self.cache; - let mut saw_impls = FxHashSet::default(); - let mut v: Vec<&formats::Impl> = cache - .impls - .get(&did) - .map(Vec::as_slice) - .unwrap_or(&[]) - .iter() - .filter(|i| saw_impls.insert(i.def_id())) - .collect(); - if let TypeAliasItem(ait) = &*it.kind && - let aliased_clean_type = ait.item_type.as_ref().unwrap_or(&ait.type_) && - let Some(aliased_type_defid) = aliased_clean_type.def_id(cache) && - let Some(av) = cache.impls.get(&aliased_type_defid) && - let Some(alias_def_id) = it.item_id.as_def_id() - { - // This branch of the compiler compares types structually, but does - // not check trait bounds. That's probably fine, since type aliases - // don't normally constrain on them anyway. - // https://github.com/rust-lang/rust/issues/21903 - // - // FIXME(lazy_type_alias): Once the feature is complete or stable, rewrite this to use type unification. - // Be aware of `tests/rustdoc/issue-112515-impl-ty-alias.rs` which might regress. - let aliased_ty = tcx.type_of(alias_def_id).skip_binder(); - let reject_cx = DeepRejectCtxt { - treat_obligation_params: TreatParams::AsCandidateKey, - }; - v.extend(av.iter().filter(|impl_| { - if let Some(impl_def_id) = impl_.impl_item.item_id.as_def_id() { - reject_cx.types_may_unify(aliased_ty, tcx.type_of(impl_def_id).skip_binder()) - && saw_impls.insert(impl_def_id) - } else { - false - } - })); - } - v - } } impl<'tcx> Context<'tcx> { @@ -277,6 +228,7 @@ impl<'tcx> Context<'tcx> { title: &title, description: &desc, resource_suffix: &clone_shared.resource_suffix, + rust_logo: has_doc_flag(self.tcx(), LOCAL_CRATE.as_def_id(), sym::rust_logo), }; let mut page_buffer = Buffer::html(); print_item(self, it, &mut page_buffer, &page); @@ -528,12 +480,14 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> { if let Some(url) = playground_url { playground = Some(markdown::Playground { crate_name: Some(krate.name(tcx)), url }); } + let krate_version = cache.crate_version.as_deref().unwrap_or_default(); let mut layout = layout::Layout { logo: String::new(), favicon: String::new(), external_html, default_settings, krate: krate.name(tcx).to_string(), + krate_version: krate_version.to_string(), css_file_extension: extension_css, scrape_examples_extension: !call_locations.is_empty(), }; @@ -658,21 +612,22 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> { let shared = Rc::clone(&self.shared); let mut page = layout::Page { title: "List of all items in this crate", - css_class: "mod", + css_class: "mod sys", root_path: "../", static_root_path: shared.static_root_path.as_deref(), description: "List of all items in this crate", resource_suffix: &shared.resource_suffix, + rust_logo: has_doc_flag(self.tcx(), LOCAL_CRATE.as_def_id(), sym::rust_logo), }; let all = shared.all.replace(AllTypes::new()); let mut sidebar = Buffer::html(); let blocks = sidebar_module_like(all.item_sections()); let bar = Sidebar { - title_prefix: "Crate ", - title: crate_name.as_str(), + title_prefix: "", + title: "", is_crate: false, - version: "", + is_mod: false, blocks: vec![blocks], path: String::new(), }; @@ -689,9 +644,10 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> { shared.fs.write(final_file, v)?; // Generating settings page. - page.title = "Rustdoc settings"; + page.title = "Settings"; page.description = "Settings of Rustdoc"; page.root_path = "./"; + page.rust_logo = true; let sidebar = "

Settings

"; let v = layout::render( @@ -739,9 +695,10 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> { shared.fs.write(settings_file, v)?; // Generating help page. - page.title = "Rustdoc help"; + page.title = "Help"; page.description = "Documentation for Rustdoc"; page.root_path = "./"; + page.rust_logo = true; let sidebar = "

Help

"; let v = layout::render( diff --git a/src/librustdoc/html/render/mod.rs b/src/librustdoc/html/render/mod.rs index 3e671a64b..c52fa01bd 100644 --- a/src/librustdoc/html/render/mod.rs +++ b/src/librustdoc/html/render/mod.rs @@ -48,13 +48,13 @@ use std::str; use std::string::ToString; use askama::Template; -use rustc_attr::{ConstStability, Deprecation, StabilityLevel}; +use rustc_attr::{ConstStability, DeprecatedSince, Deprecation, StabilityLevel, StableSince}; use rustc_data_structures::captures::Captures; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_hir::def_id::{DefId, DefIdSet}; use rustc_hir::Mutability; -use rustc_middle::middle::stability; -use rustc_middle::ty::TyCtxt; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_session::RustcVersion; use rustc_span::{ symbol::{sym, Symbol}, BytePos, FileName, RealFileName, @@ -102,6 +102,7 @@ pub(crate) struct IndexItem { pub(crate) desc: String, pub(crate) parent: Option, pub(crate) parent_idx: Option, + pub(crate) impl_id: Option, pub(crate) search_type: Option, pub(crate) aliases: Box<[Symbol]>, pub(crate) deprecation: Option, @@ -615,24 +616,22 @@ fn short_item_info( ) -> Vec { let mut extra_info = vec![]; - if let Some(depr @ Deprecation { note, since, is_since_rustc_version: _, suggestion: _ }) = - item.deprecation(cx.tcx()) - { + if let Some(depr @ Deprecation { note, since, suggestion: _ }) = item.deprecation(cx.tcx()) { // We display deprecation messages for #[deprecated], but only display // the future-deprecation messages for rustc versions. - let mut message = if let Some(since) = since { - let since = since.as_str(); - if !stability::deprecation_in_effect(&depr) { - if since == "TBD" { - String::from("Deprecating in a future Rust version") + let mut message = match since { + DeprecatedSince::RustcVersion(version) => { + if depr.is_in_effect() { + format!("Deprecated since {version}") } else { - format!("Deprecating in {}", Escape(since)) + format!("Deprecating in {version}") } - } else { - format!("Deprecated since {}", Escape(since)) } - } else { - String::from("Deprecated") + DeprecatedSince::Future => String::from("Deprecating in a future Rust version"), + DeprecatedSince::NonStandard(since) => { + format!("Deprecated since {}", Escape(since.as_str())) + } + DeprecatedSince::Unspecified | DeprecatedSince::Err => String::from("Deprecated"), }; if let Some(note) = note { @@ -867,10 +866,10 @@ fn assoc_method( let (indent, indent_str, end_newline) = if parent == ItemType::Trait { header_len += 4; let indent_str = " "; - write!(w, "{}", render_attributes_in_pre(meth, indent_str, tcx)); + write!(w, "{}", render_attributes_in_pre(meth, indent_str, cx)); (4, indent_str, Ending::NoNewline) } else { - render_attributes_in_code(w, meth, tcx); + render_attributes_in_code(w, meth, cx); (0, "", Ending::Newline) }; w.reserve(header_len + "{".len() + "".len()); @@ -910,13 +909,17 @@ fn assoc_method( /// consequence of the above rules. fn render_stability_since_raw_with_extra( w: &mut Buffer, - ver: Option, + ver: Option, const_stability: Option, - containing_ver: Option, - containing_const_ver: Option, + containing_ver: Option, + containing_const_ver: Option, extra_class: &str, ) -> bool { - let stable_version = ver.filter(|inner| !inner.is_empty() && Some(*inner) != containing_ver); + let stable_version = if ver != containing_ver && let Some(ver) = &ver { + since_to_string(ver) + } else { + None + }; let mut title = String::new(); let mut stability = String::new(); @@ -930,7 +933,8 @@ fn render_stability_since_raw_with_extra( Some(ConstStability { level: StabilityLevel::Stable { since, .. }, .. }) if Some(since) != containing_const_ver => { - Some((format!("const since {since}"), format!("const: {since}"))) + since_to_string(&since) + .map(|since| (format!("const since {since}"), format!("const: {since}"))) } Some(ConstStability { level: StabilityLevel::Unstable { issue, .. }, feature, .. }) => { let unstable = if let Some(n) = issue { @@ -970,13 +974,21 @@ fn render_stability_since_raw_with_extra( !stability.is_empty() } +fn since_to_string(since: &StableSince) -> Option { + match since { + StableSince::Version(since) => Some(since.to_string()), + StableSince::Current => Some(RustcVersion::CURRENT.to_string()), + StableSince::Err => None, + } +} + #[inline] fn render_stability_since_raw( w: &mut Buffer, - ver: Option, + ver: Option, const_stability: Option, - containing_ver: Option, - containing_const_ver: Option, + containing_ver: Option, + containing_const_ver: Option, ) -> bool { render_stability_since_raw_with_extra( w, @@ -1046,13 +1058,13 @@ fn render_assoc_item( // When an attribute is rendered inside a `
` tag, it is formatted using
 // a whitespace prefix and newline.
-fn render_attributes_in_pre<'a, 'b: 'a>(
+fn render_attributes_in_pre<'a, 'tcx: 'a>(
     it: &'a clean::Item,
     prefix: &'a str,
-    tcx: TyCtxt<'b>,
-) -> impl fmt::Display + Captures<'a> + Captures<'b> {
+    cx: &'a Context<'tcx>,
+) -> impl fmt::Display + Captures<'a> + Captures<'tcx> {
     crate::html::format::display_fn(move |f| {
-        for a in it.attributes(tcx, false) {
+        for a in it.attributes(cx.tcx(), cx.cache(), false) {
             writeln!(f, "{prefix}{a}")?;
         }
         Ok(())
@@ -1061,8 +1073,8 @@ fn render_attributes_in_pre<'a, 'b: 'a>(
 
 // When an attribute is rendered inside a  tag, it is formatted using
 // a div to produce a newline after it.
-fn render_attributes_in_code(w: &mut impl fmt::Write, it: &clean::Item, tcx: TyCtxt<'_>) {
-    for attr in it.attributes(tcx, false) {
+fn render_attributes_in_code(w: &mut impl fmt::Write, it: &clean::Item, cx: &Context<'_>) {
+    for attr in it.attributes(cx.tcx(), cx.cache(), false) {
         write!(w, "
{attr}
").unwrap();
     }
 }
@@ -1131,13 +1143,13 @@ pub(crate) fn render_all_impls(
 fn render_assoc_items<'a, 'cx: 'a>(
     cx: &'a mut Context<'cx>,
     containing_item: &'a clean::Item,
-    did: DefId,
+    it: DefId,
     what: AssocItemRender<'a>,
 ) -> impl fmt::Display + 'a + Captures<'cx> {
     let mut derefs = DefIdSet::default();
-    derefs.insert(did);
+    derefs.insert(it);
     display_fn(move |f| {
-        render_assoc_items_inner(f, cx, containing_item, did, what, &mut derefs);
+        render_assoc_items_inner(f, cx, containing_item, it, what, &mut derefs);
         Ok(())
     })
 }
@@ -1146,17 +1158,15 @@ fn render_assoc_items_inner(
     mut w: &mut dyn fmt::Write,
     cx: &mut Context<'_>,
     containing_item: &clean::Item,
-    did: DefId,
+    it: DefId,
     what: AssocItemRender<'_>,
     derefs: &mut DefIdSet,
 ) {
     info!("Documenting associated items of {:?}", containing_item.name);
     let shared = Rc::clone(&cx.shared);
-    let v = shared.all_impls_for_item(containing_item, did);
-    let v = v.as_slice();
-    let (non_trait, traits): (Vec<&Impl>, _) =
-        v.iter().partition(|i| i.inner_impl().trait_.is_none());
-    let mut saw_impls = FxHashSet::default();
+    let cache = &shared.cache;
+    let Some(v) = cache.impls.get(&it) else { return };
+    let (non_trait, traits): (Vec<_>, _) = v.iter().partition(|i| i.inner_impl().trait_.is_none());
     if !non_trait.is_empty() {
         let mut tmp_buf = Buffer::html();
         let (render_mode, id, class_html) = match what {
@@ -1185,9 +1195,6 @@ fn render_assoc_items_inner(
         };
         let mut impls_buf = Buffer::html();
         for i in &non_trait {
-            if !saw_impls.insert(i.def_id()) {
-                continue;
-            }
             render_impl(
                 &mut impls_buf,
                 cx,
@@ -1233,10 +1240,8 @@ fn render_assoc_items_inner(
 
         let (synthetic, concrete): (Vec<&Impl>, Vec<&Impl>) =
             traits.into_iter().partition(|t| t.inner_impl().kind.is_auto());
-        let (blanket_impl, concrete): (Vec<&Impl>, _) = concrete
-            .into_iter()
-            .filter(|t| saw_impls.insert(t.def_id()))
-            .partition(|t| t.inner_impl().kind.is_blanket());
+        let (blanket_impl, concrete): (Vec<&Impl>, _) =
+            concrete.into_iter().partition(|t| t.inner_impl().kind.is_blanket());
 
         render_all_impls(w, cx, containing_item, &concrete, &synthetic, &blanket_impl);
     }
@@ -1877,7 +1882,7 @@ pub(crate) fn render_impl_summary(
     aliases: &[String],
 ) {
     let inner_impl = i.inner_impl();
-    let id = cx.derive_id(get_id_for_impl(&inner_impl.for_, inner_impl.trait_.as_ref(), cx));
+    let id = cx.derive_id(get_id_for_impl(cx.tcx(), i.impl_item.item_id));
     let aliases = if aliases.is_empty() {
         String::new()
     } else {
@@ -1994,21 +1999,35 @@ pub(crate) fn small_url_encode(s: String) -> String {
     }
 }
 
-fn get_id_for_impl(for_: &clean::Type, trait_: Option<&clean::Path>, cx: &Context<'_>) -> String {
-    match trait_ {
-        Some(t) => small_url_encode(format!("impl-{:#}-for-{:#}", t.print(cx), for_.print(cx))),
-        None => small_url_encode(format!("impl-{:#}", for_.print(cx))),
-    }
+fn get_id_for_impl<'tcx>(tcx: TyCtxt<'tcx>, impl_id: ItemId) -> String {
+    use rustc_middle::ty::print::with_forced_trimmed_paths;
+    let (type_, trait_) = match impl_id {
+        ItemId::Auto { trait_, for_ } => {
+            let ty = tcx.type_of(for_).skip_binder();
+            (ty, Some(ty::TraitRef::new(tcx, trait_, [ty])))
+        }
+        ItemId::Blanket { impl_id, .. } | ItemId::DefId(impl_id) => {
+            match tcx.impl_subject(impl_id).skip_binder() {
+                ty::ImplSubject::Trait(trait_ref) => {
+                    (trait_ref.args[0].expect_ty(), Some(trait_ref))
+                }
+                ty::ImplSubject::Inherent(ty) => (ty, None),
+            }
+        }
+    };
+    with_forced_trimmed_paths!(small_url_encode(if let Some(trait_) = trait_ {
+        format!("impl-{trait_}-for-{type_}", trait_ = trait_.print_only_trait_path())
+    } else {
+        format!("impl-{type_}")
+    }))
 }
 
 fn extract_for_impl_name(item: &clean::Item, cx: &Context<'_>) -> Option<(String, String)> {
     match *item.kind {
-        clean::ItemKind::ImplItem(ref i) => {
-            i.trait_.as_ref().map(|trait_| {
-                // Alternative format produces no URLs,
-                // so this parameter does nothing.
-                (format!("{:#}", i.for_.print(cx)), get_id_for_impl(&i.for_, Some(trait_), cx))
-            })
+        clean::ItemKind::ImplItem(ref i) if i.trait_.is_some() => {
+            // Alternative format produces no URLs,
+            // so this parameter does nothing.
+            Some((format!("{:#}", i.for_.print(cx)), get_id_for_impl(cx.tcx(), item.item_id)))
         }
         _ => None,
     }
@@ -2079,6 +2098,7 @@ impl ItemSection {
     const ALL: &'static [Self] = {
         use ItemSection::*;
         // NOTE: The order here affects the order in the UI.
+        // Keep this synchronized with addSidebarItems in main.js
         &[
             Reexports,
             PrimitiveTypes,
diff --git a/src/librustdoc/html/render/print_item.rs b/src/librustdoc/html/render/print_item.rs
index c6751c958..d226701ba 100644
--- a/src/librustdoc/html/render/print_item.rs
+++ b/src/librustdoc/html/render/print_item.rs
@@ -5,10 +5,12 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir as hir;
 use rustc_hir::def::CtorKind;
 use rustc_hir::def_id::DefId;
-use rustc_middle::middle::stability;
+use rustc_index::IndexVec;
+use rustc_middle::query::Key;
 use rustc_middle::ty::{self, TyCtxt};
 use rustc_span::hygiene::MacroKind;
 use rustc_span::symbol::{kw, sym, Symbol};
+use rustc_target::abi::VariantIdx;
 use std::cell::{RefCell, RefMut};
 use std::cmp::Ordering;
 use std::fmt;
@@ -117,8 +119,7 @@ macro_rules! item_template_methods {
         fn render_attributes_in_pre<'b>(&'b self) -> impl fmt::Display + Captures<'a> + 'b + Captures<'cx> {
             display_fn(move |f| {
                 let (item, cx) = self.item_and_mut_cx();
-                let tcx = cx.tcx();
-                let v = render_attributes_in_pre(item, "", tcx);
+                let v = render_attributes_in_pre(item, "", &cx);
                 write!(f, "{v}")
             })
         }
@@ -589,11 +590,7 @@ fn extra_info_tags<'a, 'tcx: 'a>(
 
         // The trailing space after each tag is to space it properly against the rest of the docs.
         if let Some(depr) = &item.deprecation(tcx) {
-            let message = if stability::deprecation_in_effect(depr) {
-                "Deprecated"
-            } else {
-                "Deprecation planned"
-            };
+            let message = if depr.is_in_effect() { "Deprecated" } else { "Deprecation planned" };
             write!(f, "{}", tag_html("deprecated", "", message))?;
         }
 
@@ -656,7 +653,7 @@ fn item_function(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, f: &cle
             w,
             "{attrs}{vis}{constness}{asyncness}{unsafety}{abi}fn \
                 {name}{generics}{decl}{notable_traits}{where_clause}",
-            attrs = render_attributes_in_pre(it, "", tcx),
+            attrs = render_attributes_in_pre(it, "", cx),
             vis = visibility,
             constness = constness,
             asyncness = asyncness,
@@ -691,7 +688,7 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
         write!(
             w,
             "{attrs}{vis}{unsafety}{is_auto}trait {name}{generics}{bounds}",
-            attrs = render_attributes_in_pre(it, "", tcx),
+            attrs = render_attributes_in_pre(it, "", cx),
             vis = visibility_print_with_space(it.visibility(tcx), it.item_id, cx),
             unsafety = t.unsafety(tcx).print_with_space(),
             is_auto = if t.is_auto(tcx) { "auto " } else { "" },
@@ -957,6 +954,21 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
     let cloned_shared = Rc::clone(&cx.shared);
     let cache = &cloned_shared.cache;
     let mut extern_crates = FxHashSet::default();
+
+    if !t.is_object_safe(cx.tcx()) {
+        write_small_section_header(
+            w,
+            "object-safety",
+            "Object Safety",
+            &format!(
+                "
This trait is not \
+                \
+                object safe.
",
+                base = crate::clean::utils::DOC_RUST_LANG_ORG_CHANNEL
+            ),
+        );
+    }
+
     if let Some(implementors) = cache.implementors.get(&it.item_id.expect_def_id()) {
         // The DefId is for the first Type found with that name. The bool is
         // if any Types with the same name but different DefId have been found.
@@ -979,7 +991,7 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
             }
         }
 
-        let (local, foreign) =
+        let (local, mut foreign) =
             implementors.iter().partition::, _>(|i| i.is_on_local_type(cx));
 
         let (mut synthetic, mut concrete): (Vec<&&Impl>, Vec<&&Impl>) =
@@ -987,6 +999,7 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
 
         synthetic.sort_by_cached_key(|i| ImplString::new(i, cx));
         concrete.sort_by_cached_key(|i| ImplString::new(i, cx));
+        foreign.sort_by_cached_key(|i| ImplString::new(i, cx));
 
         if !foreign.is_empty() {
             write_small_section_header(w, "foreign-impls", "Implementations on Foreign Types", "");
@@ -1064,6 +1077,8 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
         }
     }
 
+    // [RUSTDOCIMPL] trait.impl
+    //
     // Include implementors in crates that depend on the current crate.
     //
     // This is complicated by the way rustdoc is invoked, which is basically
@@ -1099,7 +1114,7 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
     // ```
     //
     // Basically, we want `C::Baz` and `A::Foo` to show the same set of
-    // impls, which is easier if they both treat `/implementors/A/trait.Foo.js`
+    // impls, which is easier if they both treat `/trait.impl/A/trait.Foo.js`
     // as the Single Source of Truth.
     //
     // We also want the `impl Baz for Quux` to be written to
@@ -1108,7 +1123,7 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
     // because that'll load faster, and it's better for SEO. And we don't want
     // the same impl to show up twice on the same page.
     //
-    // To make this work, the implementors JS file has a structure kinda
+    // To make this work, the trait.impl/A/trait.Foo.js JS file has a structure kinda
     // like this:
     //
     // ```js
@@ -1125,7 +1140,7 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
     // So C's HTML will have something like this:
     //
     // ```html
-    // 
     // ```
     //
@@ -1135,7 +1150,7 @@ fn item_trait(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &clean:
     // [JSONP]: https://en.wikipedia.org/wiki/JSONP
     let mut js_src_path: UrlPartsBuilder = std::iter::repeat("..")
         .take(cx.current.len())
-        .chain(std::iter::once("implementors"))
+        .chain(std::iter::once("trait.impl"))
         .collect();
     if let Some(did) = it.item_id.as_def_id() &&
         let get_extern = { || cache.external_paths.get(&did).map(|s| &s.0) } &&
@@ -1170,7 +1185,7 @@ fn item_trait_alias(
         write!(
             w,
             "{attrs}trait {name}{generics}{where_b} = {bounds};",
-            attrs = render_attributes_in_pre(it, "", cx.tcx()),
+            attrs = render_attributes_in_pre(it, "", cx),
             name = it.name.unwrap(),
             generics = t.generics.print(cx),
             where_b = print_where_clause(&t.generics, cx, 0, Ending::Newline),
@@ -1198,7 +1213,7 @@ fn item_opaque_ty(
         write!(
             w,
             "{attrs}type {name}{generics}{where_clause} = impl {bounds};",
-            attrs = render_attributes_in_pre(it, "", cx.tcx()),
+            attrs = render_attributes_in_pre(it, "", cx),
             name = it.name.unwrap(),
             generics = t.generics.print(cx),
             where_clause = print_where_clause(&t.generics, cx, 0, Ending::Newline),
@@ -1223,7 +1238,7 @@ fn item_type_alias(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &c
             write!(
                 w,
                 "{attrs}{vis}type {name}{generics}{where_clause} = {type_};",
-                attrs = render_attributes_in_pre(it, "", cx.tcx()),
+                attrs = render_attributes_in_pre(it, "", cx),
                 vis = visibility_print_with_space(it.visibility(cx.tcx()), it.item_id, cx),
                 name = it.name.unwrap(),
                 generics = t.generics.print(cx),
@@ -1247,6 +1262,9 @@ fn item_type_alias(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &c
         match inner_type {
             clean::TypeAliasInnerType::Enum { variants, is_non_exhaustive } => {
                 let variants_iter = || variants.iter().filter(|i| !i.is_stripped());
+                let ty = cx.tcx().type_of(it.def_id().unwrap()).instantiate_identity();
+                let enum_def_id = ty.ty_adt_id().unwrap();
+
                 wrap_item(w, |w| {
                     let variants_len = variants.len();
                     let variants_count = variants_iter().count();
@@ -1257,13 +1275,14 @@ fn item_type_alias(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &c
                         w,
                         cx,
                         Some(&t.generics),
-                        variants_iter(),
+                        &variants,
                         variants_count,
                         has_stripped_entries,
                         *is_non_exhaustive,
+                        enum_def_id,
                     )
                 });
-                item_variants(w, cx, it, variants_iter());
+                item_variants(w, cx, it, &variants, enum_def_id);
             }
             clean::TypeAliasInnerType::Union { fields } => {
                 wrap_item(w, |w| {
@@ -1313,6 +1332,102 @@ fn item_type_alias(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, t: &c
     // we need #14072 to make sense of the generics.
     write!(w, "{}", render_assoc_items(cx, it, def_id, AssocItemRender::All));
     write!(w, "{}", document_type_layout(cx, def_id));
+
+    // [RUSTDOCIMPL] type.impl
+    //
+    // Include type definitions from the alias target type.
+    //
+    // Earlier versions of this code worked by having `render_assoc_items`
+    // include this data directly. That generates *O*`(types*impls)` of HTML
+    // text, and some real crates have a lot of types and impls.
+    //
+    // To create the same UX without generating half a gigabyte of HTML for a
+    // crate that only contains 20 megabytes of actual documentation[^115718],
+    // rustdoc stashes these type-alias-inlined docs in a [JSONP]
+    // "database-lite". The file itself is generated in `write_shared.rs`,
+    // and hooks into functions provided by `main.js`.
+    //
+    // The format of `trait.impl` and `type.impl` JS files are superficially
+    // similar. Each line, except the JSONP wrapper itself, belongs to a crate,
+    // and they are otherwise separate (rustdoc should be idempotent). The
+    // "meat" of the file is HTML strings, so the frontend code is very simple.
+    // Links are relative to the doc root, though, so the frontend needs to fix
+    // that up, and inlined docs can reuse these files.
+    //
+    // However, there are a few differences, caused by the sophisticated
+    // features that type aliases have. Consider this crate graph:
+    //
+    // ```text
+    //  ---------------------------------
+    //  | crate A: struct Foo        |
+    //  |          type Bar = Foo  |
+    //  |          impl X for Foo   |
+    //  |          impl Y for Foo  |
+    //  ---------------------------------
+    //      |
+    //  ----------------------------------
+    //  | crate B: type Baz = A::Foo |
+    //  |          type Xyy = A::Foo |
+    //  |          impl Z for Xyy        |
+    //  ----------------------------------
+    // ```
+    //
+    // The type.impl/A/struct.Foo.js JS file has a structure kinda like this:
+    //
+    // ```js
+    // JSONP({
+    // "A": [["impl Y for Foo", "Y", "A::Bar"]],
+    // "B": [["impl X for Foo", "X", "B::Baz", "B::Xyy"], ["impl Z for Xyy", "Z", "B::Baz"]],
+    // });
+    // ```
+    //
+    // When the type.impl file is loaded, only the current crate's docs are
+    // actually used. The main reason to bundle them together is that there's
+    // enough duplication in them for DEFLATE to remove the redundancy.
+    //
+    // The contents of a crate are a list of impl blocks, themselves
+    // represented as lists. The first item in the sublist is the HTML block,
+    // the second item is the name of the trait (which goes in the sidebar),
+    // and all others are the names of type aliases that successfully match.
+    //
+    // This way:
+    //
+    // - There's no need to generate these files for types that have no aliases
+    //   in the current crate. If a dependent crate makes a type alias, it'll
+    //   take care of generating its own docs.
+    // - There's no need to reimplement parts of the type checker in
+    //   JavaScript. The Rust backend does the checking, and includes its
+    //   results in the file.
+    // - Docs defined directly on the type alias are dropped directly in the
+    //   HTML by `render_assoc_items`, and are accessible without JavaScript.
+    //   The JSONP file will not list impl items that are known to be part
+    //   of the main HTML file already.
+    //
+    // [JSONP]: https://en.wikipedia.org/wiki/JSONP
+    // [^115718]: https://github.com/rust-lang/rust/issues/115718
+    let cloned_shared = Rc::clone(&cx.shared);
+    let cache = &cloned_shared.cache;
+    if let Some(target_did) = t.type_.def_id(cache) &&
+        let get_extern = { || cache.external_paths.get(&target_did) } &&
+        let Some(&(ref target_fqp, target_type)) = cache.paths.get(&target_did).or_else(get_extern) &&
+        target_type.is_adt() && // primitives cannot be inlined
+        let Some(self_did) = it.item_id.as_def_id() &&
+        let get_local = { || cache.paths.get(&self_did).map(|(p, _)| p) } &&
+        let Some(self_fqp) = cache.exact_paths.get(&self_did).or_else(get_local)
+    {
+        let mut js_src_path: UrlPartsBuilder = std::iter::repeat("..")
+            .take(cx.current.len())
+            .chain(std::iter::once("type.impl"))
+            .collect();
+        js_src_path.extend(target_fqp[..target_fqp.len() - 1].iter().copied());
+        js_src_path.push_fmt(format_args!("{target_type}.{}.js", target_fqp.last().unwrap()));
+        let self_path = self_fqp.iter().map(Symbol::as_str).collect::>().join("::");
+        write!(
+            w,
+            "",
+            src = js_src_path.finish(),
+        );
+    }
 }
 
 fn item_union(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, s: &clean::Union) {
@@ -1408,7 +1523,7 @@ fn item_enum(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, e: &clean::
     let tcx = cx.tcx();
     let count_variants = e.variants().count();
     wrap_item(w, |w| {
-        render_attributes_in_code(w, it, tcx);
+        render_attributes_in_code(w, it, cx);
         write!(
             w,
             "{}enum {}{}",
@@ -1416,36 +1531,90 @@ fn item_enum(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, e: &clean::
             it.name.unwrap(),
             e.generics.print(cx),
         );
+
         render_enum_fields(
             w,
             cx,
             Some(&e.generics),
-            e.variants(),
+            &e.variants,
             count_variants,
             e.has_stripped_entries(),
             it.is_non_exhaustive(),
+            it.def_id().unwrap(),
         );
     });
 
     write!(w, "{}", document(cx, it, None, HeadingOffset::H2));
 
     if count_variants != 0 {
-        item_variants(w, cx, it, e.variants());
+        item_variants(w, cx, it, &e.variants, it.def_id().unwrap());
     }
     let def_id = it.item_id.expect_def_id();
     write!(w, "{}", render_assoc_items(cx, it, def_id, AssocItemRender::All));
     write!(w, "{}", document_type_layout(cx, def_id));
 }
 
-fn render_enum_fields<'a>(
+/// It'll return false if any variant is not a C-like variant. Otherwise it'll return true if at
+/// least one of them has an explicit discriminant or if the enum has `#[repr(C)]` or an integer
+/// `repr`.
+fn should_show_enum_discriminant(
+    cx: &Context<'_>,
+    enum_def_id: DefId,
+    variants: &IndexVec,
+) -> bool {
+    let mut has_variants_with_value = false;
+    for variant in variants {
+        if let clean::VariantItem(ref var) = *variant.kind &&
+            matches!(var.kind, clean::VariantKind::CLike)
+        {
+            has_variants_with_value |= var.discriminant.is_some();
+        } else {
+            return false;
+        }
+    }
+    if has_variants_with_value {
+        return true;
+    }
+    let repr = cx.tcx().adt_def(enum_def_id).repr();
+    repr.c() || repr.int.is_some()
+}
+
+fn display_c_like_variant(
+    w: &mut Buffer,
+    cx: &mut Context<'_>,
+    item: &clean::Item,
+    variant: &clean::Variant,
+    index: VariantIdx,
+    should_show_enum_discriminant: bool,
+    enum_def_id: DefId,
+) {
+    let name = item.name.unwrap();
+    if let Some(ref value) = variant.discriminant {
+        write!(w, "{} = {}", name.as_str(), value.value(cx.tcx(), true));
+    } else if should_show_enum_discriminant {
+        let adt_def = cx.tcx().adt_def(enum_def_id);
+        let discr = adt_def.discriminant_for_variant(cx.tcx(), index);
+        if discr.ty.is_signed() {
+            write!(w, "{} = {}", name.as_str(), discr.val as i128);
+        } else {
+            write!(w, "{} = {}", name.as_str(), discr.val);
+        }
+    } else {
+        w.write_str(name.as_str());
+    }
+}
+
+fn render_enum_fields(
     mut w: &mut Buffer,
     cx: &mut Context<'_>,
     g: Option<&clean::Generics>,
-    variants: impl Iterator,
+    variants: &IndexVec,
     count_variants: usize,
     has_stripped_entries: bool,
     is_non_exhaustive: bool,
+    enum_def_id: DefId,
 ) {
+    let should_show_enum_discriminant = should_show_enum_discriminant(cx, enum_def_id, variants);
     if !g.is_some_and(|g| print_where_clause_and_check(w, g, cx)) {
         // If there wasn't a `where` clause, we add a whitespace.
         w.write_str(" ");
@@ -1461,15 +1630,24 @@ fn render_enum_fields<'a>(
             toggle_open(&mut w, format_args!("{count_variants} variants"));
         }
         const TAB: &str = "    ";
-        for v in variants {
+        for (index, v) in variants.iter_enumerated() {
+            if v.is_stripped() {
+                continue;
+            }
             w.write_str(TAB);
-            let name = v.name.unwrap();
             match *v.kind {
-                // FIXME(#101337): Show discriminant
                 clean::VariantItem(ref var) => match var.kind {
-                    clean::VariantKind::CLike => w.write_str(name.as_str()),
+                    clean::VariantKind::CLike => display_c_like_variant(
+                        w,
+                        cx,
+                        v,
+                        var,
+                        index,
+                        should_show_enum_discriminant,
+                        enum_def_id,
+                    ),
                     clean::VariantKind::Tuple(ref s) => {
-                        write!(w, "{name}({})", print_tuple_struct_fields(cx, s),);
+                        write!(w, "{}({})", v.name.unwrap(), print_tuple_struct_fields(cx, s));
                     }
                     clean::VariantKind::Struct(ref s) => {
                         render_struct(w, v, None, None, &s.fields, TAB, false, cx);
@@ -1490,11 +1668,12 @@ fn render_enum_fields<'a>(
     }
 }
 
-fn item_variants<'a>(
+fn item_variants(
     w: &mut Buffer,
     cx: &mut Context<'_>,
     it: &clean::Item,
-    variants: impl Iterator,
+    variants: &IndexVec,
+    enum_def_id: DefId,
 ) {
     let tcx = cx.tcx();
     write!(
@@ -1507,7 +1686,11 @@ fn item_variants<'a>(
         document_non_exhaustive_header(it),
         document_non_exhaustive(it)
     );
-    for variant in variants {
+    let should_show_enum_discriminant = should_show_enum_discriminant(cx, enum_def_id, variants);
+    for (index, variant) in variants.iter_enumerated() {
+        if variant.is_stripped() {
+            continue;
+        }
         let id = cx.derive_id(format!("{}.{}", ItemType::Variant, variant.name.unwrap()));
         write!(
             w,
@@ -1522,7 +1705,22 @@ fn item_variants<'a>(
             it.const_stable_since(tcx),
             " rightside",
         );
-        write!(w, "
{name}", name = variant.name.unwrap());
+        w.write_str("
");
+        if let clean::VariantItem(ref var) = *variant.kind &&
+            let clean::VariantKind::CLike = var.kind
+        {
+            display_c_like_variant(
+                w,
+                cx,
+                variant,
+                var,
+                index,
+                should_show_enum_discriminant,
+                enum_def_id,
+            );
+        } else {
+            w.write_str(variant.name.unwrap().as_str());
+        }
 
         let clean::VariantItem(variant_data) = &*variant.kind else { unreachable!() };
 
@@ -1644,7 +1842,7 @@ fn item_primitive(w: &mut impl fmt::Write, cx: &mut Context<'_>, it: &clean::Ite
 fn item_constant(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, c: &clean::Constant) {
     wrap_item(w, |w| {
         let tcx = cx.tcx();
-        render_attributes_in_code(w, it, tcx);
+        render_attributes_in_code(w, it, cx);
 
         write!(
             w,
@@ -1693,7 +1891,7 @@ fn item_constant(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, c: &cle
 
 fn item_struct(w: &mut Buffer, cx: &mut Context<'_>, it: &clean::Item, s: &clean::Struct) {
     wrap_item(w, |w| {
-        render_attributes_in_code(w, it, cx.tcx());
+        render_attributes_in_code(w, it, cx);
         render_struct(w, it, Some(&s.generics), s.ctor_kind, &s.fields, "", true, cx);
     });
 
@@ -1753,7 +1951,7 @@ fn item_fields(
 
 fn item_static(w: &mut impl fmt::Write, cx: &mut Context<'_>, it: &clean::Item, s: &clean::Static) {
     wrap_item(w, |buffer| {
-        render_attributes_in_code(buffer, it, cx.tcx());
+        render_attributes_in_code(buffer, it, cx);
         write!(
             buffer,
             "{vis}static {mutability}{name}: {typ}",
@@ -1771,7 +1969,7 @@ fn item_static(w: &mut impl fmt::Write, cx: &mut Context<'_>, it: &clean::Item,
 fn item_foreign_type(w: &mut impl fmt::Write, cx: &mut Context<'_>, it: &clean::Item) {
     wrap_item(w, |buffer| {
         buffer.write_str("extern {\n").unwrap();
-        render_attributes_in_code(buffer, it, cx.tcx());
+        render_attributes_in_code(buffer, it, cx);
         write!(
             buffer,
             "    {}type {};\n}}",
diff --git a/src/librustdoc/html/render/search_index.rs b/src/librustdoc/html/render/search_index.rs
index 78c443b22..af1dab594 100644
--- a/src/librustdoc/html/render/search_index.rs
+++ b/src/librustdoc/html/render/search_index.rs
@@ -12,7 +12,7 @@ use crate::formats::cache::{Cache, OrphanImplItem};
 use crate::formats::item_type::ItemType;
 use crate::html::format::join_with_double_colon;
 use crate::html::markdown::short_markdown_summary;
-use crate::html::render::{IndexItem, IndexItemFunctionType, RenderType, RenderTypeId};
+use crate::html::render::{self, IndexItem, IndexItemFunctionType, RenderType, RenderTypeId};
 
 /// Builds the search index from the collected metadata
 pub(crate) fn build_index<'tcx>(
@@ -26,7 +26,8 @@ pub(crate) fn build_index<'tcx>(
 
     // Attach all orphan items to the type's definition if the type
     // has since been learned.
-    for &OrphanImplItem { parent, ref item, ref impl_generics } in &cache.orphan_impl_items {
+    for &OrphanImplItem { impl_id, parent, ref item, ref impl_generics } in &cache.orphan_impl_items
+    {
         if let Some((fqp, _)) = cache.paths.get(&parent) {
             let desc = short_markdown_summary(&item.doc_value(), &item.link_names(cache));
             cache.search_index.push(IndexItem {
@@ -36,6 +37,7 @@ pub(crate) fn build_index<'tcx>(
                 desc,
                 parent: Some(parent),
                 parent_idx: None,
+                impl_id,
                 search_type: get_function_type_for_search(item, tcx, impl_generics.as_ref(), cache),
                 aliases: item.attrs.get_doc_aliases(),
                 deprecation: item.deprecation(tcx),
@@ -222,6 +224,29 @@ pub(crate) fn build_index<'tcx>(
         })
         .collect();
 
+    // Find associated items that need disambiguators
+    let mut associated_item_duplicates = FxHashMap::<(isize, ItemType, Symbol), usize>::default();
+
+    for &item in &crate_items {
+        if item.impl_id.is_some() && let Some(parent_idx) = item.parent_idx {
+            let count = associated_item_duplicates
+                .entry((parent_idx, item.ty, item.name))
+                .or_insert(0);
+            *count += 1;
+        }
+    }
+
+    let associated_item_disambiguators = crate_items
+        .iter()
+        .enumerate()
+        .filter_map(|(index, item)| {
+            let impl_id = ItemId::DefId(item.impl_id?);
+            let parent_idx = item.parent_idx?;
+            let count = *associated_item_duplicates.get(&(parent_idx, item.ty, item.name))?;
+            if count > 1 { Some((index, render::get_id_for_impl(tcx, impl_id))) } else { None }
+        })
+        .collect::>();
+
     struct CrateData<'a> {
         doc: String,
         items: Vec<&'a IndexItem>,
@@ -230,6 +255,8 @@ pub(crate) fn build_index<'tcx>(
         //
         // To be noted: the `usize` elements are indexes to `items`.
         aliases: &'a BTreeMap>,
+        // Used when a type has more than one impl with an associated item with the same name.
+        associated_item_disambiguators: &'a Vec<(usize, String)>,
     }
 
     struct Paths {
@@ -382,6 +409,7 @@ pub(crate) fn build_index<'tcx>(
             crate_data.serialize_field("f", &functions)?;
             crate_data.serialize_field("c", &deprecated)?;
             crate_data.serialize_field("p", &paths)?;
+            crate_data.serialize_field("b", &self.associated_item_disambiguators)?;
             if has_aliases {
                 crate_data.serialize_field("a", &self.aliases)?;
             }
@@ -398,6 +426,7 @@ pub(crate) fn build_index<'tcx>(
             items: crate_items,
             paths: crate_paths,
             aliases: &aliases,
+            associated_item_disambiguators: &associated_item_disambiguators,
         })
         .expect("failed serde conversion")
         // All these `replace` calls are because we have to go through JS string for JSON content.
diff --git a/src/librustdoc/html/render/sidebar.rs b/src/librustdoc/html/render/sidebar.rs
index 76f63c6f6..ba4aaaff5 100644
--- a/src/librustdoc/html/render/sidebar.rs
+++ b/src/librustdoc/html/render/sidebar.rs
@@ -19,7 +19,7 @@ pub(super) struct Sidebar<'a> {
     pub(super) title_prefix: &'static str,
     pub(super) title: &'a str,
     pub(super) is_crate: bool,
-    pub(super) version: &'a str,
+    pub(super) is_mod: bool,
     pub(super) blocks: Vec>,
     pub(super) path: String,
 }
@@ -38,18 +38,19 @@ pub(crate) struct LinkBlock<'a> {
     /// as well as the link to it, e.g. `#implementations`.
     /// Will be rendered inside an `
` tag
     heading: Link<'a>,
+    class: &'static str,
     links: Vec>,
     /// Render the heading even if there are no links
     force_render: bool,
 }
 
 impl<'a> LinkBlock<'a> {
-    pub fn new(heading: Link<'a>, links: Vec>) -> Self {
-        Self { heading, links, force_render: false }
+    pub fn new(heading: Link<'a>, class: &'static str, links: Vec>) -> Self {
+        Self { heading, links, class, force_render: false }
     }
 
-    pub fn forced(heading: Link<'a>) -> Self {
-        Self { heading, links: vec![], force_render: true }
+    pub fn forced(heading: Link<'a>, class: &'static str) -> Self {
+        Self { heading, links: vec![], class, force_render: true }
     }
 
     pub fn should_render(&self) -> bool {
@@ -99,12 +100,12 @@ pub(super) fn print_sidebar(cx: &Context<'_>, it: &clean::Item, buffer: &mut Buf
         || it.is_primitive()
         || it.is_union()
         || it.is_enum()
-        || it.is_mod()
+        // crate title is displayed as part of logo lockup
+        || (it.is_mod() && !it.is_crate())
         || it.is_type_alias()
     {
         (
             match *it.kind {
-                clean::ModuleItem(..) if it.is_crate() => "Crate ",
                 clean::ModuleItem(..) => "Module ",
                 _ => "",
             },
@@ -113,14 +114,22 @@ pub(super) fn print_sidebar(cx: &Context<'_>, it: &clean::Item, buffer: &mut Buf
     } else {
         ("", "")
     };
-    let version =
-        if it.is_crate() { cx.cache().crate_version.as_deref().unwrap_or_default() } else { "" };
-    let path: String = if !it.is_mod() {
-        cx.current.iter().map(|s| s.as_str()).intersperse("::").collect()
+    // need to show parent path header if:
+    //   - it's a child module, instead of the crate root
+    //   - there's a sidebar section for the item itself
+    //
+    // otherwise, the parent path header is redundant with the big crate
+    // branding area at the top of the sidebar
+    let sidebar_path =
+        if it.is_mod() { &cx.current[..cx.current.len() - 1] } else { &cx.current[..] };
+    let path: String = if sidebar_path.len() > 1 || !title.is_empty() {
+        let path = sidebar_path.iter().map(|s| s.as_str()).intersperse("::").collect();
+        if sidebar_path.len() == 1 { format!("crate {path}") } else { path }
     } else {
         "".into()
     };
-    let sidebar = Sidebar { title_prefix, title, is_crate: it.is_crate(), version, blocks, path };
+    let sidebar =
+        Sidebar { title_prefix, title, is_mod: it.is_mod(), is_crate: it.is_crate(), blocks, path };
     sidebar.render_into(buffer).unwrap();
 }
 
@@ -149,7 +158,7 @@ fn sidebar_struct<'a>(
     };
     let mut items = vec![];
     if let Some(name) = field_name {
-        items.push(LinkBlock::new(Link::new("fields", name), fields));
+        items.push(LinkBlock::new(Link::new("fields", name), "structfield", fields));
     }
     sidebar_assoc_items(cx, it, &mut items);
     items
@@ -206,12 +215,23 @@ fn sidebar_trait<'a>(
         ("foreign-impls", "Implementations on Foreign Types", foreign_impls),
     ]
     .into_iter()
-    .map(|(id, title, items)| LinkBlock::new(Link::new(id, title), items))
+    .map(|(id, title, items)| LinkBlock::new(Link::new(id, title), "", items))
     .collect();
     sidebar_assoc_items(cx, it, &mut blocks);
-    blocks.push(LinkBlock::forced(Link::new("implementors", "Implementors")));
+
+    if !t.is_object_safe(cx.tcx()) {
+        blocks.push(LinkBlock::forced(
+            Link::new("object-safety", "Object Safety"),
+            "object-safety-note",
+        ));
+    }
+
+    blocks.push(LinkBlock::forced(Link::new("implementors", "Implementors"), "impl"));
     if t.is_auto(cx.tcx()) {
-        blocks.push(LinkBlock::forced(Link::new("synthetic-implementors", "Auto Implementors")));
+        blocks.push(LinkBlock::forced(
+            Link::new("synthetic-implementors", "Auto Implementors"),
+            "impl-auto",
+        ));
     }
     blocks
 }
@@ -237,7 +257,7 @@ fn sidebar_type_alias<'a>(
 ) -> Vec> {
     let mut items = vec![];
     if let Some(inner_type) = &t.inner_type {
-        items.push(LinkBlock::forced(Link::new("aliased-type", "Aliased type")));
+        items.push(LinkBlock::forced(Link::new("aliased-type", "Aliased type"), "type"));
         match inner_type {
             clean::TypeAliasInnerType::Enum { variants, is_non_exhaustive: _ } => {
                 let mut variants = variants
@@ -248,12 +268,12 @@ fn sidebar_type_alias<'a>(
                     .collect::>();
                 variants.sort_unstable();
 
-                items.push(LinkBlock::new(Link::new("variants", "Variants"), variants));
+                items.push(LinkBlock::new(Link::new("variants", "Variants"), "variant", variants));
             }
             clean::TypeAliasInnerType::Union { fields }
             | clean::TypeAliasInnerType::Struct { ctor_kind: _, fields } => {
                 let fields = get_struct_fields_name(fields);
-                items.push(LinkBlock::new(Link::new("fields", "Fields"), fields));
+                items.push(LinkBlock::new(Link::new("fields", "Fields"), "field", fields));
             }
         }
     }
@@ -267,7 +287,7 @@ fn sidebar_union<'a>(
     u: &'a clean::Union,
 ) -> Vec> {
     let fields = get_struct_fields_name(&u.fields);
-    let mut items = vec![LinkBlock::new(Link::new("fields", "Fields"), fields)];
+    let mut items = vec![LinkBlock::new(Link::new("fields", "Fields"), "structfield", fields)];
     sidebar_assoc_items(cx, it, &mut items);
     items
 }
@@ -279,12 +299,11 @@ fn sidebar_assoc_items<'a>(
     links: &mut Vec>,
 ) {
     let did = it.item_id.expect_def_id();
-    let v = cx.shared.all_impls_for_item(it, it.item_id.expect_def_id());
-    let v = v.as_slice();
+    let cache = cx.cache();
 
     let mut assoc_consts = Vec::new();
     let mut methods = Vec::new();
-    if !v.is_empty() {
+    if let Some(v) = cache.impls.get(&did) {
         let mut used_links = FxHashSet::default();
         let mut id_map = IdMap::new();
 
@@ -320,7 +339,7 @@ fn sidebar_assoc_items<'a>(
                     cx,
                     &mut deref_methods,
                     impl_,
-                    v.iter().copied(),
+                    v,
                     &mut derefs,
                     &mut used_links,
                 );
@@ -333,12 +352,16 @@ fn sidebar_assoc_items<'a>(
 
             sidebar_render_assoc_items(cx, &mut id_map, concrete, synthetic, blanket_impl)
         } else {
-            std::array::from_fn(|_| LinkBlock::new(Link::empty(), vec![]))
+            std::array::from_fn(|_| LinkBlock::new(Link::empty(), "", vec![]))
         };
 
         let mut blocks = vec![
-            LinkBlock::new(Link::new("implementations", "Associated Constants"), assoc_consts),
-            LinkBlock::new(Link::new("implementations", "Methods"), methods),
+            LinkBlock::new(
+                Link::new("implementations", "Associated Constants"),
+                "associatedconstant",
+                assoc_consts,
+            ),
+            LinkBlock::new(Link::new("implementations", "Methods"), "method", methods),
         ];
         blocks.append(&mut deref_methods);
         blocks.extend([concrete, synthetic, blanket]);
@@ -350,7 +373,7 @@ fn sidebar_deref_methods<'a>(
     cx: &'a Context<'_>,
     out: &mut Vec>,
     impl_: &Impl,
-    v: impl Iterator,
+    v: &[Impl],
     derefs: &mut DefIdSet,
     used_links: &mut FxHashSet,
 ) {
@@ -375,7 +398,7 @@ fn sidebar_deref_methods<'a>(
             // Avoid infinite cycles
             return;
         }
-        let deref_mut = { v }.any(|i| i.trait_did() == cx.tcx().lang_items().deref_mut_trait());
+        let deref_mut = v.iter().any(|i| i.trait_did() == cx.tcx().lang_items().deref_mut_trait());
         let inner_impl = target
             .def_id(c)
             .or_else(|| {
@@ -407,7 +430,7 @@ fn sidebar_deref_methods<'a>(
                 );
                 // We want links' order to be reproducible so we don't use unstable sort.
                 ret.sort();
-                out.push(LinkBlock::new(Link::new(id, title), ret));
+                out.push(LinkBlock::new(Link::new(id, title), "deref-methods", ret));
             }
         }
 
@@ -426,7 +449,7 @@ fn sidebar_deref_methods<'a>(
                 cx,
                 out,
                 target_deref_impl,
-                target_impls.iter(),
+                target_impls,
                 derefs,
                 used_links,
             );
@@ -446,7 +469,7 @@ fn sidebar_enum<'a>(
         .collect::>();
     variants.sort_unstable();
 
-    let mut items = vec![LinkBlock::new(Link::new("variants", "Variants"), variants)];
+    let mut items = vec![LinkBlock::new(Link::new("variants", "Variants"), "variant", variants)];
     sidebar_assoc_items(cx, it, &mut items);
     items
 }
@@ -460,7 +483,7 @@ pub(crate) fn sidebar_module_like(
         .filter(|sec| item_sections_in_use.contains(sec))
         .map(|sec| Link::new(sec.id(), sec.name()))
         .collect();
-    LinkBlock::new(Link::empty(), item_sections)
+    LinkBlock::new(Link::empty(), "", item_sections)
 }
 
 fn sidebar_module(items: &[clean::Item]) -> LinkBlock<'static> {
@@ -503,8 +526,7 @@ fn sidebar_render_assoc_items(
             .iter()
             .filter_map(|it| {
                 let trait_ = it.inner_impl().trait_.as_ref()?;
-                let encoded =
-                    id_map.derive(super::get_id_for_impl(&it.inner_impl().for_, Some(trait_), cx));
+                let encoded = id_map.derive(super::get_id_for_impl(cx.tcx(), it.impl_item.item_id));
 
                 let prefix = match it.inner_impl().polarity {
                     ty::ImplPolarity::Positive | ty::ImplPolarity::Reservation => "",
@@ -522,12 +544,21 @@ fn sidebar_render_assoc_items(
     let synthetic = format_impls(synthetic, id_map);
     let blanket = format_impls(blanket_impl, id_map);
     [
-        LinkBlock::new(Link::new("trait-implementations", "Trait Implementations"), concrete),
+        LinkBlock::new(
+            Link::new("trait-implementations", "Trait Implementations"),
+            "trait-implementation",
+            concrete,
+        ),
         LinkBlock::new(
             Link::new("synthetic-implementations", "Auto Trait Implementations"),
+            "synthetic-implementation",
             synthetic,
         ),
-        LinkBlock::new(Link::new("blanket-implementations", "Blanket Implementations"), blanket),
+        LinkBlock::new(
+            Link::new("blanket-implementations", "Blanket Implementations"),
+            "blanket-implementation",
+            blanket,
+        ),
     ]
 }
 
diff --git a/src/librustdoc/html/render/write_shared.rs b/src/librustdoc/html/render/write_shared.rs
index e824651e7..d2c7c578c 100644
--- a/src/librustdoc/html/render/write_shared.rs
+++ b/src/librustdoc/html/render/write_shared.rs
@@ -5,18 +5,28 @@ use std::io::{self, BufReader};
 use std::path::{Component, Path};
 use std::rc::{Rc, Weak};
 
+use indexmap::IndexMap;
 use itertools::Itertools;
 use rustc_data_structures::flock;
 use rustc_data_structures::fx::{FxHashMap, FxHashSet};
+use rustc_middle::ty::fast_reject::{DeepRejectCtxt, TreatParams};
+use rustc_span::def_id::DefId;
+use rustc_span::Symbol;
 use serde::ser::SerializeSeq;
 use serde::{Serialize, Serializer};
 
 use super::{collect_paths_for_type, ensure_trailing_slash, Context};
-use crate::clean::Crate;
+use crate::clean::{Crate, Item, ItemId, ItemKind};
 use crate::config::{EmitType, RenderOptions};
 use crate::docfs::PathError;
 use crate::error::Error;
+use crate::formats::cache::Cache;
+use crate::formats::item_type::ItemType;
+use crate::formats::{Impl, RenderMode};
+use crate::html::format::Buffer;
+use crate::html::render::{AssocItemLink, ImplRenderingParameters};
 use crate::html::{layout, static_files};
+use crate::visit::DocVisitor;
 use crate::{try_err, try_none};
 
 /// Rustdoc writes out two kinds of shared files:
@@ -336,33 +346,286 @@ if (typeof exports !== 'undefined') {exports.searchIndex = searchIndex};
             let dst = cx.dst.join("index.html");
             let page = layout::Page {
                 title: "Index of crates",
-                css_class: "mod",
+                css_class: "mod sys",
                 root_path: "./",
                 static_root_path: shared.static_root_path.as_deref(),
                 description: "List of crates",
                 resource_suffix: &shared.resource_suffix,
+                rust_logo: true,
             };
 
             let content = format!(
                 "
List of all crates
    {}
",
-                krates
-                    .iter()
-                    .map(|s| {
-                        format!(
-                            "
  • {s}
    • ",
-                            trailing_slash = ensure_trailing_slash(s),
-                        )
-                    })
-                    .collect::()
+                krates.iter().format_with("", |k, f| {
+                    f(&format_args!(
+                        "
  • {k}
    • ",
+                        trailing_slash = ensure_trailing_slash(k),
+                    ))
+                })
             );
             let v = layout::render(&shared.layout, &page, "", content, &shared.style_files);
             shared.fs.write(dst, v)?;
         }
     }
 
+    let cloned_shared = Rc::clone(&cx.shared);
+    let cache = &cloned_shared.cache;
+
+    // Collect the list of aliased types and their aliases.
+    // 
+    //
+    // The clean AST has type aliases that point at their types, but
+    // this visitor works to reverse that: `aliased_types` is a map
+    // from target to the aliases that reference it, and each one
+    // will generate one file.
+    struct TypeImplCollector<'cx, 'cache> {
+        // Map from DefId-of-aliased-type to its data.
+        aliased_types: IndexMap>,
+        visited_aliases: FxHashSet,
+        cache: &'cache Cache,
+        cx: &'cache mut Context<'cx>,
+    }
+    // Data for an aliased type.
+    //
+    // In the final file, the format will be roughly:
+    //
+    // ```json
+    // // type.impl/CRATE/TYPENAME.js
+    // JSONP(
+    // "CRATE": [
+    //   ["IMPL1 HTML", "ALIAS1", "ALIAS2", ...],
+    //   ["IMPL2 HTML", "ALIAS3", "ALIAS4", ...],
+    //    ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ struct AliasedType
+    //   ...
+    // ]
+    // )
+    // ```
+    struct AliasedType<'cache> {
+        // This is used to generate the actual filename of this aliased type.
+        target_fqp: &'cache [Symbol],
+        target_type: ItemType,
+        // This is the data stored inside the file.
+        // ItemId is used to deduplicate impls.
+        impl_: IndexMap>,
+    }
+    // The `impl_` contains data that's used to figure out if an alias will work,
+    // and to generate the HTML at the end.
+    //
+    // The `type_aliases` list is built up with each type alias that matches.
+    struct AliasedTypeImpl<'cache> {
+        impl_: &'cache Impl,
+        type_aliases: Vec<(&'cache [Symbol], Item)>,
+    }
+    impl<'cx, 'cache> DocVisitor for TypeImplCollector<'cx, 'cache> {
+        fn visit_item(&mut self, it: &Item) {
+            self.visit_item_recur(it);
+            let cache = self.cache;
+            let ItemKind::TypeAliasItem(ref t) = *it.kind else { return };
+            let Some(self_did) = it.item_id.as_def_id() else { return };
+            if !self.visited_aliases.insert(self_did) {
+                return;
+            }
+            let Some(target_did) = t.type_.def_id(cache) else { return };
+            let get_extern = { || cache.external_paths.get(&target_did) };
+            let Some(&(ref target_fqp, target_type)) =
+                cache.paths.get(&target_did).or_else(get_extern)
+            else {
+                return;
+            };
+            let aliased_type = self.aliased_types.entry(target_did).or_insert_with(|| {
+                let impl_ = cache
+                    .impls
+                    .get(&target_did)
+                    .map(|v| &v[..])
+                    .unwrap_or_default()
+                    .iter()
+                    .map(|impl_| {
+                        (
+                            impl_.impl_item.item_id,
+                            AliasedTypeImpl { impl_, type_aliases: Vec::new() },
+                        )
+                    })
+                    .collect();
+                AliasedType { target_fqp: &target_fqp[..], target_type, impl_ }
+            });
+            let get_local = { || cache.paths.get(&self_did).map(|(p, _)| p) };
+            let Some(self_fqp) = cache.exact_paths.get(&self_did).or_else(get_local) else {
+                return;
+            };
+            let aliased_ty = self.cx.tcx().type_of(self_did).skip_binder();
+            // Exclude impls that are directly on this type. They're already in the HTML.
+            // Some inlining scenarios can cause there to be two versions of the same
+            // impl: one on the type alias and one on the underlying target type.
+            let mut seen_impls: FxHashSet = cache
+                .impls
+                .get(&self_did)
+                .map(|s| &s[..])
+                .unwrap_or_default()
+                .iter()
+                .map(|i| i.impl_item.item_id)
+                .collect();
+            for (impl_item_id, aliased_type_impl) in &mut aliased_type.impl_ {
+                // Only include this impl if it actually unifies with this alias.
+                // Synthetic impls are not included; those are also included in the HTML.
+                //
+                // FIXME(lazy_type_alias): Once the feature is complete or stable, rewrite this
+                // to use type unification.
+                // Be aware of `tests/rustdoc/type-alias/deeply-nested-112515.rs` which might regress.
+                let Some(impl_did) = impl_item_id.as_def_id() else { continue };
+                let for_ty = self.cx.tcx().type_of(impl_did).skip_binder();
+                let reject_cx =
+                    DeepRejectCtxt { treat_obligation_params: TreatParams::AsCandidateKey };
+                if !reject_cx.types_may_unify(aliased_ty, for_ty) {
+                    continue;
+                }
+                // Avoid duplicates
+                if !seen_impls.insert(*impl_item_id) {
+                    continue;
+                }
+                // This impl was not found in the set of rejected impls
+                aliased_type_impl.type_aliases.push((&self_fqp[..], it.clone()));
+            }
+        }
+    }
+    let mut type_impl_collector = TypeImplCollector {
+        aliased_types: IndexMap::default(),
+        visited_aliases: FxHashSet::default(),
+        cache,
+        cx,
+    };
+    DocVisitor::visit_crate(&mut type_impl_collector, &krate);
+    // Final serialized form of the alias impl
+    struct AliasSerializableImpl {
+        text: String,
+        trait_: Option,
+        aliases: Vec,
+    }
+    impl Serialize for AliasSerializableImpl {
+        fn serialize(&self, serializer: S) -> Result
+        where
+            S: Serializer,
+        {
+            let mut seq = serializer.serialize_seq(None)?;
+            seq.serialize_element(&self.text)?;
+            if let Some(trait_) = &self.trait_ {
+                seq.serialize_element(trait_)?;
+            } else {
+                seq.serialize_element(&0)?;
+            }
+            for type_ in &self.aliases {
+                seq.serialize_element(type_)?;
+            }
+            seq.end()
+        }
+    }
+    let cx = type_impl_collector.cx;
+    let dst = cx.dst.join("type.impl");
+    let aliased_types = type_impl_collector.aliased_types;
+    for aliased_type in aliased_types.values() {
+        let impls = aliased_type
+            .impl_
+            .values()
+            .flat_map(|AliasedTypeImpl { impl_, type_aliases }| {
+                let mut ret = Vec::new();
+                let trait_ = impl_
+                    .inner_impl()
+                    .trait_
+                    .as_ref()
+                    .map(|trait_| format!("{:#}", trait_.print(cx)));
+                // render_impl will filter out "impossible-to-call" methods
+                // to make that functionality work here, it needs to be called with
+                // each type alias, and if it gives a different result, split the impl
+                for &(type_alias_fqp, ref type_alias_item) in type_aliases {
+                    let mut buf = Buffer::html();
+                    cx.id_map = Default::default();
+                    cx.deref_id_map = Default::default();
+                    let target_did = impl_
+                        .inner_impl()
+                        .trait_
+                        .as_ref()
+                        .map(|trait_| trait_.def_id())
+                        .or_else(|| impl_.inner_impl().for_.def_id(cache));
+                    let provided_methods;
+                    let assoc_link = if let Some(target_did) = target_did {
+                        provided_methods = impl_.inner_impl().provided_trait_methods(cx.tcx());
+                        AssocItemLink::GotoSource(ItemId::DefId(target_did), &provided_methods)
+                    } else {
+                        AssocItemLink::Anchor(None)
+                    };
+                    super::render_impl(
+                        &mut buf,
+                        cx,
+                        *impl_,
+                        &type_alias_item,
+                        assoc_link,
+                        RenderMode::Normal,
+                        None,
+                        &[],
+                        ImplRenderingParameters {
+                            show_def_docs: true,
+                            show_default_items: true,
+                            show_non_assoc_items: true,
+                            toggle_open_by_default: true,
+                        },
+                    );
+                    let text = buf.into_inner();
+                    let type_alias_fqp = (*type_alias_fqp).iter().join("::");
+                    if Some(&text) == ret.last().map(|s: &AliasSerializableImpl| &s.text) {
+                        ret.last_mut()
+                            .expect("already established that ret.last() is Some()")
+                            .aliases
+                            .push(type_alias_fqp);
+                    } else {
+                        ret.push(AliasSerializableImpl {
+                            text,
+                            trait_: trait_.clone(),
+                            aliases: vec![type_alias_fqp],
+                        })
+                    }
+                }
+                ret
+            })
+            .collect::>();
+        let impls = format!(
+            r#""{}":{}"#,
+            krate.name(cx.tcx()),
+            serde_json::to_string(&impls).expect("failed serde conversion"),
+        );
+
+        let mut mydst = dst.clone();
+        for part in &aliased_type.target_fqp[..aliased_type.target_fqp.len() - 1] {
+            mydst.push(part.to_string());
+        }
+        cx.shared.ensure_dir(&mydst)?;
+        let aliased_item_type = aliased_type.target_type;
+        mydst.push(&format!(
+            "{aliased_item_type}.{}.js",
+            aliased_type.target_fqp[aliased_type.target_fqp.len() - 1]
+        ));
+
+        let (mut all_impls, _) = try_err!(collect(&mydst, krate.name(cx.tcx()).as_str()), &mydst);
+        all_impls.push(impls);
+        // Sort the implementors by crate so the file will be generated
+        // identically even with rustdoc running in parallel.
+        all_impls.sort();
+
+        let mut v = String::from("(function() {var type_impls = {\n");
+        v.push_str(&all_impls.join(",\n"));
+        v.push_str("\n};");
+        v.push_str(
+            "if (window.register_type_impls) {\
+                 window.register_type_impls(type_impls);\
+             } else {\
+                 window.pending_type_impls = type_impls;\
+             }",
+        );
+        v.push_str("})()");
+        cx.shared.fs.write(mydst, v)?;
+    }
+
     // Update the list of all implementors for traits
-    let dst = cx.dst.join("implementors");
-    let cache = cx.cache();
+    // 
+    let dst = cx.dst.join("trait.impl");
     for (&did, imps) in &cache.implementors {
         // Private modules can leak through to this phase of rustdoc, which
         // could contain implementations for otherwise private types. In some
diff --git a/src/librustdoc/html/sources.rs b/src/librustdoc/html/sources.rs
index 1d6eafe51..ce620c226 100644
--- a/src/librustdoc/html/sources.rs
+++ b/src/librustdoc/html/sources.rs
@@ -1,4 +1,5 @@
 use crate::clean;
+use crate::clean::utils::has_doc_flag;
 use crate::docfs::PathError;
 use crate::error::Error;
 use crate::html::format;
@@ -12,7 +13,7 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet};
 use rustc_hir::def_id::LOCAL_CRATE;
 use rustc_middle::ty::TyCtxt;
 use rustc_session::Session;
-use rustc_span::source_map::FileName;
+use rustc_span::{sym, FileName};
 
 use std::cell::RefCell;
 use std::ffi::OsStr;
@@ -223,7 +224,8 @@ impl SourceCollector<'_, '_> {
         cur.push(&fname);
 
         let title = format!("{} - source", src_fname.to_string_lossy());
-        let desc = format!("Source of the Rust file `{}`.", filename.prefer_remapped());
+        let desc =
+            format!("Source of the Rust file `{}`.", filename.prefer_remapped_unconditionaly());
         let page = layout::Page {
             title: &title,
             css_class: "src",
@@ -231,6 +233,7 @@ impl SourceCollector<'_, '_> {
             static_root_path: shared.static_root_path.as_deref(),
             description: &desc,
             resource_suffix: &shared.resource_suffix,
+            rust_logo: has_doc_flag(self.cx.tcx(), LOCAL_CRATE.as_def_id(), sym::rust_logo),
         };
         let v = layout::render(
             &shared.layout,
diff --git a/src/librustdoc/html/static/css/rustdoc.css b/src/librustdoc/html/static/css/rustdoc.css
index 47f9e6502..9efdcd601 100644
--- a/src/librustdoc/html/static/css/rustdoc.css
+++ b/src/librustdoc/html/static/css/rustdoc.css
@@ -461,19 +461,9 @@ img {
 	display: none !important;
 }
 
-.sidebar .logo-container {
-	margin-top: 10px;
-	margin-bottom: 10px;
-	text-align: center;
-}
-
-.version {
-	overflow-wrap: break-word;
-}
-
 .logo-container > img {
-	height: 100px;
-	width: 100px;
+	height: 48px;
+	width: 48px;
 }
 
 ul.block, .block li {
@@ -502,6 +492,7 @@ ul.block, .block li {
 }
 
 .sidebar-elems,
+.sidebar > .version,
 .sidebar > h2 {
 	padding-left: 24px;
 }
@@ -510,6 +501,8 @@ ul.block, .block li {
 	color: var(--sidebar-link-color);
 }
 .sidebar .current,
+.sidebar .current a,
+.sidebar-crate a.logo-container:hover + h2 a,
 .sidebar a:hover:not(.logo-container) {
 	background-color: var(--sidebar-current-link-background-color);
 }
@@ -524,6 +517,75 @@ ul.block, .block li {
 	overflow: hidden;
 }
 
+.sidebar-crate {
+	display: flex;
+	align-items: center;
+	justify-content: center;
+	/* there's a 10px padding at the top of 
    , and a 4px margin at the
+		top of the search form. To line them up, add them. */
+	margin: 14px 32px 1rem;
+	row-gap: 10px;
+	column-gap: 32px;
+	flex-wrap: wrap;
+}
+
+.sidebar-crate h2 {
+	flex-grow: 1;
+	/* This setup with the margins and row-gap is designed to make flex-wrap
+		work the way we want. If they're in the side-by-side lockup, there
+		should be a 16px margin to the left of the logo (visually the same as
+		the 24px one on everything else, which are not giant circles) and 8px
+		between it and the crate's name and version. When they're line wrapped,
+		the logo needs to have the same margin on both sides of itself (to
+		center properly) and the crate name and version need 24px on their
+		left margin. */
+	margin: 0 -8px;
+	/* To align this with the search bar, it should not be centered, even when
+		the logo is. */
+	align-self: start;
+}
+
+.sidebar-crate .logo-container {
+	/* The logo is expected to have 8px "slop" along its edges, so we can optically
+		center it. */
+	margin: 0 -16px 0 -16px;
+	text-align: center;
+}
+
+.sidebar-crate h2 a {
+	display: block;
+	margin: 0 calc(-24px + 0.25rem) 0 -0.5rem;
+	/* Align the sidebar crate link with the search bar, which have different
+		font sizes.
+
+		|        | font-size | line-height | total line-height | padding-y |     total    |
+		|:-------|----------:|------------:|------------------:|----------:|-------------:|
+		| crate  |  1.375rem |        1.25 |           1.72rem |         x |   2x+1.72rem |
+		| search |      1rem |        1.15 |           1.15rem |       8px | 1.15rem+16px |
+
+		2x + 1.72rem = 1.15rem + 16px
+		2x = 1.15rem + 16px - 1.72rem
+		2x = 16px - 0.57rem
+		x = ( 16px - 0.57rem ) / 2
+	*/
+	padding: calc( ( 16px - 0.57rem ) / 2 ) 0.25rem;
+	padding-left: 0.5rem;
+}
+
+.sidebar-crate h2 .version {
+	display: block;
+	font-weight: normal;
+	font-size: 1rem;
+	overflow-wrap: break-word;
+	/* opposite of the link padding, cut in half again */
+	margin-top: calc( ( -16px + 0.57rem ) / 2 );
+}
+
+.sidebar-crate + .version {
+	margin-top: -1rem;
+	margin-bottom: 1rem;
+}
+
 .mobile-topbar {
 	display: none;
 }
@@ -1045,6 +1107,7 @@ so that we can apply CSS-filters to change the arrow color in themes */
 	white-space: pre-wrap;
 	border-radius: 3px;
 	display: inline;
+	vertical-align: baseline;
 }
 
 .stab.portability > code {
diff --git a/src/librustdoc/html/static/js/main.js b/src/librustdoc/html/static/js/main.js
index eb256455b..7c052606a 100644
--- a/src/librustdoc/html/static/js/main.js
+++ b/src/librustdoc/html/static/js/main.js
@@ -51,9 +51,14 @@ function setMobileTopbar() {
     // but with the current code it's hard to get the right information in the right place.
     const mobileTopbar = document.querySelector(".mobile-topbar");
     const locationTitle = document.querySelector(".sidebar h2.location");
-    if (mobileTopbar && locationTitle) {
+    if (mobileTopbar) {
         const mobileTitle = document.createElement("h2");
-        mobileTitle.innerHTML = locationTitle.innerHTML;
+        mobileTitle.className = "location";
+        if (hasClass(document.body, "crate")) {
+            mobileTitle.innerText = `Crate ${window.currentCrate}`;
+        } else if (locationTitle) {
+            mobileTitle.innerHTML = locationTitle.innerHTML;
+        }
         mobileTopbar.appendChild(mobileTitle);
     }
 }
@@ -354,6 +359,34 @@ function preLoadCss(cssUrl) {
                 expandSection(pageId);
             }
         }
+        if (savedHash.startsWith("impl-")) {
+            // impl-disambiguated links, used by the search engine
+            // format: impl-X[-for-Y]/method.WHATEVER
+            // turn this into method.WHATEVER[-NUMBER]
+            const splitAt = savedHash.indexOf("/");
+            if (splitAt !== -1) {
+                const implId = savedHash.slice(0, splitAt);
+                const assocId = savedHash.slice(splitAt + 1);
+                const implElem = document.getElementById(implId);
+                if (implElem && implElem.parentElement.tagName === "SUMMARY" &&
+                    implElem.parentElement.parentElement.tagName === "DETAILS") {
+                    onEachLazy(implElem.parentElement.parentElement.querySelectorAll(
+                        `[id^="${assocId}"]`),
+                        item => {
+                            const numbered = /([^-]+)-([0-9]+)/.exec(item.id);
+                            if (item.id === assocId || (numbered && numbered[1] === assocId)) {
+                                openParentDetails(item);
+                                item.scrollIntoView();
+                                // Let the section expand itself before trying to highlight
+                                setTimeout(() => {
+                                    window.location.replace("#" + item.id);
+                                }, 0);
+                            }
+                        }
+                    );
+                }
+            }
+        }
     }
 
     function onHashChange(ev) {
@@ -452,22 +485,27 @@ function preLoadCss(cssUrl) {
                 return;
             }
 
+            const modpath = hasClass(document.body, "mod") ? "../" : "";
+
             const h3 = document.createElement("h3");
-            h3.innerHTML = `${longty}`;
+            h3.innerHTML = `${longty}`;
             const ul = document.createElement("ul");
             ul.className = "block " + shortty;
 
             for (const name of filtered) {
                 let path;
                 if (shortty === "mod") {
-                    path = name + "/index.html";
+                    path = `${modpath}${name}/index.html`;
                 } else {
-                    path = shortty + "." + name + ".html";
+                    path = `${modpath}${shortty}.${name}.html`;
+                }
+                let current_page = document.location.href.toString();
+                if (current_page.endsWith("/")) {
+                    current_page += "index.html";
                 }
-                const current_page = document.location.href.split("/").pop();
                 const link = document.createElement("a");
                 link.href = path;
-                if (path === current_page) {
+                if (link.href === current_page) {
                     link.className = "current";
                 }
                 link.textContent = name;
@@ -480,23 +518,38 @@ function preLoadCss(cssUrl) {
         }
 
         if (sidebar) {
+            // keep this synchronized with ItemSection::ALL in html/render/mod.rs
+            // Re-exports aren't shown here, because they don't have child pages
+            //block("reexport", "reexports", "Re-exports");
             block("primitive", "primitives", "Primitive Types");
             block("mod", "modules", "Modules");
             block("macro", "macros", "Macros");
             block("struct", "structs", "Structs");
             block("enum", "enums", "Enums");
-            block("union", "unions", "Unions");
             block("constant", "constants", "Constants");
             block("static", "static", "Statics");
             block("trait", "traits", "Traits");
             block("fn", "functions", "Functions");
             block("type", "types", "Type Aliases");
+            block("union", "unions", "Unions");
+            // No point, because these items don't appear in modules
+            //block("impl", "impls", "Implementations");
+            //block("tymethod", "tymethods", "Type Methods");
+            //block("method", "methods", "Methods");
+            //block("structfield", "fields", "Fields");
+            //block("variant", "variants", "Variants");
+            //block("associatedtype", "associated-types", "Associated Types");
+            //block("associatedconstant", "associated-consts", "Associated Constants");
             block("foreigntype", "foreign-types", "Foreign Types");
             block("keyword", "keywords", "Keywords");
+            block("opaque", "opaque-types", "Opaque Types");
+            block("attr", "attributes", "Attribute Macros");
+            block("derive", "derives", "Derive Macros");
             block("traitalias", "trait-aliases", "Trait Aliases");
         }
     }
 
+    // 
     window.register_implementors = imp => {
         const implementors = document.getElementById("implementors-list");
         const synthetic_implementors = document.getElementById("synthetic-implementors-list");
@@ -563,7 +616,7 @@ function preLoadCss(cssUrl) {
                 onEachLazy(code.getElementsByTagName("a"), elem => {
                     const href = elem.getAttribute("href");
 
-                    if (href && !/^(?:[a-z+]+:)?\/\//.test(href)) {
+                    if (href && !href.startsWith("#") && !/^(?:[a-z+]+:)?\/\//.test(href)) {
                         elem.setAttribute("href", window.rootPath + href);
                     }
                 });
@@ -587,6 +640,216 @@ function preLoadCss(cssUrl) {
         window.register_implementors(window.pending_implementors);
     }
 
+    /**
+     * 
+     *
+     * [RUSTDOCIMPL] type.impl
+     *
+     * This code inlines implementations into the type alias docs at runtime. It's done at
+     * runtime because some crates have many type aliases and many methods, and we don't want
+     * to generate *O*`(types*methods)` HTML text. The data inside is mostly HTML fragments,
+     * wrapped in JSON.
+     *
+     * - It only includes docs generated for the current crate. This function accepts an
+     *   object mapping crate names to the set of impls.
+     *
+     * - It filters down to the set of applicable impls. The Rust type checker is used to
+     *   tag each HTML blob with the set of type aliases that can actually use it, so the
+     *   JS only needs to consult the attached list of type aliases.
+     *
+     * - It renames the ID attributes, to avoid conflicting IDs in the resulting DOM.
+     *
+     * - It adds the necessary items to the sidebar. If it's an inherent impl, that means
+     *   adding methods, associated types, and associated constants. If it's a trait impl,
+     *   that means adding it to the trait impl sidebar list.
+     *
+     * - It adds the HTML block itself. If it's an inherent impl, it goes after the type
+     *   alias's own inherent impls. If it's a trait impl, it goes in the Trait
+     *   Implementations section.
+     *
+     * - After processing all of the impls, it sorts the sidebar items by name.
+     *
+     * @param {{[cratename: string]: Array>}} impl
+     */
+    window.register_type_impls = imp => {
+        if (!imp || !imp[window.currentCrate]) {
+            return;
+        }
+        window.pending_type_impls = null;
+        const idMap = new Map();
+
+        let implementations = document.getElementById("implementations-list");
+        let trait_implementations = document.getElementById("trait-implementations-list");
+        let trait_implementations_header = document.getElementById("trait-implementations");
+
+        // We want to include the current type alias's impls, and no others.
+        const script = document.querySelector("script[data-self-path]");
+        const selfPath = script ? script.getAttribute("data-self-path") : null;
+
+        // These sidebar blocks need filled in, too.
+        const mainContent = document.querySelector("#main-content");
+        const sidebarSection = document.querySelector(".sidebar section");
+        let methods = document.querySelector(".sidebar .block.method");
+        let associatedTypes = document.querySelector(".sidebar .block.associatedtype");
+        let associatedConstants = document.querySelector(".sidebar .block.associatedconstant");
+        let sidebarTraitList = document.querySelector(".sidebar .block.trait-implementation");
+
+        for (const impList of imp[window.currentCrate]) {
+            const types = impList.slice(2);
+            const text = impList[0];
+            const isTrait = impList[1] !== 0;
+            const traitName = impList[1];
+            if (types.indexOf(selfPath) === -1) {
+                continue;
+            }
+            let outputList = isTrait ? trait_implementations : implementations;
+            if (outputList === null) {
+                const outputListName = isTrait ? "Trait Implementations" : "Implementations";
+                const outputListId = isTrait ?
+                    "trait-implementations-list" :
+                    "implementations-list";
+                const outputListHeaderId = isTrait ? "trait-implementations" : "implementations";
+                const outputListHeader = document.createElement("h2");
+                outputListHeader.id = outputListHeaderId;
+                outputListHeader.innerText = outputListName;
+                outputList = document.createElement("div");
+                outputList.id = outputListId;
+                if (isTrait) {
+                    const link = document.createElement("a");
+                    link.href = `#${outputListHeaderId}`;
+                    link.innerText = "Trait Implementations";
+                    const h = document.createElement("h3");
+                    h.appendChild(link);
+                    trait_implementations = outputList;
+                    trait_implementations_header = outputListHeader;
+                    sidebarSection.appendChild(h);
+                    sidebarTraitList = document.createElement("ul");
+                    sidebarTraitList.className = "block trait-implementation";
+                    sidebarSection.appendChild(sidebarTraitList);
+                    mainContent.appendChild(outputListHeader);
+                    mainContent.appendChild(outputList);
+                } else {
+                    implementations = outputList;
+                    if (trait_implementations) {
+                        mainContent.insertBefore(outputListHeader, trait_implementations_header);
+                        mainContent.insertBefore(outputList, trait_implementations_header);
+                    } else {
+                        const mainContent = document.querySelector("#main-content");
+                        mainContent.appendChild(outputListHeader);
+                        mainContent.appendChild(outputList);
+                    }
+                }
+            }
+            const template = document.createElement("template");
+            template.innerHTML = text;
+
+            onEachLazy(template.content.querySelectorAll("a"), elem => {
+                const href = elem.getAttribute("href");
+
+                if (href && !href.startsWith("#") && !/^(?:[a-z+]+:)?\/\//.test(href)) {
+                    elem.setAttribute("href", window.rootPath + href);
+                }
+            });
+            onEachLazy(template.content.querySelectorAll("[id]"), el => {
+                let i = 0;
+                if (idMap.has(el.id)) {
+                    i = idMap.get(el.id);
+                } else if (document.getElementById(el.id)) {
+                    i = 1;
+                    while (document.getElementById(`${el.id}-${2 * i}`)) {
+                        i = 2 * i;
+                    }
+                    while (document.getElementById(`${el.id}-${i}`)) {
+                        i += 1;
+                    }
+                }
+                if (i !== 0) {
+                    const oldHref = `#${el.id}`;
+                    const newHref = `#${el.id}-${i}`;
+                    el.id = `${el.id}-${i}`;
+                    onEachLazy(template.content.querySelectorAll("a[href]"), link => {
+                        if (link.getAttribute("href") === oldHref) {
+                            link.href = newHref;
+                        }
+                    });
+                }
+                idMap.set(el.id, i + 1);
+            });
+            const templateAssocItems = template.content.querySelectorAll("section.tymethod, " +
+                "section.method, section.associatedtype, section.associatedconstant");
+            if (isTrait) {
+                const li = document.createElement("li");
+                const a = document.createElement("a");
+                a.href = `#${template.content.querySelector(".impl").id}`;
+                a.textContent = traitName;
+                li.appendChild(a);
+                sidebarTraitList.append(li);
+            } else {
+                onEachLazy(templateAssocItems, item => {
+                    let block = hasClass(item, "associatedtype") ? associatedTypes : (
+                        hasClass(item, "associatedconstant") ? associatedConstants : (
+                        methods));
+                    if (!block) {
+                        const blockTitle = hasClass(item, "associatedtype") ? "Associated Types" : (
+                            hasClass(item, "associatedconstant") ? "Associated Constants" : (
+                            "Methods"));
+                        const blockClass = hasClass(item, "associatedtype") ? "associatedtype" : (
+                            hasClass(item, "associatedconstant") ? "associatedconstant" : (
+                            "method"));
+                        const blockHeader = document.createElement("h3");
+                        const blockLink = document.createElement("a");
+                        blockLink.href = "#implementations";
+                        blockLink.innerText = blockTitle;
+                        blockHeader.appendChild(blockLink);
+                        block = document.createElement("ul");
+                        block.className = `block ${blockClass}`;
+                        const insertionReference = methods || sidebarTraitList;
+                        if (insertionReference) {
+                            const insertionReferenceH = insertionReference.previousElementSibling;
+                            sidebarSection.insertBefore(blockHeader, insertionReferenceH);
+                            sidebarSection.insertBefore(block, insertionReferenceH);
+                        } else {
+                            sidebarSection.appendChild(blockHeader);
+                            sidebarSection.appendChild(block);
+                        }
+                        if (hasClass(item, "associatedtype")) {
+                            associatedTypes = block;
+                        } else if (hasClass(item, "associatedconstant")) {
+                            associatedConstants = block;
+                        } else {
+                            methods = block;
+                        }
+                    }
+                    const li = document.createElement("li");
+                    const a = document.createElement("a");
+                    a.innerText = item.id.split("-")[0].split(".")[1];
+                    a.href = `#${item.id}`;
+                    li.appendChild(a);
+                    block.appendChild(li);
+                });
+            }
+            outputList.appendChild(template.content);
+        }
+
+        for (const list of [methods, associatedTypes, associatedConstants, sidebarTraitList]) {
+            if (!list) {
+                continue;
+            }
+            const newChildren = Array.prototype.slice.call(list.children);
+            newChildren.sort((a, b) => {
+                const aI = a.innerText;
+                const bI = b.innerText;
+                return aI < bI ? -1 :
+                    aI > bI ? 1 :
+                    0;
+            });
+            list.replaceChildren(...newChildren);
+        }
+    };
+    if (window.pending_type_impls) {
+        window.register_type_impls(window.pending_type_impls);
+    }
+
     function addSidebarCrates() {
         if (!window.ALL_CRATES) {
             return;
diff --git a/src/librustdoc/html/static/js/search.js b/src/librustdoc/html/static/js/search.js
index 2f0cae0a4..48c9a53a2 100644
--- a/src/librustdoc/html/static/js/search.js
+++ b/src/librustdoc/html/static/js/search.js
@@ -1555,7 +1555,7 @@ function initSearch(rawSearchIndex) {
                         return false;
                     }
                 }
-            } else if (fnType.id !== null) {
+            } else {
                 if (queryElem.id === typeNameIdOfArrayOrSlice &&
                     (fnType.id === typeNameIdOfSlice || fnType.id === typeNameIdOfArray)
                 ) {
@@ -1752,6 +1752,7 @@ function initSearch(rawSearchIndex) {
                 type: item.type,
                 is_alias: true,
                 deprecated: item.deprecated,
+                implDisambiguator: item.implDisambiguator,
             };
         }
 
@@ -2218,7 +2219,7 @@ function initSearch(rawSearchIndex) {
             href = ROOT_PATH + name + "/index.html";
         } else if (item.parent !== undefined) {
             const myparent = item.parent;
-            let anchor = "#" + type + "." + name;
+            let anchor = type + "." + name;
             const parentType = itemTypes[myparent.ty];
             let pageType = parentType;
             let pageName = myparent.name;
@@ -2232,16 +2233,19 @@ function initSearch(rawSearchIndex) {
                 const enumName = item.path.substr(enumNameIdx + 2);
                 path = item.path.substr(0, enumNameIdx);
                 displayPath = path + "::" + enumName + "::" + myparent.name + "::";
-                anchor = "#variant." + myparent.name + ".field." + name;
+                anchor = "variant." + myparent.name + ".field." + name;
                 pageType = "enum";
                 pageName = enumName;
             } else {
                 displayPath = path + "::" + myparent.name + "::";
             }
+            if (item.implDisambiguator !== null) {
+                anchor = item.implDisambiguator + "/" + anchor;
+            }
             href = ROOT_PATH + path.replace(/::/g, "/") +
                 "/" + pageType +
                 "." + pageName +
-                ".html" + anchor;
+                ".html#" + anchor;
         } else {
             displayPath = item.path + "::";
             href = ROOT_PATH + item.path.replace(/::/g, "/") +
@@ -2727,6 +2731,10 @@ ${item.displayPath}${name}\
              * Types are also represented as arrays; the first item is an index into the `p`
              * array, while the second is a list of types representing any generic parameters.
              *
+             * b[i] contains an item's impl disambiguator. This is only present if an item
+             * is defined in an impl block and, the impl block's type has more than one associated
+             * item with the same name.
+             *
              * `a` defines aliases with an Array of pairs: [name, offset], where `offset`
              * points into the n/t/d/q/i/f arrays.
              *
@@ -2746,6 +2754,7 @@ ${item.displayPath}${name}\
              *   i: Array,
              *   f: Array,
              *   p: Array,
+             *   b: Array<[Number, String]>,
              *   c: Array
              * }}
              */
@@ -2766,6 +2775,7 @@ ${item.displayPath}${name}\
                 id: id,
                 normalizedName: crate.indexOf("_") === -1 ? crate : crate.replace(/_/g, ""),
                 deprecated: null,
+                implDisambiguator: null,
             };
             id += 1;
             searchIndex.push(crateRow);
@@ -2789,6 +2799,8 @@ ${item.displayPath}${name}\
             const itemFunctionSearchTypes = crateCorpus.f;
             // an array of (Number) indices for the deprecated items
             const deprecatedItems = new Set(crateCorpus.c);
+            // an array of (Number) indices for the deprecated items
+            const implDisambiguator = new Map(crateCorpus.b);
             // an array of [(Number) item type,
             //              (String) name]
             const paths = crateCorpus.p;
@@ -2849,6 +2861,7 @@ ${item.displayPath}${name}\
                     id: id,
                     normalizedName: word.indexOf("_") === -1 ? word : word.replace(/_/g, ""),
                     deprecated: deprecatedItems.has(i),
+                    implDisambiguator: implDisambiguator.has(i) ? implDisambiguator.get(i) : null,
                 };
                 id += 1;
                 searchIndex.push(row);
diff --git a/src/librustdoc/html/templates/page.html b/src/librustdoc/html/templates/page.html
index 579c782be..3f6147bb9 100644
--- a/src/librustdoc/html/templates/page.html
+++ b/src/librustdoc/html/templates/page.html
@@ -10,7 +10,6 @@
      {# #}
      {# #}
      {# #}
-     {# #}
      {# #}
      {# #}
@@ -42,6 +41,8 @@
      {# #}
     {% else if !page.css_class.contains("mod") %}
      {# #}
+    {% else if !page.css_class.contains("sys") %}
+     {# #}
     {% endif %}
      {# #}
     {% if layout.scrape_examples_extension %}
@@ -77,36 +78,51 @@
     {% if page.css_class != "src" %}
     
     {% endif %}
      {# #}
     
     {# #}
         {% if page.css_class != "src" %}
    {% endif %}