Merging upstream version 1.73.0+dfsg1.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

13 files changed, 661 insertions, 23 deletions

diff --git a/src/tools/tidy/config/black.toml b/src/tools/tidy/config/black.toml
new file mode 100644
index 000000000..51a722979
--- /dev/null
+++ b/src/tools/tidy/config/black.toml
@@ -0,0 +1,15 @@
+[tool.black]
+# Ignore all submodules
+extend-exclude = """(\
+    src/doc/nomicon|\
+    src/tools/cargo/|\
+    src/doc/reference/|\
+    src/doc/book/|\
+    src/doc/rust-by-example/|\
+    library/stdarch/|\
+    src/doc/rustc-dev-guide/|\
+    src/doc/edition-guide/|\
+    src/llvm-project/|\
+    src/doc/embedded-book/|\
+    library/backtrace/
+    )"""
diff --git a/src/tools/tidy/config/requirements.in b/src/tools/tidy/config/requirements.in
new file mode 100644
index 000000000..882e09dae
--- /dev/null
+++ b/src/tools/tidy/config/requirements.in
@@ -0,0 +1,10 @@
+# requirements.in This is the source file for our pinned version requirements
+# file "requirements.txt" To regenerate that file, pip-tools is required
+# (`python -m pip install pip-tools`). Once installed, run: `pip-compile
+# --generate-hashes src/tools/tidy/config/requirements.in`
+#
+# Note: this generation step should be run with the oldest supported python
+# version (currently 3.7) to ensure backward compatibility
+
+black==23.3.0
+ruff==0.0.272
diff --git a/src/tools/tidy/config/requirements.txt b/src/tools/tidy/config/requirements.txt
new file mode 100644
index 000000000..9fd617ad6
--- /dev/null
+++ b/src/tools/tidy/config/requirements.txt
@@ -0,0 +1,117 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --generate-hashes src/tools/tidy/config/requirements.in
+#
+black==23.3.0 \
+    --hash=sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5 \
+    --hash=sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915 \
+    --hash=sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326 \
+    --hash=sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940 \
+    --hash=sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b \
+    --hash=sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30 \
+    --hash=sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c \
+    --hash=sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c \
+    --hash=sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab \
+    --hash=sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27 \
+    --hash=sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2 \
+    --hash=sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961 \
+    --hash=sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9 \
+    --hash=sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb \
+    --hash=sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70 \
+    --hash=sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331 \
+    --hash=sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2 \
+    --hash=sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266 \
+    --hash=sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d \
+    --hash=sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6 \
+    --hash=sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b \
+    --hash=sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925 \
+    --hash=sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8 \
+    --hash=sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4 \
+    --hash=sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3
+    # via -r src/tools/tidy/config/requirements.in
+click==8.1.3 \
+    --hash=sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e \
+    --hash=sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48
+    # via black
+importlib-metadata==6.7.0 \
+    --hash=sha256:1aaf550d4f73e5d6783e7acb77aec43d49da8017410afae93822cc9cca98c4d4 \
+    --hash=sha256:cb52082e659e97afc5dac71e79de97d8681de3aa07ff18578330904a9d18e5b5
+    # via click
+mypy-extensions==1.0.0 \
+    --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \
+    --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782
+    # via black
+packaging==23.1 \
+    --hash=sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61 \
+    --hash=sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f
+    # via black
+pathspec==0.11.1 \
+    --hash=sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687 \
+    --hash=sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293
+    # via black
+platformdirs==3.6.0 \
+    --hash=sha256:57e28820ca8094678b807ff529196506d7a21e17156cb1cddb3e74cebce54640 \
+    --hash=sha256:ffa199e3fbab8365778c4a10e1fbf1b9cd50707de826eb304b50e57ec0cc8d38
+    # via black
+ruff==0.0.272 \
+    --hash=sha256:06b8ee4eb8711ab119db51028dd9f5384b44728c23586424fd6e241a5b9c4a3b \
+    --hash=sha256:1609b864a8d7ee75a8c07578bdea0a7db75a144404e75ef3162e0042bfdc100d \
+    --hash=sha256:19643d448f76b1eb8a764719072e9c885968971bfba872e14e7257e08bc2f2b7 \
+    --hash=sha256:273a01dc8c3c4fd4c2af7ea7a67c8d39bb09bce466e640dd170034da75d14cab \
+    --hash=sha256:27b2ea68d2aa69fff1b20b67636b1e3e22a6a39e476c880da1282c3e4bf6ee5a \
+    --hash=sha256:48eccf225615e106341a641f826b15224b8a4240b84269ead62f0afd6d7e2d95 \
+    --hash=sha256:677284430ac539bb23421a2b431b4ebc588097ef3ef918d0e0a8d8ed31fea216 \
+    --hash=sha256:691d72a00a99707a4e0b2846690961157aef7b17b6b884f6b4420a9f25cd39b5 \
+    --hash=sha256:86bc788245361a8148ff98667da938a01e1606b28a45e50ac977b09d3ad2c538 \
+    --hash=sha256:905ff8f3d6206ad56fcd70674453527b9011c8b0dc73ead27618426feff6908e \
+    --hash=sha256:9c4bfb75456a8e1efe14c52fcefb89cfb8f2a0d31ed8d804b82c6cf2dc29c42c \
+    --hash=sha256:a37ec80e238ead2969b746d7d1b6b0d31aa799498e9ba4281ab505b93e1f4b28 \
+    --hash=sha256:ae9b57546e118660175d45d264b87e9b4c19405c75b587b6e4d21e6a17bf4fdf \
+    --hash=sha256:bd2bbe337a3f84958f796c77820d55ac2db1e6753f39d1d1baed44e07f13f96d \
+    --hash=sha256:d5a208f8ef0e51d4746930589f54f9f92f84bb69a7d15b1de34ce80a7681bc00 \
+    --hash=sha256:dc406e5d756d932da95f3af082814d2467943631a587339ee65e5a4f4fbe83eb \
+    --hash=sha256:ee76b4f05fcfff37bd6ac209d1370520d509ea70b5a637bdf0a04d0c99e13dff
+    # via -r src/tools/tidy/config/requirements.in
+tomli==2.0.1 \
+    --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
+    --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
+    # via black
+typed-ast==1.5.4 \
+    --hash=sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2 \
+    --hash=sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1 \
+    --hash=sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6 \
+    --hash=sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62 \
+    --hash=sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac \
+    --hash=sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d \
+    --hash=sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc \
+    --hash=sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2 \
+    --hash=sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97 \
+    --hash=sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35 \
+    --hash=sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6 \
+    --hash=sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1 \
+    --hash=sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4 \
+    --hash=sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c \
+    --hash=sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e \
+    --hash=sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec \
+    --hash=sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f \
+    --hash=sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72 \
+    --hash=sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47 \
+    --hash=sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72 \
+    --hash=sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe \
+    --hash=sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6 \
+    --hash=sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3 \
+    --hash=sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66
+    # via black
+typing-extensions==4.6.3 \
+    --hash=sha256:88a4153d8505aabbb4e13aacb7c486c2b4a33ca3b3f807914a9b4c844c471c26 \
+    --hash=sha256:d91d5919357fe7f681a9f2b5b4cb2a5f1ef0a1e9f59c4d8ff0d3491e05c0ffd5
+    # via
+    #   black
+    #   importlib-metadata
+    #   platformdirs
+zipp==3.15.0 \
+    --hash=sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b \
+    --hash=sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556
+    # via importlib-metadata
diff --git a/src/tools/tidy/config/ruff.toml b/src/tools/tidy/config/ruff.toml
new file mode 100644
index 000000000..cf08c6264
--- /dev/null
+++ b/src/tools/tidy/config/ruff.toml
@@ -0,0 +1,41 @@
+# Configuration for ruff python linter, run as part of tidy external tools
+
+# B (bugbear), E (pycodestyle, standard), EXE (executables) F (flakes, standard)
+# ERM for error messages would be beneficial at some point
+select = ["B", "E", "EXE", "F"]
+
+ignore = [
+    "E501", # line-too-long
+    "F403", # undefined-local-with-import-star
+    "F405", # undefined-local-with-import-star-usage
+]
+
+# lowest possible for ruff
+target-version = "py37"
+
+# Ignore all submodules
+extend-exclude = [
+    "src/doc/nomicon/",
+    "src/tools/cargo/",
+    "src/doc/reference/",
+    "src/doc/book/",
+    "src/doc/rust-by-example/",
+    "library/stdarch/",
+    "src/doc/rustc-dev-guide/",
+    "src/doc/edition-guide/",
+    "src/llvm-project/",
+    "src/doc/embedded-book/",
+    "library/backtrace/",
+    # Hack: CI runs from a subdirectory under the main checkout
+    "../src/doc/nomicon/",
+    "../src/tools/cargo/",
+    "../src/doc/reference/",
+    "../src/doc/book/",
+    "../src/doc/rust-by-example/",
+    "../library/stdarch/",
+    "../src/doc/rustc-dev-guide/",
+    "../src/doc/edition-guide/",
+    "../src/llvm-project/",
+    "../src/doc/embedded-book/",
+    "../library/backtrace/",
+]
diff --git a/src/tools/tidy/src/deps.rs b/src/tools/tidy/src/deps.rs
index ecc84c161..410852b6a 100644
--- a/src/tools/tidy/src/deps.rs
+++ b/src/tools/tidy/src/deps.rs
@@ -40,10 +40,13 @@ const EXCEPTIONS: &[(&str, &str)] = &[
     ("ar_archive_writer", "Apache-2.0 WITH LLVM-exception"), // rustc
     ("colored", "MPL-2.0"),                                  // rustfmt
     ("dissimilar", "Apache-2.0"),                            // rustdoc, rustc_lexer (few tests) via expect-test, (dev deps)
+    ("encoding_rs", "(Apache-2.0 OR MIT) AND BSD-3-Clause"), // opt-dist
     ("fluent-langneg", "Apache-2.0"),                        // rustc (fluent translations)
     ("fortanix-sgx-abi", "MPL-2.0"),                         // libstd but only for `sgx` target. FIXME: this dependency violates the documentation comment above.
     ("instant", "BSD-3-Clause"),                             // rustc_driver/tracing-subscriber/parking_lot
     ("mdbook", "MPL-2.0"),                                   // mdbook
+    ("openssl", "Apache-2.0"),                               // opt-dist
+    ("rustc_apfloat", "Apache-2.0 WITH LLVM-exception"),     // rustc (license is the same as LLVM uses)
     ("ryu", "Apache-2.0 OR BSL-1.0"),                        // cargo/... (because of serde)
     ("self_cell", "Apache-2.0"),                             // rustc (fluent translations)
     ("snap", "BSD-3-Clause"),                                // rustc
@@ -54,6 +57,9 @@ const EXCEPTIONS_CARGO: &[(&str, &str)] = &[
     // tidy-alphabetical-start
     ("bitmaps", "MPL-2.0+"),
     ("bytesize", "Apache-2.0"),
+    ("ciborium", "Apache-2.0"),
+    ("ciborium-io", "Apache-2.0"),
+    ("ciborium-ll", "Apache-2.0"),
     ("dunce", "CC0-1.0 OR MIT-0 OR Apache-2.0"),
     ("fiat-crypto", "MIT OR Apache-2.0 OR BSD-1-Clause"),
     ("im-rc", "MPL-2.0+"),
@@ -107,7 +113,6 @@ const PERMITTED_DEPS_LOCATION: &str = concat!(file!(), ":", line!());
 /// rustc. Please check with the compiler team before adding an entry.
 const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     // tidy-alphabetical-start
-    "addr2line",
     "adler",
     "ahash",
     "aho-corasick",
@@ -132,8 +137,12 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "crossbeam-utils",
     "crypto-common",
     "cstr",
+    "darling",
+    "darling_core",
+    "darling_macro",
     "datafrog",
     "derive_more",
+    "derive_setters",
     "digest",
     "displaydoc",
     "dissimilar",
@@ -142,6 +151,8 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "elsa",
     "ena",
     "equivalent",
+    "errno",
+    "errno-dragonfly",
     "expect-test",
     "fallible-iterator", // dependency of `thorin`
     "fastrand",
@@ -150,6 +161,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "fluent-bundle",
     "fluent-langneg",
     "fluent-syntax",
+    "fnv",
     "fortanix-sgx-abi",
     "generic-array",
     "getopts",
@@ -163,6 +175,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "icu_provider",
     "icu_provider_adapters",
     "icu_provider_macros",
+    "ident_case",
     "indexmap",
     "instant",
     "intl-memoizer",
@@ -217,6 +230,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "rustc-hash",
     "rustc-rayon",
     "rustc-rayon-core",
+    "rustc_apfloat",
     "rustc_version",
     "rustix",
     "ruzstd", // via object in thorin-dwp
@@ -236,6 +250,7 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "stable_deref_trait",
     "stacker",
     "static_assertions",
+    "strsim",
     "syn",
     "synstructure",
     "tempfile",
@@ -243,9 +258,14 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "termize",
     "thin-vec",
     "thiserror",
+    "thiserror-core",
+    "thiserror-core-impl",
     "thiserror-impl",
     "thorin-dwp",
     "thread_local",
+    "time",
+    "time-core",
+    "time-macros",
     "tinystr",
     "tinyvec",
     "tinyvec_macros",
@@ -258,18 +278,14 @@ const PERMITTED_RUSTC_DEPENDENCIES: &[&str] = &[
     "twox-hash",
     "type-map",
     "typenum",
-    "unic-char-property",
-    "unic-char-range",
-    "unic-common",
-    "unic-emoji-char",
     "unic-langid",
     "unic-langid-impl",
     "unic-langid-macros",
     "unic-langid-macros-impl",
-    "unic-ucd-version",
     "unicase",
     "unicode-ident",
     "unicode-normalization",
+    "unicode-properties",
     "unicode-script",
     "unicode-security",
     "unicode-width",
@@ -324,6 +340,7 @@ const PERMITTED_CRANELIFT_DEPENDENCIES: &[&str] = &[
     "cranelift-native",
     "cranelift-object",
     "crc32fast",
+    "equivalent",
     "fallible-iterator",
     "gimli",
     "hashbrown",
diff --git a/src/tools/tidy/src/ext_tool_checks.rs b/src/tools/tidy/src/ext_tool_checks.rs
new file mode 100644
index 000000000..40e75d1d3
--- /dev/null
+++ b/src/tools/tidy/src/ext_tool_checks.rs
@@ -0,0 +1,435 @@
+//! Optional checks for file types other than Rust source
+//!
+//! Handles python tool version managment via a virtual environment in
+//! `build/venv`.
+//!
+//! # Functional outline
+//!
+//! 1. Run tidy with an extra option: `--extra-checks=py,shell`,
+//!    `--extra-checks=py:lint`, or similar. Optionally provide specific
+//!    configuration after a double dash (`--extra-checks=py -- foo.py`)
+//! 2. Build configuration based on args/environment:
+//!    - Formatters by default are in check only mode
+//!    - If in CI (TIDY_PRINT_DIFF=1 is set), check and print the diff
+//!    - If `--bless` is provided, formatters may run
+//!    - Pass any additional config after the `--`. If no files are specified,
+//!      use a default.
+//! 3. Print the output of the given command. If it fails and `TIDY_PRINT_DIFF`
+//!    is set, rerun the tool to print a suggestion diff (for e.g. CI)
+
+use std::ffi::OsStr;
+use std::fmt;
+use std::fs;
+use std::io;
+use std::path::{Path, PathBuf};
+use std::process::Command;
+
+/// Minimum python revision is 3.7 for ruff
+const MIN_PY_REV: (u32, u32) = (3, 7);
+const MIN_PY_REV_STR: &str = "≥3.7";
+
+/// Path to find the python executable within a virtual environment
+#[cfg(target_os = "windows")]
+const REL_PY_PATH: &[&str] = &["Scripts", "python3.exe"];
+#[cfg(not(target_os = "windows"))]
+const REL_PY_PATH: &[&str] = &["bin", "python3"];
+
+const RUFF_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "ruff.toml"];
+const BLACK_CONFIG_PATH: &[&str] = &["src", "tools", "tidy", "config", "black.toml"];
+/// Location within build directory
+const RUFF_CACH_PATH: &[&str] = &["cache", "ruff_cache"];
+const PIP_REQ_PATH: &[&str] = &["src", "tools", "tidy", "config", "requirements.txt"];
+
+pub fn check(
+    root_path: &Path,
+    outdir: &Path,
+    bless: bool,
+    extra_checks: Option<&str>,
+    pos_args: &[String],
+    bad: &mut bool,
+) {
+    if let Err(e) = check_impl(root_path, outdir, bless, extra_checks, pos_args) {
+        tidy_error!(bad, "{e}");
+    }
+}
+
+fn check_impl(
+    root_path: &Path,
+    outdir: &Path,
+    bless: bool,
+    extra_checks: Option<&str>,
+    pos_args: &[String],
+) -> Result<(), Error> {
+    let show_diff = std::env::var("TIDY_PRINT_DIFF")
+        .map_or(false, |v| v.eq_ignore_ascii_case("true") || v == "1");
+
+    // Split comma-separated args up
+    let lint_args = match extra_checks {
+        Some(s) => s.strip_prefix("--extra-checks=").unwrap().split(',').collect(),
+        None => vec![],
+    };
+
+    let python_all = lint_args.contains(&"py");
+    let python_lint = lint_args.contains(&"py:lint") || python_all;
+    let python_fmt = lint_args.contains(&"py:fmt") || python_all;
+    let shell_all = lint_args.contains(&"shell");
+    let shell_lint = lint_args.contains(&"shell:lint") || shell_all;
+
+    let mut py_path = None;
+
+    let (cfg_args, file_args): (Vec<_>, Vec<_>) = pos_args
+        .into_iter()
+        .map(OsStr::new)
+        .partition(|arg| arg.to_str().is_some_and(|s| s.starts_with("-")));
+
+    if python_lint || python_fmt {
+        let venv_path = outdir.join("venv");
+        let mut reqs_path = root_path.to_owned();
+        reqs_path.extend(PIP_REQ_PATH);
+        py_path = Some(get_or_create_venv(&venv_path, &reqs_path)?);
+    }
+
+    if python_lint {
+        eprintln!("linting python files");
+        let mut cfg_args_ruff = cfg_args.clone();
+        let mut file_args_ruff = file_args.clone();
+
+        let mut cfg_path = root_path.to_owned();
+        cfg_path.extend(RUFF_CONFIG_PATH);
+        let mut cache_dir = outdir.to_owned();
+        cache_dir.extend(RUFF_CACH_PATH);
+
+        cfg_args_ruff.extend([
+            "--config".as_ref(),
+            cfg_path.as_os_str(),
+            "--cache-dir".as_ref(),
+            cache_dir.as_os_str(),
+        ]);
+
+        if file_args_ruff.is_empty() {
+            file_args_ruff.push(root_path.as_os_str());
+        }
+
+        let mut args = merge_args(&cfg_args_ruff, &file_args_ruff);
+        let res = py_runner(py_path.as_ref().unwrap(), "ruff", &args);
+
+        if res.is_err() && show_diff {
+            eprintln!("\npython linting failed! Printing diff suggestions:");
+
+            args.insert(0, "--diff".as_ref());
+            let _ = py_runner(py_path.as_ref().unwrap(), "ruff", &args);
+        }
+        // Rethrow error
+        let _ = res?;
+    }
+
+    if python_fmt {
+        let mut cfg_args_black = cfg_args.clone();
+        let mut file_args_black = file_args.clone();
+
+        if bless {
+            eprintln!("formatting python files");
+        } else {
+            eprintln!("checking python file formatting");
+            cfg_args_black.push("--check".as_ref());
+        }
+
+        let mut cfg_path = root_path.to_owned();
+        cfg_path.extend(BLACK_CONFIG_PATH);
+
+        cfg_args_black.extend(["--config".as_ref(), cfg_path.as_os_str()]);
+
+        if file_args_black.is_empty() {
+            file_args_black.push(root_path.as_os_str());
+        }
+
+        let mut args = merge_args(&cfg_args_black, &file_args_black);
+        let res = py_runner(py_path.as_ref().unwrap(), "black", &args);
+
+        if res.is_err() && show_diff {
+            eprintln!("\npython formatting does not match! Printing diff:");
+
+            args.insert(0, "--diff".as_ref());
+            let _ = py_runner(py_path.as_ref().unwrap(), "black", &args);
+        }
+        // Rethrow error
+        let _ = res?;
+    }
+
+    if shell_lint {
+        eprintln!("linting shell files");
+
+        let mut file_args_shc = file_args.clone();
+        let files;
+        if file_args_shc.is_empty() {
+            files = find_with_extension(root_path, "sh")?;
+            file_args_shc.extend(files.iter().map(|p| p.as_os_str()));
+        }
+
+        shellcheck_runner(&merge_args(&cfg_args, &file_args_shc))?;
+    }
+
+    Ok(())
+}
+
+/// Helper to create `cfg1 cfg2 -- file1 file2` output
+fn merge_args<'a>(cfg_args: &[&'a OsStr], file_args: &[&'a OsStr]) -> Vec<&'a OsStr> {
+    let mut args = cfg_args.to_owned();
+    args.push("--".as_ref());
+    args.extend(file_args);
+    args
+}
+
+/// Run a python command with given arguments. `py_path` should be a virtualenv.
+fn py_runner(py_path: &Path, bin: &'static str, args: &[&OsStr]) -> Result<(), Error> {
+    let status = Command::new(py_path).arg("-m").arg(bin).args(args).status()?;
+    if status.success() { Ok(()) } else { Err(Error::FailedCheck(bin)) }
+}
+
+/// Create a virtuaenv at a given path if it doesn't already exist, or validate
+/// the install if it does. Returns the path to that venv's python executable.
+fn get_or_create_venv(venv_path: &Path, src_reqs_path: &Path) -> Result<PathBuf, Error> {
+    let mut should_create = true;
+    let dst_reqs_path = venv_path.join("requirements.txt");
+    let mut py_path = venv_path.to_owned();
+    py_path.extend(REL_PY_PATH);
+
+    if let Ok(req) = fs::read_to_string(&dst_reqs_path) {
+        if req == fs::read_to_string(src_reqs_path)? {
+            // found existing environment
+            should_create = false;
+        } else {
+            eprintln!("requirements.txt file mismatch, recreating environment");
+        }
+    }
+
+    if should_create {
+        eprintln!("removing old virtual environment");
+        if venv_path.is_dir() {
+            fs::remove_dir_all(venv_path).unwrap_or_else(|_| {
+                panic!("failed to remove directory at {}", venv_path.display())
+            });
+        }
+        create_venv_at_path(venv_path)?;
+        install_requirements(&py_path, src_reqs_path, &dst_reqs_path)?;
+    }
+
+    verify_py_version(&py_path)?;
+    Ok(py_path)
+}
+
+/// Attempt to create a virtualenv at this path. Cycles through all expected
+/// valid python versions to find one that is installed.
+fn create_venv_at_path(path: &Path) -> Result<(), Error> {
+    /// Preferred python versions in order. Newest to oldest then current
+    /// development versions
+    const TRY_PY: &[&str] = &[
+        "python3.11",
+        "python3.10",
+        "python3.9",
+        "python3.8",
+        "python3.7",
+        "python3",
+        "python",
+        "python3.12",
+        "python3.13",
+    ];
+
+    let mut sys_py = None;
+    let mut found = Vec::new();
+
+    for py in TRY_PY {
+        match verify_py_version(Path::new(py)) {
+            Ok(_) => {
+                sys_py = Some(*py);
+                break;
+            }
+            // Skip not found errors
+            Err(Error::Io(e)) if e.kind() == io::ErrorKind::NotFound => (),
+            // Skip insufficient version errors
+            Err(Error::Version { installed, .. }) => found.push(installed),
+            // just log and skip unrecognized errors
+            Err(e) => eprintln!("note: error running '{py}': {e}"),
+        }
+    }
+
+    let Some(sys_py) = sys_py else {
+        let ret = if found.is_empty() {
+            Error::MissingReq("python3", "python file checks", None)
+        } else {
+            found.sort();
+            found.dedup();
+            Error::Version {
+                program: "python3",
+                required: MIN_PY_REV_STR,
+                installed: found.join(", "),
+            }
+        };
+        return Err(ret);
+    };
+
+    eprintln!("creating virtual environment at '{}' using '{sys_py}'", path.display());
+    let out = Command::new(sys_py).args(["-m", "virtualenv"]).arg(path).output().unwrap();
+
+    if out.status.success() {
+        return Ok(());
+    }
+    let err = if String::from_utf8_lossy(&out.stderr).contains("No module named virtualenv") {
+        Error::Generic(format!(
+            "virtualenv not found: you may need to install it \
+                               (`python3 -m pip install venv`)"
+        ))
+    } else {
+        Error::Generic(format!("failed to create venv at '{}' using {sys_py}", path.display()))
+    };
+    Err(err)
+}
+
+/// Parse python's version output (`Python x.y.z`) and ensure we have a
+/// suitable version.
+fn verify_py_version(py_path: &Path) -> Result<(), Error> {
+    let out = Command::new(py_path).arg("--version").output()?;
+    let outstr = String::from_utf8_lossy(&out.stdout);
+    let vers = outstr.trim().split_ascii_whitespace().nth(1).unwrap().trim();
+    let mut vers_comps = vers.split('.');
+    let major: u32 = vers_comps.next().unwrap().parse().unwrap();
+    let minor: u32 = vers_comps.next().unwrap().parse().unwrap();
+
+    if (major, minor) < MIN_PY_REV {
+        Err(Error::Version {
+            program: "python",
+            required: MIN_PY_REV_STR,
+            installed: vers.to_owned(),
+        })
+    } else {
+        Ok(())
+    }
+}
+
+fn install_requirements(
+    py_path: &Path,
+    src_reqs_path: &Path,
+    dst_reqs_path: &Path,
+) -> Result<(), Error> {
+    let stat = Command::new(py_path)
+        .args(["-m", "pip", "install", "--upgrade", "pip"])
+        .status()
+        .expect("failed to launch pip");
+    if !stat.success() {
+        return Err(Error::Generic(format!("pip install failed with status {stat}")));
+    }
+
+    let stat = Command::new(py_path)
+        .args(["-m", "pip", "install", "--require-hashes", "-r"])
+        .arg(src_reqs_path)
+        .status()?;
+    if !stat.success() {
+        return Err(Error::Generic(format!(
+            "failed to install requirements at {}",
+            src_reqs_path.display()
+        )));
+    }
+    fs::copy(src_reqs_path, dst_reqs_path)?;
+    assert_eq!(
+        fs::read_to_string(src_reqs_path).unwrap(),
+        fs::read_to_string(dst_reqs_path).unwrap()
+    );
+    Ok(())
+}
+
+/// Check that shellcheck is installed then run it at the given path
+fn shellcheck_runner(args: &[&OsStr]) -> Result<(), Error> {
+    match Command::new("shellcheck").arg("--version").status() {
+        Ok(_) => (),
+        Err(e) if e.kind() == io::ErrorKind::NotFound => {
+            return Err(Error::MissingReq(
+                "shellcheck",
+                "shell file checks",
+                Some(
+                    "see <https://github.com/koalaman/shellcheck#installing> \
+                    for installation instructions"
+                        .to_owned(),
+                ),
+            ));
+        }
+        Err(e) => return Err(e.into()),
+    }
+
+    let status = Command::new("shellcheck").args(args).status()?;
+    if status.success() { Ok(()) } else { Err(Error::FailedCheck("black")) }
+}
+
+/// Check git for tracked files matching an extension
+fn find_with_extension(root_path: &Path, extension: &str) -> Result<Vec<PathBuf>, Error> {
+    // Untracked files show up for short status and are indicated with a leading `?`
+    // -C changes git to be as if run from that directory
+    let stat_output =
+        Command::new("git").arg("-C").arg(root_path).args(["status", "--short"]).output()?.stdout;
+
+    if String::from_utf8_lossy(&stat_output).lines().filter(|ln| ln.starts_with('?')).count() > 0 {
+        eprintln!("found untracked files, ignoring");
+    }
+
+    let mut output = Vec::new();
+    let binding = Command::new("git").arg("-C").arg(root_path).args(["ls-files"]).output()?;
+    let tracked = String::from_utf8_lossy(&binding.stdout);
+
+    for line in tracked.lines() {
+        let line = line.trim();
+        let path = Path::new(line);
+
+        if path.extension() == Some(OsStr::new(extension)) {
+            output.push(path.to_owned());
+        }
+    }
+
+    Ok(output)
+}
+
+#[derive(Debug)]
+enum Error {
+    Io(io::Error),
+    /// a is required to run b. c is extra info
+    MissingReq(&'static str, &'static str, Option<String>),
+    /// Tool x failed the check
+    FailedCheck(&'static str),
+    /// Any message, just print it
+    Generic(String),
+    /// Installed but wrong version
+    Version {
+        program: &'static str,
+        required: &'static str,
+        installed: String,
+    },
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::MissingReq(a, b, ex) => {
+                write!(
+                    f,
+                    "{a} is required to run {b} but it could not be located. Is it installed?"
+                )?;
+                if let Some(s) = ex {
+                    write!(f, "\n{s}")?;
+                };
+                Ok(())
+            }
+            Self::Version { program, required, installed } => write!(
+                f,
+                "insufficient version of '{program}' to run external tools: \
+                {required} required but found {installed}",
+            ),
+            Self::Generic(s) => f.write_str(s),
+            Self::Io(e) => write!(f, "IO error: {e}"),
+            Self::FailedCheck(s) => write!(f, "checks with external tool '{s}' failed"),
+        }
+    }
+}
+
+impl From<io::Error> for Error {
+    fn from(value: io::Error) -> Self {
+        Self::Io(value)
+    }
+}
diff --git a/src/tools/tidy/src/features.rs b/src/tools/tidy/src/features.rs
index 7ad8f5c5c..d900c04c1 100644
--- a/src/tools/tidy/src/features.rs
+++ b/src/tools/tidy/src/features.rs
@@ -338,6 +338,7 @@ fn collect_lang_features_in(features: &mut Features, base: &Path, file: &str, ba
         let level = match parts.next().map(|l| l.trim().trim_start_matches('(')) {
             Some("active") => Status::Unstable,
             Some("incomplete") => Status::Unstable,
+            Some("internal") => Status::Unstable,
             Some("removed") => Status::Removed,
             Some("accepted") => Status::Stable,
             _ => continue,
diff --git a/src/tools/tidy/src/fluent_alphabetical.rs b/src/tools/tidy/src/fluent_alphabetical.rs
index 5f8eaebf5..67b745373 100644
--- a/src/tools/tidy/src/fluent_alphabetical.rs
+++ b/src/tools/tidy/src/fluent_alphabetical.rs
@@ -23,7 +23,7 @@ fn check_alphabetic(filename: &str, fluent: &str, bad: &mut bool) {
                 tidy_error!(
                     bad,
                     "{filename}: message `{}` appears before `{}`, but is alphabetically later than it
-run tidy with `--bless` to sort the file correctly",
+run `./x.py test tidy --bless` to sort the file correctly",
                     name.as_str(),
                     next.as_str()
                 );
diff --git a/src/tools/tidy/src/lib.rs b/src/tools/tidy/src/lib.rs
index e467514a7..9b19b8eec 100644
--- a/src/tools/tidy/src/lib.rs
+++ b/src/tools/tidy/src/lib.rs
@@ -57,6 +57,7 @@ pub mod debug_artifacts;
 pub mod deps;
 pub mod edition;
 pub mod error_codes;
+pub mod ext_tool_checks;
 pub mod extdeps;
 pub mod features;
 pub mod fluent_alphabetical;
diff --git a/src/tools/tidy/src/main.rs b/src/tools/tidy/src/main.rs
index e21068490..5fa91715a 100644
--- a/src/tools/tidy/src/main.rs
+++ b/src/tools/tidy/src/main.rs
@@ -37,9 +37,14 @@ fn main() {
     let librustdoc_path = src_path.join("librustdoc");
 
     let args: Vec<String> = env::args().skip(1).collect();
-
-    let verbose = args.iter().any(|s| *s == "--verbose");
-    let bless = args.iter().any(|s| *s == "--bless");
+    let (cfg_args, pos_args) = match args.iter().position(|arg| arg == "--") {
+        Some(pos) => (&args[..pos], &args[pos + 1..]),
+        None => (&args[..], [].as_slice()),
+    };
+    let verbose = cfg_args.iter().any(|s| *s == "--verbose");
+    let bless = cfg_args.iter().any(|s| *s == "--bless");
+    let extra_checks =
+        cfg_args.iter().find(|s| s.starts_with("--extra-checks=")).map(String::as_str);
 
     let bad = std::sync::Arc::new(AtomicBool::new(false));
 
@@ -150,6 +155,8 @@ fn main() {
             r
         };
         check!(unstable_book, &src_path, collected);
+
+        check!(ext_tool_checks, &root_path, &output_directory, bless, extra_checks, pos_args);
     });
 
     if bad.load(Ordering::Relaxed) {
diff --git a/src/tools/tidy/src/pal.rs b/src/tools/tidy/src/pal.rs
index 6fd41e833..3a4d9c53d 100644
--- a/src/tools/tidy/src/pal.rs
+++ b/src/tools/tidy/src/pal.rs
@@ -39,7 +39,6 @@ const EXCEPTION_PATHS: &[&str] = &[
     "library/panic_unwind",
     "library/unwind",
     "library/rtstartup", // Not sure what to do about this. magic stuff for mingw
-    "library/term",      // Not sure how to make this crate portable, but test crate needs it.
     "library/test",      // Probably should defer to unstable `std::sys` APIs.
     // The `VaList` implementation must have platform specific code.
     // The Windows implementation of a `va_list` is always a character
@@ -53,13 +52,10 @@ const EXCEPTION_PATHS: &[&str] = &[
     // FIXME: platform-specific code should be moved to `sys`
     "library/std/src/io/copy.rs",
     "library/std/src/io/stdio.rs",
-    "library/std/src/f32.rs",
-    "library/std/src/f64.rs",
+    "library/std/src/lib.rs", // for miniz_oxide leaking docs, which itself workaround
     "library/std/src/path.rs",
     "library/std/src/sys_common", // Should only contain abstractions over platforms
     "library/std/src/net/test.rs", // Utility helpers for tests
-    "library/std/src/personality.rs",
-    "library/std/src/personality/",
 ];
 
 pub fn check(path: &Path, bad: &mut bool) {
diff --git a/src/tools/tidy/src/style.rs b/src/tools/tidy/src/style.rs
index d0257d716..11480e2be 100644
--- a/src/tools/tidy/src/style.rs
+++ b/src/tools/tidy/src/style.rs
@@ -302,10 +302,6 @@ pub fn check(path: &Path, bad: &mut bool) {
                 return;
             }
         }
-        // apfloat shouldn't be changed because of license problems
-        if is_in(file, "compiler", "rustc_apfloat") {
-            return;
-        }
         let mut skip_cr = contains_ignore_directive(can_contain, &contents, "cr");
         let mut skip_undocumented_unsafe =
             contains_ignore_directive(can_contain, &contents, "undocumented-unsafe");
diff --git a/src/tools/tidy/src/ui_tests.rs b/src/tools/tidy/src/ui_tests.rs
index 55bf38110..341492400 100644
--- a/src/tools/tidy/src/ui_tests.rs
+++ b/src/tools/tidy/src/ui_tests.rs
@@ -10,8 +10,8 @@ use std::path::{Path, PathBuf};
 
 const ENTRY_LIMIT: usize = 900;
 // FIXME: The following limits should be reduced eventually.
-const ISSUES_ENTRY_LIMIT: usize = 1896;
-const ROOT_ENTRY_LIMIT: usize = 870;
+const ISSUES_ENTRY_LIMIT: usize = 1891;
+const ROOT_ENTRY_LIMIT: usize = 866;
 
 const EXPECTED_TEST_FILE_EXTENSIONS: &[&str] = &[
     "rs",     // test source files
@@ -100,7 +100,7 @@ pub fn check(path: &Path, bad: &mut bool) {
             {
                 tidy_error!(bad, "file {} has unexpected extension {}", file_path.display(), ext);
             }
-            if ext == "stderr" || ext == "stdout" {
+            if ext == "stderr" || ext == "stdout" || ext == "fixed" {
                 // Test output filenames have one of the formats:
                 // ```
                 // $testname.stderr
@@ -116,7 +116,9 @@ pub fn check(path: &Path, bad: &mut bool) {
                 // must strip all of them.
                 let testname =
                     file_path.file_name().unwrap().to_str().unwrap().split_once('.').unwrap().0;
-                if !file_path.with_file_name(testname).with_extension("rs").exists() {
+                if !file_path.with_file_name(testname).with_extension("rs").exists()
+                    && !testname.contains("ignore-tidy")
+                {
                     tidy_error!(bad, "Stray file with UI testing output: {:?}", file_path);
                 }