Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

8 files changed, 2576 insertions, 0 deletions

diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
new file mode 100644
index 000000000..5673bb83b
--- /dev/null
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "rustc_query_impl"
+version = "0.0.0"
+edition = "2021"
+
+[lib]
+doctest = false
+
+[dependencies]
+measureme = "10.0.0"
+rustc-rayon-core = { version = "0.4.0", optional = true }
+rustc_ast = { path = "../rustc_ast" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_index = { path = "../rustc_index" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_query_system = { path = "../rustc_query_system" }
+rustc_serialize = { path = "../rustc_serialize" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+tracing = "0.1"
+
+[features]
+rustc_use_parallel_compiler = ["rustc-rayon-core", "rustc_query_system/rustc_use_parallel_compiler"]
diff --git a/compiler/rustc_query_impl/src/README.md b/compiler/rustc_query_impl/src/README.md
new file mode 100644
index 000000000..8ec07b9fd
--- /dev/null
+++ b/compiler/rustc_query_impl/src/README.md
@@ -0,0 +1,3 @@
+For more information about how the query system works, see the [rustc dev guide].
+
+[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/query.html
diff --git a/compiler/rustc_query_impl/src/keys.rs b/compiler/rustc_query_impl/src/keys.rs
new file mode 100644
index 000000000..49175e97f
--- /dev/null
+++ b/compiler/rustc_query_impl/src/keys.rs
@@ -0,0 +1,545 @@
+//! Defines the set of legal keys that can be used in queries.
+
+use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE};
+use rustc_middle::infer::canonical::Canonical;
+use rustc_middle::mir;
+use rustc_middle::traits;
+use rustc_middle::ty::fast_reject::SimplifiedType;
+use rustc_middle::ty::subst::{GenericArg, SubstsRef};
+use rustc_middle::ty::{self, layout::TyAndLayout, Ty, TyCtxt};
+use rustc_span::symbol::{Ident, Symbol};
+use rustc_span::{Span, DUMMY_SP};
+
+/// The `Key` trait controls what types can legally be used as the key
+/// for a query.
+pub trait Key {
+    /// Given an instance of this key, what crate is it referring to?
+    /// This is used to find the provider.
+    fn query_crate_is_local(&self) -> bool;
+
+    /// In the event that a cycle occurs, if no explicit span has been
+    /// given for a query with key `self`, what span should we use?
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span;
+
+    /// If the key is a [`DefId`] or `DefId`--equivalent, return that `DefId`.
+    /// Otherwise, return `None`.
+    fn key_as_def_id(&self) -> Option<DefId> {
+        None
+    }
+}
+
+impl Key for () {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for ty::InstanceDef<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.def_id())
+    }
+}
+
+impl<'tcx> Key for ty::Instance<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.def_id())
+    }
+}
+
+impl<'tcx> Key for mir::interpret::GlobalId<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.instance.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for CrateNum {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        *self == LOCAL_CRATE
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for LocalDefId {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.to_def_id().default_span(tcx)
+    }
+    fn key_as_def_id(&self) -> Option<DefId> {
+        Some(self.to_def_id())
+    }
+}
+
+impl Key for DefId {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(*self)
+    }
+    #[inline(always)]
+    fn key_as_def_id(&self) -> Option<DefId> {
+        Some(*self)
+    }
+}
+
+impl Key for ty::WithOptConstParam<LocalDefId> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.did.default_span(tcx)
+    }
+}
+
+impl Key for SimplifiedType {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for (DefId, DefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.1.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (ty::Instance<'tcx>, LocalDefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl Key for (DefId, LocalDefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.1.default_span(tcx)
+    }
+}
+
+impl Key for (LocalDefId, DefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl Key for (LocalDefId, LocalDefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl Key for (DefId, Option<Ident>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.0)
+    }
+    #[inline(always)]
+    fn key_as_def_id(&self) -> Option<DefId> {
+        Some(self.0)
+    }
+}
+
+impl Key for (DefId, LocalDefId, Ident) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.1.default_span(tcx)
+    }
+}
+
+impl Key for (CrateNum, DefId) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0 == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.1.default_span(tcx)
+    }
+}
+
+impl Key for (CrateNum, SimplifiedType) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0 == LOCAL_CRATE
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for (DefId, SimplifiedType) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for SubstsRef<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (DefId, SubstsRef<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (ty::Unevaluated<'tcx, ()>, ty::Unevaluated<'tcx, ()>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        (self.0).def.did.krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        (self.0).def.did.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.1.def_id().krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.1.def_id())
+    }
+}
+
+impl<'tcx> Key for (ty::Const<'tcx>, mir::Field) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for mir::interpret::ConstAlloc<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.def_id().krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.def_id())
+    }
+}
+
+impl<'tcx> Key for ty::PolyExistentialTraitRef<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.def_id().krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.def_id())
+    }
+}
+
+impl<'tcx> Key for (ty::PolyTraitRef<'tcx>, ty::PolyTraitRef<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.0.def_id().krate == LOCAL_CRATE
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        tcx.def_span(self.0.def_id())
+    }
+}
+
+impl<'tcx> Key for GenericArg<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for mir::ConstantKind<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for ty::Const<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for Ty<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for TyAndLayout<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (Ty<'tcx>, Ty<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for &'tcx ty::List<ty::Predicate<'tcx>> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for ty::ParamEnv<'tcx> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        self.value.query_crate_is_local()
+    }
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.value.default_span(tcx)
+    }
+}
+
+impl Key for Symbol {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for Option<Symbol> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+/// Canonical query goals correspond to abstract trait operations that
+/// are not tied to any crate in particular.
+impl<'tcx, T> Key for Canonical<'tcx, T> {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl Key for (Symbol, u32, u32) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (DefId, Ty<'tcx>, SubstsRef<'tcx>, ty::ParamEnv<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (ty::Predicate<'tcx>, traits::WellFormedLoc) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
+
+impl<'tcx> Key for (ty::Instance<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+        self.0.default_span(tcx)
+    }
+}
+
+impl<'tcx> Key for (Ty<'tcx>, ty::ValTree<'tcx>) {
+    #[inline(always)]
+    fn query_crate_is_local(&self) -> bool {
+        true
+    }
+
+    fn default_span(&self, _: TyCtxt<'_>) -> Span {
+        DUMMY_SP
+    }
+}
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
new file mode 100644
index 000000000..eda61df77
--- /dev/null
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -0,0 +1,63 @@
+//! Support for serializing the dep-graph and reloading it.
+
+#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(min_specialization)]
+#![feature(never_type)]
+#![feature(once_cell)]
+#![feature(rustc_attrs)]
+#![recursion_limit = "256"]
+#![allow(rustc::potential_query_instability)]
+
+#[macro_use]
+extern crate rustc_macros;
+#[macro_use]
+extern crate rustc_middle;
+
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_data_structures::sync::AtomicU64;
+use rustc_middle::arena::Arena;
+use rustc_middle::dep_graph::{self, DepKindStruct, SerializedDepNodeIndex};
+use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
+use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
+use rustc_middle::ty::{self, TyCtxt};
+use rustc_span::def_id::{LocalDefId, LOCAL_CRATE};
+use rustc_span::Span;
+
+#[macro_use]
+mod plumbing;
+pub use plumbing::QueryCtxt;
+use rustc_query_system::query::*;
+#[cfg(parallel_compiler)]
+pub use rustc_query_system::query::{deadlock, QueryContext};
+
+mod keys;
+use keys::Key;
+
+mod values;
+use self::values::Value;
+
+pub use rustc_query_system::query::QueryConfig;
+pub(crate) use rustc_query_system::query::{QueryDescription, QueryVTable};
+
+mod on_disk_cache;
+pub use on_disk_cache::OnDiskCache;
+
+mod profiling_support;
+pub use self::profiling_support::alloc_self_profile_query_strings;
+
+fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String {
+    if def_id.is_top_level_module() {
+        "top-level module".to_string()
+    } else {
+        format!("module `{}`", tcx.def_path_str(def_id.to_def_id()))
+    }
+}
+
+rustc_query_append! { [define_queries!][<'tcx>] }
+
+impl<'tcx> Queries<'tcx> {
+    // Force codegen in the dyn-trait transformation in this crate.
+    pub fn as_dyn(&'tcx self) -> &'tcx dyn QueryEngine<'tcx> {
+        self
+    }
+}
diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs
new file mode 100644
index 000000000..56fd90c98
--- /dev/null
+++ b/compiler/rustc_query_impl/src/on_disk_cache.rs
@@ -0,0 +1,1012 @@
+use crate::QueryCtxt;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
+use rustc_data_structures::memmap::Mmap;
+use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, RwLock};
+use rustc_data_structures::unhash::UnhashMap;
+use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE};
+use rustc_hir::definitions::DefPathHash;
+use rustc_index::vec::{Idx, IndexVec};
+use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState};
+use rustc_middle::mir::{self, interpret};
+use rustc_middle::ty::codec::{RefDecodable, TyDecoder, TyEncoder};
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_query_system::dep_graph::DepContext;
+use rustc_query_system::query::{QueryCache, QueryContext, QuerySideEffects};
+use rustc_serialize::{
+    opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder},
+    Decodable, Decoder, Encodable, Encoder,
+};
+use rustc_session::Session;
+use rustc_span::hygiene::{
+    ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextData,
+};
+use rustc_span::source_map::{SourceMap, StableSourceFileId};
+use rustc_span::CachingSourceMapView;
+use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, SourceFile, Span};
+use std::io;
+use std::mem;
+
+const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE;
+
+// A normal span encoded with both location information and a `SyntaxContext`
+const TAG_FULL_SPAN: u8 = 0;
+// A partial span with no location information, encoded only with a `SyntaxContext`
+const TAG_PARTIAL_SPAN: u8 = 1;
+const TAG_RELATIVE_SPAN: u8 = 2;
+
+const TAG_SYNTAX_CONTEXT: u8 = 0;
+const TAG_EXPN_DATA: u8 = 1;
+
+/// Provides an interface to incremental compilation data cached from the
+/// previous compilation session. This data will eventually include the results
+/// of a few selected queries (like `typeck` and `mir_optimized`) and
+/// any side effects that have been emitted during a query.
+pub struct OnDiskCache<'sess> {
+    // The complete cache data in serialized form.
+    serialized_data: RwLock<Option<Mmap>>,
+
+    // Collects all `QuerySideEffects` created during the current compilation
+    // session.
+    current_side_effects: Lock<FxHashMap<DepNodeIndex, QuerySideEffects>>,
+
+    source_map: &'sess SourceMap,
+    file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
+
+    // Caches that are populated lazily during decoding.
+    file_index_to_file: Lock<FxHashMap<SourceFileIndex, Lrc<SourceFile>>>,
+
+    // A map from dep-node to the position of the cached query result in
+    // `serialized_data`.
+    query_result_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+
+    // A map from dep-node to the position of any associated `QuerySideEffects` in
+    // `serialized_data`.
+    prev_side_effects_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+
+    alloc_decoding_state: AllocDecodingState,
+
+    // A map from syntax context ids to the position of their associated
+    // `SyntaxContextData`. We use a `u32` instead of a `SyntaxContext`
+    // to represent the fact that we are storing *encoded* ids. When we decode
+    // a `SyntaxContext`, a new id will be allocated from the global `HygieneData`,
+    // which will almost certainly be different than the serialized id.
+    syntax_contexts: FxHashMap<u32, AbsoluteBytePos>,
+    // A map from the `DefPathHash` of an `ExpnId` to the position
+    // of their associated `ExpnData`. Ideally, we would store a `DefId`,
+    // but we need to decode this before we've constructed a `TyCtxt` (which
+    // makes it difficult to decode a `DefId`).
+
+    // Note that these `DefPathHashes` correspond to both local and foreign
+    // `ExpnData` (e.g `ExpnData.krate` may not be `LOCAL_CRATE`). Alternatively,
+    // we could look up the `ExpnData` from the metadata of foreign crates,
+    // but it seemed easier to have `OnDiskCache` be independent of the `CStore`.
+    expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
+    // Additional information used when decoding hygiene data.
+    hygiene_context: HygieneDecodeContext,
+    // Maps `ExpnHash`es to their raw value from the *previous*
+    // compilation session. This is used as an initial 'guess' when
+    // we try to map an `ExpnHash` to its value in the current
+    // compilation session.
+    foreign_expn_data: UnhashMap<ExpnHash, u32>,
+}
+
+// This type is used only for serialization and deserialization.
+#[derive(Encodable, Decodable)]
+struct Footer {
+    file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
+    query_result_index: EncodedDepNodeIndex,
+    side_effects_index: EncodedDepNodeIndex,
+    // The location of all allocations.
+    interpret_alloc_index: Vec<u32>,
+    // See `OnDiskCache.syntax_contexts`
+    syntax_contexts: FxHashMap<u32, AbsoluteBytePos>,
+    // See `OnDiskCache.expn_data`
+    expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
+    foreign_expn_data: UnhashMap<ExpnHash, u32>,
+}
+
+pub type EncodedDepNodeIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Encodable, Decodable)]
+struct SourceFileIndex(u32);
+
+#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Encodable, Decodable)]
+pub struct AbsoluteBytePos(u32);
+
+impl AbsoluteBytePos {
+    fn new(pos: usize) -> AbsoluteBytePos {
+        debug_assert!(pos <= u32::MAX as usize);
+        AbsoluteBytePos(pos as u32)
+    }
+
+    fn to_usize(self) -> usize {
+        self.0 as usize
+    }
+}
+
+/// An `EncodedSourceFileId` is the same as a `StableSourceFileId` except that
+/// the source crate is represented as a [StableCrateId] instead of as a
+/// `CrateNum`. This way `EncodedSourceFileId` can be encoded and decoded
+/// without any additional context, i.e. with a simple `opaque::Decoder` (which
+/// is the only thing available when decoding the cache's [Footer].
+#[derive(Encodable, Decodable, Clone, Debug)]
+struct EncodedSourceFileId {
+    file_name_hash: u64,
+    stable_crate_id: StableCrateId,
+}
+
+impl EncodedSourceFileId {
+    fn translate(&self, tcx: TyCtxt<'_>) -> StableSourceFileId {
+        let cnum = tcx.stable_crate_id_to_crate_num(self.stable_crate_id);
+        StableSourceFileId { file_name_hash: self.file_name_hash, cnum }
+    }
+
+    fn new(tcx: TyCtxt<'_>, file: &SourceFile) -> EncodedSourceFileId {
+        let source_file_id = StableSourceFileId::new(file);
+        EncodedSourceFileId {
+            file_name_hash: source_file_id.file_name_hash,
+            stable_crate_id: tcx.stable_crate_id(source_file_id.cnum),
+        }
+    }
+}
+
+impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
+    /// Creates a new `OnDiskCache` instance from the serialized data in `data`.
+    fn new(sess: &'sess Session, data: Mmap, start_pos: usize) -> Self {
+        debug_assert!(sess.opts.incremental.is_some());
+
+        // Wrap in a scope so we can borrow `data`.
+        let footer: Footer = {
+            let mut decoder = MemDecoder::new(&data, start_pos);
+
+            // Decode the *position* of the footer, which can be found in the
+            // last 8 bytes of the file.
+            decoder.set_position(data.len() - IntEncodedWithFixedSize::ENCODED_SIZE);
+            let footer_pos = IntEncodedWithFixedSize::decode(&mut decoder).0 as usize;
+
+            // Decode the file footer, which contains all the lookup tables, etc.
+            decoder.set_position(footer_pos);
+
+            decode_tagged(&mut decoder, TAG_FILE_FOOTER)
+        };
+
+        Self {
+            serialized_data: RwLock::new(Some(data)),
+            file_index_to_stable_id: footer.file_index_to_stable_id,
+            file_index_to_file: Default::default(),
+            source_map: sess.source_map(),
+            current_side_effects: Default::default(),
+            query_result_index: footer.query_result_index.into_iter().collect(),
+            prev_side_effects_index: footer.side_effects_index.into_iter().collect(),
+            alloc_decoding_state: AllocDecodingState::new(footer.interpret_alloc_index),
+            syntax_contexts: footer.syntax_contexts,
+            expn_data: footer.expn_data,
+            foreign_expn_data: footer.foreign_expn_data,
+            hygiene_context: Default::default(),
+        }
+    }
+
+    fn new_empty(source_map: &'sess SourceMap) -> Self {
+        Self {
+            serialized_data: RwLock::new(None),
+            file_index_to_stable_id: Default::default(),
+            file_index_to_file: Default::default(),
+            source_map,
+            current_side_effects: Default::default(),
+            query_result_index: Default::default(),
+            prev_side_effects_index: Default::default(),
+            alloc_decoding_state: AllocDecodingState::new(Vec::new()),
+            syntax_contexts: FxHashMap::default(),
+            expn_data: UnhashMap::default(),
+            foreign_expn_data: UnhashMap::default(),
+            hygiene_context: Default::default(),
+        }
+    }
+
+    /// Execute all cache promotions and release the serialized backing Mmap.
+    ///
+    /// Cache promotions require invoking queries, which needs to read the serialized data.
+    /// In order to serialize the new on-disk cache, the former on-disk cache file needs to be
+    /// deleted, hence we won't be able to refer to its memmapped data.
+    fn drop_serialized_data(&self, tcx: TyCtxt<'_>) {
+        // Load everything into memory so we can write it out to the on-disk
+        // cache. The vast majority of cacheable query results should already
+        // be in memory, so this should be a cheap operation.
+        // Do this *before* we clone 'latest_foreign_def_path_hashes', since
+        // loading existing queries may cause us to create new DepNodes, which
+        // may in turn end up invoking `store_foreign_def_id_hash`
+        tcx.dep_graph.exec_cache_promotions(tcx);
+
+        *self.serialized_data.write() = None;
+    }
+
+    fn serialize<'tcx>(&self, tcx: TyCtxt<'tcx>, encoder: FileEncoder) -> FileEncodeResult {
+        // Serializing the `DepGraph` should not modify it.
+        tcx.dep_graph.with_ignore(|| {
+            // Allocate `SourceFileIndex`es.
+            let (file_to_file_index, file_index_to_stable_id) = {
+                let files = tcx.sess.source_map().files();
+                let mut file_to_file_index =
+                    FxHashMap::with_capacity_and_hasher(files.len(), Default::default());
+                let mut file_index_to_stable_id =
+                    FxHashMap::with_capacity_and_hasher(files.len(), Default::default());
+
+                for (index, file) in files.iter().enumerate() {
+                    let index = SourceFileIndex(index as u32);
+                    let file_ptr: *const SourceFile = &**file as *const _;
+                    file_to_file_index.insert(file_ptr, index);
+                    let source_file_id = EncodedSourceFileId::new(tcx, &file);
+                    file_index_to_stable_id.insert(index, source_file_id);
+                }
+
+                (file_to_file_index, file_index_to_stable_id)
+            };
+
+            let hygiene_encode_context = HygieneEncodeContext::default();
+
+            let mut encoder = CacheEncoder {
+                tcx,
+                encoder,
+                type_shorthands: Default::default(),
+                predicate_shorthands: Default::default(),
+                interpret_allocs: Default::default(),
+                source_map: CachingSourceMapView::new(tcx.sess.source_map()),
+                file_to_file_index,
+                hygiene_context: &hygiene_encode_context,
+            };
+
+            // Encode query results.
+            let mut query_result_index = EncodedDepNodeIndex::new();
+
+            tcx.sess.time("encode_query_results", || {
+                let enc = &mut encoder;
+                let qri = &mut query_result_index;
+                QueryCtxt::from_tcx(tcx).encode_query_results(enc, qri);
+            });
+
+            // Encode side effects.
+            let side_effects_index: EncodedDepNodeIndex = self
+                .current_side_effects
+                .borrow()
+                .iter()
+                .map(|(dep_node_index, side_effects)| {
+                    let pos = AbsoluteBytePos::new(encoder.position());
+                    let dep_node_index = SerializedDepNodeIndex::new(dep_node_index.index());
+                    encoder.encode_tagged(dep_node_index, side_effects);
+
+                    (dep_node_index, pos)
+                })
+                .collect();
+
+            let interpret_alloc_index = {
+                let mut interpret_alloc_index = Vec::new();
+                let mut n = 0;
+                loop {
+                    let new_n = encoder.interpret_allocs.len();
+                    // If we have found new IDs, serialize those too.
+                    if n == new_n {
+                        // Otherwise, abort.
+                        break;
+                    }
+                    interpret_alloc_index.reserve(new_n - n);
+                    for idx in n..new_n {
+                        let id = encoder.interpret_allocs[idx];
+                        let pos = encoder.position() as u32;
+                        interpret_alloc_index.push(pos);
+                        interpret::specialized_encode_alloc_id(&mut encoder, tcx, id);
+                    }
+                    n = new_n;
+                }
+                interpret_alloc_index
+            };
+
+            let mut syntax_contexts = FxHashMap::default();
+            let mut expn_data = UnhashMap::default();
+            let mut foreign_expn_data = UnhashMap::default();
+
+            // Encode all hygiene data (`SyntaxContextData` and `ExpnData`) from the current
+            // session.
+
+            hygiene_encode_context.encode(
+                &mut encoder,
+                |encoder, index, ctxt_data| {
+                    let pos = AbsoluteBytePos::new(encoder.position());
+                    encoder.encode_tagged(TAG_SYNTAX_CONTEXT, ctxt_data);
+                    syntax_contexts.insert(index, pos);
+                },
+                |encoder, expn_id, data, hash| {
+                    if expn_id.krate == LOCAL_CRATE {
+                        let pos = AbsoluteBytePos::new(encoder.position());
+                        encoder.encode_tagged(TAG_EXPN_DATA, data);
+                        expn_data.insert(hash, pos);
+                    } else {
+                        foreign_expn_data.insert(hash, expn_id.local_id.as_u32());
+                    }
+                },
+            );
+
+            // `Encode the file footer.
+            let footer_pos = encoder.position() as u64;
+            encoder.encode_tagged(
+                TAG_FILE_FOOTER,
+                &Footer {
+                    file_index_to_stable_id,
+                    query_result_index,
+                    side_effects_index,
+                    interpret_alloc_index,
+                    syntax_contexts,
+                    expn_data,
+                    foreign_expn_data,
+                },
+            );
+
+            // Encode the position of the footer as the last 8 bytes of the
+            // file so we know where to look for it.
+            IntEncodedWithFixedSize(footer_pos).encode(&mut encoder.encoder);
+
+            // DO NOT WRITE ANYTHING TO THE ENCODER AFTER THIS POINT! The address
+            // of the footer must be the last thing in the data stream.
+
+            encoder.finish()
+        })
+    }
+}
+
+impl<'sess> OnDiskCache<'sess> {
+    pub fn as_dyn(&self) -> &dyn rustc_middle::ty::OnDiskCache<'sess> {
+        self as _
+    }
+
+    /// Loads a `QuerySideEffects` created during the previous compilation session.
+    pub fn load_side_effects(
+        &self,
+        tcx: TyCtxt<'_>,
+        dep_node_index: SerializedDepNodeIndex,
+    ) -> QuerySideEffects {
+        let side_effects: Option<QuerySideEffects> =
+            self.load_indexed(tcx, dep_node_index, &self.prev_side_effects_index);
+
+        side_effects.unwrap_or_default()
+    }
+
+    /// Stores a `QuerySideEffects` emitted during the current compilation session.
+    /// Anything stored like this will be available via `load_side_effects` in
+    /// the next compilation session.
+    #[inline(never)]
+    #[cold]
+    pub fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
+        let mut current_side_effects = self.current_side_effects.borrow_mut();
+        let prev = current_side_effects.insert(dep_node_index, side_effects);
+        debug_assert!(prev.is_none());
+    }
+
+    /// Returns the cached query result if there is something in the cache for
+    /// the given `SerializedDepNodeIndex`; otherwise returns `None`.
+    pub fn try_load_query_result<'tcx, T>(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        dep_node_index: SerializedDepNodeIndex,
+    ) -> Option<T>
+    where
+        T: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
+    {
+        self.load_indexed(tcx, dep_node_index, &self.query_result_index)
+    }
+
+    /// Stores side effect emitted during computation of an anonymous query.
+    /// Since many anonymous queries can share the same `DepNode`, we aggregate
+    /// them -- as opposed to regular queries where we assume that there is a
+    /// 1:1 relationship between query-key and `DepNode`.
+    #[inline(never)]
+    #[cold]
+    pub fn store_side_effects_for_anon_node(
+        &self,
+        dep_node_index: DepNodeIndex,
+        side_effects: QuerySideEffects,
+    ) {
+        let mut current_side_effects = self.current_side_effects.borrow_mut();
+
+        let x = current_side_effects.entry(dep_node_index).or_default();
+        x.append(side_effects);
+    }
+
+    fn load_indexed<'tcx, T>(
+        &self,
+        tcx: TyCtxt<'tcx>,
+        dep_node_index: SerializedDepNodeIndex,
+        index: &FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+    ) -> Option<T>
+    where
+        T: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
+    {
+        let pos = index.get(&dep_node_index).cloned()?;
+
+        self.with_decoder(tcx, pos, |decoder| Some(decode_tagged(decoder, dep_node_index)))
+    }
+
+    fn with_decoder<'a, 'tcx, T, F: for<'s> FnOnce(&mut CacheDecoder<'s, 'tcx>) -> T>(
+        &'sess self,
+        tcx: TyCtxt<'tcx>,
+        pos: AbsoluteBytePos,
+        f: F,
+    ) -> T
+    where
+        T: Decodable<CacheDecoder<'a, 'tcx>>,
+    {
+        let serialized_data = self.serialized_data.read();
+        let mut decoder = CacheDecoder {
+            tcx,
+            opaque: MemDecoder::new(serialized_data.as_deref().unwrap_or(&[]), pos.to_usize()),
+            source_map: self.source_map,
+            file_index_to_file: &self.file_index_to_file,
+            file_index_to_stable_id: &self.file_index_to_stable_id,
+            alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(),
+            syntax_contexts: &self.syntax_contexts,
+            expn_data: &self.expn_data,
+            foreign_expn_data: &self.foreign_expn_data,
+            hygiene_context: &self.hygiene_context,
+        };
+        f(&mut decoder)
+    }
+}
+
+//- DECODING -------------------------------------------------------------------
+
+/// A decoder that can read from the incremental compilation cache. It is similar to the one
+/// we use for crate metadata decoding in that it can rebase spans and eventually
+/// will also handle things that contain `Ty` instances.
+pub struct CacheDecoder<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    opaque: MemDecoder<'a>,
+    source_map: &'a SourceMap,
+    file_index_to_file: &'a Lock<FxHashMap<SourceFileIndex, Lrc<SourceFile>>>,
+    file_index_to_stable_id: &'a FxHashMap<SourceFileIndex, EncodedSourceFileId>,
+    alloc_decoding_session: AllocDecodingSession<'a>,
+    syntax_contexts: &'a FxHashMap<u32, AbsoluteBytePos>,
+    expn_data: &'a UnhashMap<ExpnHash, AbsoluteBytePos>,
+    foreign_expn_data: &'a UnhashMap<ExpnHash, u32>,
+    hygiene_context: &'a HygieneDecodeContext,
+}
+
+impl<'a, 'tcx> CacheDecoder<'a, 'tcx> {
+    fn file_index_to_file(&self, index: SourceFileIndex) -> Lrc<SourceFile> {
+        let CacheDecoder {
+            tcx,
+            ref file_index_to_file,
+            ref file_index_to_stable_id,
+            ref source_map,
+            ..
+        } = *self;
+
+        file_index_to_file
+            .borrow_mut()
+            .entry(index)
+            .or_insert_with(|| {
+                let stable_id = file_index_to_stable_id[&index].translate(tcx);
+
+                // If this `SourceFile` is from a foreign crate, then make sure
+                // that we've imported all of the source files from that crate.
+                // This has usually already been done during macro invocation.
+                // However, when encoding query results like `TypeckResults`,
+                // we might encode an `AdtDef` for a foreign type (because it
+                // was referenced in the body of the function). There is no guarantee
+                // that we will load the source files from that crate during macro
+                // expansion, so we use `import_source_files` to ensure that the foreign
+                // source files are actually imported before we call `source_file_by_stable_id`.
+                if stable_id.cnum != LOCAL_CRATE {
+                    self.tcx.cstore_untracked().import_source_files(self.tcx.sess, stable_id.cnum);
+                }
+
+                source_map
+                    .source_file_by_stable_id(stable_id)
+                    .expect("failed to lookup `SourceFile` in new context")
+            })
+            .clone()
+    }
+}
+
+trait DecoderWithPosition: Decoder {
+    fn position(&self) -> usize;
+}
+
+impl<'a> DecoderWithPosition for MemDecoder<'a> {
+    fn position(&self) -> usize {
+        self.position()
+    }
+}
+
+impl<'a, 'tcx> DecoderWithPosition for CacheDecoder<'a, 'tcx> {
+    fn position(&self) -> usize {
+        self.opaque.position()
+    }
+}
+
+// Decodes something that was encoded with `encode_tagged()` and verify that the
+// tag matches and the correct amount of bytes was read.
+fn decode_tagged<D, T, V>(decoder: &mut D, expected_tag: T) -> V
+where
+    T: Decodable<D> + Eq + std::fmt::Debug,
+    V: Decodable<D>,
+    D: DecoderWithPosition,
+{
+    let start_pos = decoder.position();
+
+    let actual_tag = T::decode(decoder);
+    assert_eq!(actual_tag, expected_tag);
+    let value = V::decode(decoder);
+    let end_pos = decoder.position();
+
+    let expected_len: u64 = Decodable::decode(decoder);
+    assert_eq!((end_pos - start_pos) as u64, expected_len);
+
+    value
+}
+
+impl<'a, 'tcx> TyDecoder for CacheDecoder<'a, 'tcx> {
+    type I = TyCtxt<'tcx>;
+    const CLEAR_CROSS_CRATE: bool = false;
+
+    #[inline]
+    fn interner(&self) -> TyCtxt<'tcx> {
+        self.tcx
+    }
+
+    #[inline]
+    fn position(&self) -> usize {
+        self.opaque.position()
+    }
+
+    #[inline]
+    fn peek_byte(&self) -> u8 {
+        self.opaque.data[self.opaque.position()]
+    }
+
+    fn cached_ty_for_shorthand<F>(&mut self, shorthand: usize, or_insert_with: F) -> Ty<'tcx>
+    where
+        F: FnOnce(&mut Self) -> Ty<'tcx>,
+    {
+        let tcx = self.tcx;
+
+        let cache_key = ty::CReaderCacheKey { cnum: None, pos: shorthand };
+
+        if let Some(&ty) = tcx.ty_rcache.borrow().get(&cache_key) {
+            return ty;
+        }
+
+        let ty = or_insert_with(self);
+        // This may overwrite the entry, but it should overwrite with the same value.
+        tcx.ty_rcache.borrow_mut().insert_same(cache_key, ty);
+        ty
+    }
+
+    fn with_position<F, R>(&mut self, pos: usize, f: F) -> R
+    where
+        F: FnOnce(&mut Self) -> R,
+    {
+        debug_assert!(pos < self.opaque.data.len());
+
+        let new_opaque = MemDecoder::new(self.opaque.data, pos);
+        let old_opaque = mem::replace(&mut self.opaque, new_opaque);
+        let r = f(self);
+        self.opaque = old_opaque;
+        r
+    }
+
+    fn decode_alloc_id(&mut self) -> interpret::AllocId {
+        let alloc_decoding_session = self.alloc_decoding_session;
+        alloc_decoding_session.decode_alloc_id(self)
+    }
+}
+
+rustc_middle::implement_ty_decoder!(CacheDecoder<'a, 'tcx>);
+
+// This ensures that the `Decodable<opaque::Decoder>::decode` specialization for `Vec<u8>` is used
+// when a `CacheDecoder` is passed to `Decodable::decode`. Unfortunately, we have to manually opt
+// into specializations this way, given how `CacheDecoder` and the decoding traits currently work.
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Vec<u8> {
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        Decodable::decode(&mut d.opaque)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for SyntaxContext {
+    fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        let syntax_contexts = decoder.syntax_contexts;
+        rustc_span::hygiene::decode_syntax_context(decoder, decoder.hygiene_context, |this, id| {
+            // This closure is invoked if we haven't already decoded the data for the `SyntaxContext` we are deserializing.
+            // We look up the position of the associated `SyntaxData` and decode it.
+            let pos = syntax_contexts.get(&id).unwrap();
+            this.with_position(pos.to_usize(), |decoder| {
+                let data: SyntaxContextData = decode_tagged(decoder, TAG_SYNTAX_CONTEXT);
+                data
+            })
+        })
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for ExpnId {
+    fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        let hash = ExpnHash::decode(decoder);
+        if hash.is_root() {
+            return ExpnId::root();
+        }
+
+        if let Some(expn_id) = ExpnId::from_hash(hash) {
+            return expn_id;
+        }
+
+        let krate = decoder.tcx.stable_crate_id_to_crate_num(hash.stable_crate_id());
+
+        let expn_id = if krate == LOCAL_CRATE {
+            // We look up the position of the associated `ExpnData` and decode it.
+            let pos = decoder
+                .expn_data
+                .get(&hash)
+                .unwrap_or_else(|| panic!("Bad hash {:?} (map {:?})", hash, decoder.expn_data));
+
+            let data: ExpnData = decoder
+                .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA));
+            let expn_id = rustc_span::hygiene::register_local_expn_id(data, hash);
+
+            #[cfg(debug_assertions)]
+            {
+                use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+                let local_hash: u64 = decoder.tcx.with_stable_hashing_context(|mut hcx| {
+                    let mut hasher = StableHasher::new();
+                    expn_id.expn_data().hash_stable(&mut hcx, &mut hasher);
+                    hasher.finish()
+                });
+                debug_assert_eq!(hash.local_hash(), local_hash);
+            }
+
+            expn_id
+        } else {
+            let index_guess = decoder.foreign_expn_data[&hash];
+            decoder.tcx.cstore_untracked().expn_hash_to_expn_id(
+                decoder.tcx.sess,
+                krate,
+                index_guess,
+                hash,
+            )
+        };
+
+        debug_assert_eq!(expn_id.krate, krate);
+        expn_id
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Span {
+    fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        let ctxt = SyntaxContext::decode(decoder);
+        let parent = Option::<LocalDefId>::decode(decoder);
+        let tag: u8 = Decodable::decode(decoder);
+
+        if tag == TAG_PARTIAL_SPAN {
+            return Span::new(BytePos(0), BytePos(0), ctxt, parent);
+        } else if tag == TAG_RELATIVE_SPAN {
+            let dlo = u32::decode(decoder);
+            let dto = u32::decode(decoder);
+
+            let enclosing = decoder.tcx.source_span_untracked(parent.unwrap()).data_untracked();
+            let span = Span::new(
+                enclosing.lo + BytePos::from_u32(dlo),
+                enclosing.lo + BytePos::from_u32(dto),
+                ctxt,
+                parent,
+            );
+
+            return span;
+        } else {
+            debug_assert_eq!(tag, TAG_FULL_SPAN);
+        }
+
+        let file_lo_index = SourceFileIndex::decode(decoder);
+        let line_lo = usize::decode(decoder);
+        let col_lo = BytePos::decode(decoder);
+        let len = BytePos::decode(decoder);
+
+        let file_lo = decoder.file_index_to_file(file_lo_index);
+        let lo = file_lo.lines(|lines| lines[line_lo - 1] + col_lo);
+        let hi = lo + len;
+
+        Span::new(lo, hi, ctxt, parent)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for CrateNum {
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        let stable_id = StableCrateId::decode(d);
+        let cnum = d.tcx.stable_crate_id_to_crate_num(stable_id);
+        cnum
+    }
+}
+
+// This impl makes sure that we get a runtime error when we try decode a
+// `DefIndex` that is not contained in a `DefId`. Such a case would be problematic
+// because we would not know how to transform the `DefIndex` to the current
+// context.
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefIndex {
+    fn decode(_d: &mut CacheDecoder<'a, 'tcx>) -> DefIndex {
+        panic!("trying to decode `DefIndex` outside the context of a `DefId`")
+    }
+}
+
+// Both the `CrateNum` and the `DefIndex` of a `DefId` can change in between two
+// compilation sessions. We use the `DefPathHash`, which is stable across
+// sessions, to map the old `DefId` to the new one.
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefId {
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        // Load the `DefPathHash` which is was we encoded the `DefId` as.
+        let def_path_hash = DefPathHash::decode(d);
+
+        // Using the `DefPathHash`, we can lookup the new `DefId`.
+        // Subtle: We only encode a `DefId` as part of a query result.
+        // If we get to this point, then all of the query inputs were green,
+        // which means that the definition with this hash is guaranteed to
+        // still exist in the current compilation session.
+        d.tcx.def_path_hash_to_def_id(def_path_hash, &mut || {
+            panic!("Failed to convert DefPathHash {:?}", def_path_hash)
+        })
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx FxHashSet<LocalDefId> {
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>>
+    for &'tcx IndexVec<mir::Promoted, mir::Body<'tcx>>
+{
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [ty::abstract_const::Node<'tcx>] {
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [(ty::Predicate<'tcx>, Span)] {
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [rustc_ast::InlineAsmTemplatePiece] {
+    fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+        RefDecodable::decode(d)
+    }
+}
+
+macro_rules! impl_ref_decoder {
+    (<$tcx:tt> $($ty:ty,)*) => {
+        $(impl<'a, $tcx> Decodable<CacheDecoder<'a, $tcx>> for &$tcx [$ty] {
+            fn decode(d: &mut CacheDecoder<'a, $tcx>) -> Self {
+                RefDecodable::decode(d)
+            }
+        })*
+    };
+}
+
+impl_ref_decoder! {<'tcx>
+    Span,
+    rustc_ast::Attribute,
+    rustc_span::symbol::Ident,
+    ty::Variance,
+    rustc_span::def_id::DefId,
+    rustc_span::def_id::LocalDefId,
+    (rustc_middle::middle::exported_symbols::ExportedSymbol<'tcx>, rustc_middle::middle::exported_symbols::SymbolExportInfo),
+}
+
+//- ENCODING -------------------------------------------------------------------
+
+/// An encoder that can write to the incremental compilation cache.
+pub struct CacheEncoder<'a, 'tcx> {
+    tcx: TyCtxt<'tcx>,
+    encoder: FileEncoder,
+    type_shorthands: FxHashMap<Ty<'tcx>, usize>,
+    predicate_shorthands: FxHashMap<ty::PredicateKind<'tcx>, usize>,
+    interpret_allocs: FxIndexSet<interpret::AllocId>,
+    source_map: CachingSourceMapView<'tcx>,
+    file_to_file_index: FxHashMap<*const SourceFile, SourceFileIndex>,
+    hygiene_context: &'a HygieneEncodeContext,
+}
+
+impl<'a, 'tcx> CacheEncoder<'a, 'tcx> {
+    fn source_file_index(&mut self, source_file: Lrc<SourceFile>) -> SourceFileIndex {
+        self.file_to_file_index[&(&*source_file as *const SourceFile)]
+    }
+
+    /// Encode something with additional information that allows to do some
+    /// sanity checks when decoding the data again. This method will first
+    /// encode the specified tag, then the given value, then the number of
+    /// bytes taken up by tag and value. On decoding, we can then verify that
+    /// we get the expected tag and read the expected number of bytes.
+    fn encode_tagged<T: Encodable<Self>, V: Encodable<Self>>(&mut self, tag: T, value: &V) {
+        let start_pos = self.position();
+
+        tag.encode(self);
+        value.encode(self);
+
+        let end_pos = self.position();
+        ((end_pos - start_pos) as u64).encode(self);
+    }
+
+    fn finish(self) -> Result<usize, io::Error> {
+        self.encoder.finish()
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for SyntaxContext {
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        rustc_span::hygiene::raw_encode_syntax_context(*self, s.hygiene_context, s);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for ExpnId {
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        s.hygiene_context.schedule_expn_data_for_encoding(*self);
+        self.expn_hash().encode(s);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for Span {
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        let span_data = self.data_untracked();
+        span_data.ctxt.encode(s);
+        span_data.parent.encode(s);
+
+        if span_data.is_dummy() {
+            return TAG_PARTIAL_SPAN.encode(s);
+        }
+
+        if let Some(parent) = span_data.parent {
+            let enclosing = s.tcx.source_span(parent).data_untracked();
+            if enclosing.contains(span_data) {
+                TAG_RELATIVE_SPAN.encode(s);
+                (span_data.lo - enclosing.lo).to_u32().encode(s);
+                (span_data.hi - enclosing.lo).to_u32().encode(s);
+                return;
+            }
+        }
+
+        let pos = s.source_map.byte_pos_to_line_and_col(span_data.lo);
+        let partial_span = match &pos {
+            Some((file_lo, _, _)) => !file_lo.contains(span_data.hi),
+            None => true,
+        };
+
+        if partial_span {
+            return TAG_PARTIAL_SPAN.encode(s);
+        }
+
+        let (file_lo, line_lo, col_lo) = pos.unwrap();
+
+        let len = span_data.hi - span_data.lo;
+
+        let source_file_index = s.source_file_index(file_lo);
+
+        TAG_FULL_SPAN.encode(s);
+        source_file_index.encode(s);
+        line_lo.encode(s);
+        col_lo.encode(s);
+        len.encode(s);
+    }
+}
+
+impl<'a, 'tcx> TyEncoder for CacheEncoder<'a, 'tcx> {
+    type I = TyCtxt<'tcx>;
+    const CLEAR_CROSS_CRATE: bool = false;
+
+    fn position(&self) -> usize {
+        self.encoder.position()
+    }
+    fn type_shorthands(&mut self) -> &mut FxHashMap<Ty<'tcx>, usize> {
+        &mut self.type_shorthands
+    }
+    fn predicate_shorthands(&mut self) -> &mut FxHashMap<ty::PredicateKind<'tcx>, usize> {
+        &mut self.predicate_shorthands
+    }
+    fn encode_alloc_id(&mut self, alloc_id: &interpret::AllocId) {
+        let (index, _) = self.interpret_allocs.insert_full(*alloc_id);
+
+        index.encode(self);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for CrateNum {
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        s.tcx.stable_crate_id(*self).encode(s);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for DefId {
+    fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+        s.tcx.def_path_hash(*self).encode(s);
+    }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for DefIndex {
+    fn encode(&self, _: &mut CacheEncoder<'a, 'tcx>) {
+        bug!("encoding `DefIndex` without context");
+    }
+}
+
+macro_rules! encoder_methods {
+    ($($name:ident($ty:ty);)*) => {
+        #[inline]
+        $(fn $name(&mut self, value: $ty) {
+            self.encoder.$name(value)
+        })*
+    }
+}
+
+impl<'a, 'tcx> Encoder for CacheEncoder<'a, 'tcx> {
+    encoder_methods! {
+        emit_usize(usize);
+        emit_u128(u128);
+        emit_u64(u64);
+        emit_u32(u32);
+        emit_u16(u16);
+        emit_u8(u8);
+
+        emit_isize(isize);
+        emit_i128(i128);
+        emit_i64(i64);
+        emit_i32(i32);
+        emit_i16(i16);
+        emit_i8(i8);
+
+        emit_bool(bool);
+        emit_f64(f64);
+        emit_f32(f32);
+        emit_char(char);
+        emit_str(&str);
+        emit_raw_bytes(&[u8]);
+    }
+}
+
+// This ensures that the `Encodable<opaque::FileEncoder>::encode` specialization for byte slices
+// is used when a `CacheEncoder` having an `opaque::FileEncoder` is passed to `Encodable::encode`.
+// Unfortunately, we have to manually opt into specializations this way, given how `CacheEncoder`
+// and the encoding traits currently work.
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for [u8] {
+    fn encode(&self, e: &mut CacheEncoder<'a, 'tcx>) {
+        self.encode(&mut e.encoder);
+    }
+}
+
+pub fn encode_query_results<'a, 'tcx, CTX, Q>(
+    tcx: CTX,
+    encoder: &mut CacheEncoder<'a, 'tcx>,
+    query_result_index: &mut EncodedDepNodeIndex,
+) where
+    CTX: QueryContext + 'tcx,
+    Q: super::QueryDescription<CTX>,
+    Q::Value: Encodable<CacheEncoder<'a, 'tcx>>,
+{
+    let _timer = tcx
+        .dep_context()
+        .profiler()
+        .extra_verbose_generic_activity("encode_query_results_for", std::any::type_name::<Q>());
+
+    assert!(Q::query_state(tcx).all_inactive());
+    let cache = Q::query_cache(tcx);
+    cache.iter(&mut |key, value, dep_node| {
+        if Q::cache_on_disk(*tcx.dep_context(), &key) {
+            let dep_node = SerializedDepNodeIndex::new(dep_node.index());
+
+            // Record position of the cache entry.
+            query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.encoder.position())));
+
+            // Encode the type check tables with the `SerializedDepNodeIndex`
+            // as tag.
+            encoder.encode_tagged(dep_node, value);
+        }
+    });
+}
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
new file mode 100644
index 000000000..eda4401c8
--- /dev/null
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -0,0 +1,558 @@
+//! The implementation of the query system itself. This defines the macros that
+//! generate the actual methods on tcx which find and execute the provider,
+//! manage the caches, and so forth.
+
+use crate::{on_disk_cache, Queries};
+use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use rustc_middle::ty::tls::{self, ImplicitCtxt};
+use rustc_middle::ty::TyCtxt;
+use rustc_query_system::dep_graph::HasDepContext;
+use rustc_query_system::query::{QueryContext, QueryJobId, QueryMap, QuerySideEffects};
+
+use rustc_data_structures::sync::Lock;
+use rustc_data_structures::thin_vec::ThinVec;
+use rustc_errors::{Diagnostic, Handler};
+
+use std::any::Any;
+use std::num::NonZeroU64;
+
+#[derive(Copy, Clone)]
+pub struct QueryCtxt<'tcx> {
+    pub tcx: TyCtxt<'tcx>,
+    pub queries: &'tcx Queries<'tcx>,
+}
+
+impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> {
+    type Target = TyCtxt<'tcx>;
+
+    #[inline]
+    fn deref(&self) -> &Self::Target {
+        &self.tcx
+    }
+}
+
+impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
+    type DepKind = rustc_middle::dep_graph::DepKind;
+    type DepContext = TyCtxt<'tcx>;
+
+    #[inline]
+    fn dep_context(&self) -> &Self::DepContext {
+        &self.tcx
+    }
+}
+
+impl QueryContext for QueryCtxt<'_> {
+    fn next_job_id(&self) -> QueryJobId {
+        QueryJobId(
+            NonZeroU64::new(
+                self.queries.jobs.fetch_add(1, rustc_data_structures::sync::Ordering::Relaxed),
+            )
+            .unwrap(),
+        )
+    }
+
+    fn current_query_job(&self) -> Option<QueryJobId> {
+        tls::with_related_context(**self, |icx| icx.query)
+    }
+
+    fn try_collect_active_jobs(&self) -> Option<QueryMap> {
+        self.queries.try_collect_active_jobs(**self)
+    }
+
+    // Interactions with on_disk_cache
+    fn load_side_effects(&self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects {
+        self.queries
+            .on_disk_cache
+            .as_ref()
+            .map(|c| c.load_side_effects(**self, prev_dep_node_index))
+            .unwrap_or_default()
+    }
+
+    fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
+        if let Some(c) = self.queries.on_disk_cache.as_ref() {
+            c.store_side_effects(dep_node_index, side_effects)
+        }
+    }
+
+    fn store_side_effects_for_anon_node(
+        &self,
+        dep_node_index: DepNodeIndex,
+        side_effects: QuerySideEffects,
+    ) {
+        if let Some(c) = self.queries.on_disk_cache.as_ref() {
+            c.store_side_effects_for_anon_node(dep_node_index, side_effects)
+        }
+    }
+
+    /// Executes a job by changing the `ImplicitCtxt` to point to the
+    /// new query job while it executes. It returns the diagnostics
+    /// captured during execution and the actual result.
+    #[inline(always)]
+    fn start_query<R>(
+        &self,
+        token: QueryJobId,
+        diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
+        compute: impl FnOnce() -> R,
+    ) -> R {
+        // The `TyCtxt` stored in TLS has the same global interner lifetime
+        // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
+        // when accessing the `ImplicitCtxt`.
+        tls::with_related_context(**self, move |current_icx| {
+            // Update the `ImplicitCtxt` to point to our new query job.
+            let new_icx = ImplicitCtxt {
+                tcx: **self,
+                query: Some(token),
+                diagnostics,
+                layout_depth: current_icx.layout_depth,
+                task_deps: current_icx.task_deps,
+            };
+
+            // Use the `ImplicitCtxt` while we execute the query.
+            tls::enter_context(&new_icx, |_| {
+                rustc_data_structures::stack::ensure_sufficient_stack(compute)
+            })
+        })
+    }
+}
+
+impl<'tcx> QueryCtxt<'tcx> {
+    #[inline]
+    pub fn from_tcx(tcx: TyCtxt<'tcx>) -> Self {
+        let queries = tcx.queries.as_any();
+        let queries = unsafe {
+            let queries = std::mem::transmute::<&dyn Any, &dyn Any>(queries);
+            let queries = queries.downcast_ref().unwrap();
+            let queries = std::mem::transmute::<&Queries<'_>, &Queries<'_>>(queries);
+            queries
+        };
+        QueryCtxt { tcx, queries }
+    }
+
+    pub(crate) fn on_disk_cache(self) -> Option<&'tcx on_disk_cache::OnDiskCache<'tcx>> {
+        self.queries.on_disk_cache.as_ref()
+    }
+
+    pub(super) fn encode_query_results(
+        self,
+        encoder: &mut on_disk_cache::CacheEncoder<'_, 'tcx>,
+        query_result_index: &mut on_disk_cache::EncodedDepNodeIndex,
+    ) {
+        macro_rules! encode_queries {
+            ($($query:ident,)*) => {
+                $(
+                    on_disk_cache::encode_query_results::<_, super::queries::$query<'_>>(
+                        self,
+                        encoder,
+                        query_result_index
+                    );
+                )*
+            }
+        }
+
+        rustc_cached_queries!(encode_queries!);
+    }
+
+    pub fn try_print_query_stack(
+        self,
+        query: Option<QueryJobId>,
+        handler: &Handler,
+        num_frames: Option<usize>,
+    ) -> usize {
+        rustc_query_system::query::print_query_stack(self, query, handler, num_frames)
+    }
+}
+
+macro_rules! handle_cycle_error {
+    ([][$tcx: expr, $error:expr]) => {{
+        $error.emit();
+        Value::from_cycle_error($tcx)
+    }};
+    ([(fatal_cycle) $($rest:tt)*][$tcx:expr, $error:expr]) => {{
+        $error.emit();
+        $tcx.sess.abort_if_errors();
+        unreachable!()
+    }};
+    ([(cycle_delay_bug) $($rest:tt)*][$tcx:expr, $error:expr]) => {{
+        $error.delay_as_bug();
+        Value::from_cycle_error($tcx)
+    }};
+    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
+        handle_cycle_error!([$($modifiers)*][$($args)*])
+    };
+}
+
+macro_rules! is_anon {
+    ([]) => {{
+        false
+    }};
+    ([(anon) $($rest:tt)*]) => {{
+        true
+    }};
+    ([$other:tt $($modifiers:tt)*]) => {
+        is_anon!([$($modifiers)*])
+    };
+}
+
+macro_rules! is_eval_always {
+    ([]) => {{
+        false
+    }};
+    ([(eval_always) $($rest:tt)*]) => {{
+        true
+    }};
+    ([$other:tt $($modifiers:tt)*]) => {
+        is_eval_always!([$($modifiers)*])
+    };
+}
+
+macro_rules! hash_result {
+    ([]) => {{
+        Some(dep_graph::hash_result)
+    }};
+    ([(no_hash) $($rest:tt)*]) => {{
+        None
+    }};
+    ([$other:tt $($modifiers:tt)*]) => {
+        hash_result!([$($modifiers)*])
+    };
+}
+
+macro_rules! get_provider {
+    ([][$tcx:expr, $name:ident, $key:expr]) => {{
+        $tcx.queries.local_providers.$name
+    }};
+    ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
+        if $key.query_crate_is_local() {
+            $tcx.queries.local_providers.$name
+        } else {
+            $tcx.queries.extern_providers.$name
+        }
+    }};
+    ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
+        get_provider!([$($modifiers)*][$($args)*])
+    };
+}
+
+macro_rules! opt_remap_env_constness {
+    ([][$name:ident]) => {};
+    ([(remap_env_constness) $($rest:tt)*][$name:ident]) => {
+        let $name = $name.without_const();
+    };
+    ([$other:tt $($modifiers:tt)*][$name:ident]) => {
+        opt_remap_env_constness!([$($modifiers)*][$name])
+    };
+}
+
+macro_rules! define_queries {
+    (<$tcx:tt>
+     $($(#[$attr:meta])*
+        [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
+
+        define_queries_struct! {
+            tcx: $tcx,
+            input: ($(([$($modifiers)*] [$($attr)*] [$name]))*)
+        }
+
+        mod make_query {
+            use super::*;
+
+            // Create an eponymous constructor for each query.
+            $(#[allow(nonstandard_style)] $(#[$attr])*
+            pub fn $name<$tcx>(tcx: QueryCtxt<$tcx>, key: query_keys::$name<$tcx>) -> QueryStackFrame {
+                opt_remap_env_constness!([$($modifiers)*][key]);
+                let kind = dep_graph::DepKind::$name;
+                let name = stringify!($name);
+                // Disable visible paths printing for performance reasons.
+                // Showing visible path instead of any path is not that important in production.
+                let description = ty::print::with_no_visible_paths!(
+                    // Force filename-line mode to avoid invoking `type_of` query.
+                    ty::print::with_forced_impl_filename_line!(
+                        queries::$name::describe(tcx, key)
+                    )
+                );
+                let description = if tcx.sess.verbose() {
+                    format!("{} [{}]", description, name)
+                } else {
+                    description
+                };
+                let span = if kind == dep_graph::DepKind::def_span {
+                    // The `def_span` query is used to calculate `default_span`,
+                    // so exit to avoid infinite recursion.
+                    None
+                } else {
+                    Some(key.default_span(*tcx))
+                };
+                let def_kind = if kind == dep_graph::DepKind::opt_def_kind {
+                    // Try to avoid infinite recursion.
+                    None
+                } else {
+                    key.key_as_def_id()
+                        .and_then(|def_id| def_id.as_local())
+                        .and_then(|def_id| tcx.opt_def_kind(def_id))
+                };
+                let hash = || {
+                    tcx.with_stable_hashing_context(|mut hcx|{
+                        let mut hasher = StableHasher::new();
+                        std::mem::discriminant(&kind).hash_stable(&mut hcx, &mut hasher);
+                        key.hash_stable(&mut hcx, &mut hasher);
+                        hasher.finish::<u64>()
+                    })
+                };
+
+                QueryStackFrame::new(name, description, span, def_kind, hash)
+            })*
+        }
+
+        #[allow(nonstandard_style)]
+        mod queries {
+            use std::marker::PhantomData;
+
+            $(pub struct $name<$tcx> {
+                data: PhantomData<&$tcx ()>
+            })*
+        }
+
+        $(impl<$tcx> QueryConfig for queries::$name<$tcx> {
+            type Key = query_keys::$name<$tcx>;
+            type Value = query_values::$name<$tcx>;
+            type Stored = query_stored::$name<$tcx>;
+            const NAME: &'static str = stringify!($name);
+        }
+
+        impl<$tcx> QueryDescription<QueryCtxt<$tcx>> for queries::$name<$tcx> {
+            rustc_query_description! { $name<$tcx> }
+
+            type Cache = query_storage::$name<$tcx>;
+
+            #[inline(always)]
+            fn query_state<'a>(tcx: QueryCtxt<$tcx>) -> &'a QueryState<Self::Key>
+                where QueryCtxt<$tcx>: 'a
+            {
+                &tcx.queries.$name
+            }
+
+            #[inline(always)]
+            fn query_cache<'a>(tcx: QueryCtxt<$tcx>) -> &'a Self::Cache
+                where 'tcx:'a
+            {
+                &tcx.query_caches.$name
+            }
+
+            #[inline]
+            fn make_vtable(tcx: QueryCtxt<'tcx>, key: &Self::Key) ->
+                QueryVTable<QueryCtxt<$tcx>, Self::Key, Self::Value>
+            {
+                let compute = get_provider!([$($modifiers)*][tcx, $name, key]);
+                let cache_on_disk = Self::cache_on_disk(tcx.tcx, key);
+                QueryVTable {
+                    anon: is_anon!([$($modifiers)*]),
+                    eval_always: is_eval_always!([$($modifiers)*]),
+                    dep_kind: dep_graph::DepKind::$name,
+                    hash_result: hash_result!([$($modifiers)*]),
+                    handle_cycle_error: |tcx, mut error| handle_cycle_error!([$($modifiers)*][tcx, error]),
+                    compute,
+                    cache_on_disk,
+                    try_load_from_disk: Self::TRY_LOAD_FROM_DISK,
+                }
+            }
+        })*
+
+        #[allow(nonstandard_style)]
+        mod query_callbacks {
+            use super::*;
+            use rustc_middle::dep_graph::DepNode;
+            use rustc_middle::ty::query::query_keys;
+            use rustc_query_system::dep_graph::DepNodeParams;
+            use rustc_query_system::query::{force_query, QueryDescription};
+            use rustc_query_system::dep_graph::FingerprintStyle;
+
+            // We use this for most things when incr. comp. is turned off.
+            pub fn Null() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Unit,
+                    force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
+                    try_load_from_on_disk_cache: None,
+                }
+            }
+
+            // We use this for the forever-red node.
+            pub fn Red() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Unit,
+                    force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
+                    try_load_from_on_disk_cache: None,
+                }
+            }
+
+            pub fn TraitSelect() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: true,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Unit,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                }
+            }
+
+            pub fn CompileCodegenUnit() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Opaque,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                }
+            }
+
+            pub fn CompileMonoItem() -> DepKindStruct {
+                DepKindStruct {
+                    is_anon: false,
+                    is_eval_always: false,
+                    fingerprint_style: FingerprintStyle::Opaque,
+                    force_from_dep_node: None,
+                    try_load_from_on_disk_cache: None,
+                }
+            }
+
+            $(pub(crate) fn $name()-> DepKindStruct {
+                let is_anon = is_anon!([$($modifiers)*]);
+                let is_eval_always = is_eval_always!([$($modifiers)*]);
+
+                let fingerprint_style =
+                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::fingerprint_style();
+
+                if is_anon || !fingerprint_style.reconstructible() {
+                    return DepKindStruct {
+                        is_anon,
+                        is_eval_always,
+                        fingerprint_style,
+                        force_from_dep_node: None,
+                        try_load_from_on_disk_cache: None,
+                    }
+                }
+
+                #[inline(always)]
+                fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> Option<query_keys::$name<'tcx>> {
+                    <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, &dep_node)
+                }
+
+                fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: DepNode) -> bool {
+                    if let Some(key) = recover(tcx, dep_node) {
+                        #[cfg(debug_assertions)]
+                        let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
+                        let tcx = QueryCtxt::from_tcx(tcx);
+                        force_query::<queries::$name<'_>, _>(tcx, key, dep_node);
+                        true
+                    } else {
+                        false
+                    }
+                }
+
+                fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: DepNode) {
+                    debug_assert!(tcx.dep_graph.is_green(&dep_node));
+
+                    let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
+                    if queries::$name::cache_on_disk(tcx, &key) {
+                        let _ = tcx.$name(key);
+                    }
+                }
+
+                DepKindStruct {
+                    is_anon,
+                    is_eval_always,
+                    fingerprint_style,
+                    force_from_dep_node: Some(force_from_dep_node),
+                    try_load_from_on_disk_cache: Some(try_load_from_on_disk_cache),
+                }
+            })*
+        }
+
+        pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct] {
+            arena.alloc_from_iter(make_dep_kind_array!(query_callbacks))
+        }
+    }
+}
+
+// FIXME(eddyb) this macro (and others?) use `$tcx` and `'tcx` interchangeably.
+// We should either not take `$tcx` at all and use `'tcx` everywhere, or use
+// `$tcx` everywhere (even if that isn't necessary due to lack of hygiene).
+macro_rules! define_queries_struct {
+    (tcx: $tcx:tt,
+     input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
+        pub struct Queries<$tcx> {
+            local_providers: Box<Providers>,
+            extern_providers: Box<ExternProviders>,
+
+            pub on_disk_cache: Option<OnDiskCache<$tcx>>,
+
+            jobs: AtomicU64,
+
+            $($(#[$attr])*  $name: QueryState<query_keys::$name<$tcx>>,)*
+        }
+
+        impl<$tcx> Queries<$tcx> {
+            pub fn new(
+                local_providers: Providers,
+                extern_providers: ExternProviders,
+                on_disk_cache: Option<OnDiskCache<$tcx>>,
+            ) -> Self {
+                Queries {
+                    local_providers: Box::new(local_providers),
+                    extern_providers: Box::new(extern_providers),
+                    on_disk_cache,
+                    jobs: AtomicU64::new(1),
+                    $($name: Default::default()),*
+                }
+            }
+
+            pub(crate) fn try_collect_active_jobs(
+                &$tcx self,
+                tcx: TyCtxt<$tcx>,
+            ) -> Option<QueryMap> {
+                let tcx = QueryCtxt { tcx, queries: self };
+                let mut jobs = QueryMap::default();
+
+                $(
+                    self.$name.try_collect_active_jobs(
+                        tcx,
+                        make_query::$name,
+                        &mut jobs,
+                    )?;
+                )*
+
+                Some(jobs)
+            }
+        }
+
+        impl<'tcx> QueryEngine<'tcx> for Queries<'tcx> {
+            fn as_any(&'tcx self) -> &'tcx dyn std::any::Any {
+                let this = unsafe { std::mem::transmute::<&Queries<'_>, &Queries<'_>>(self) };
+                this as _
+            }
+
+            fn try_mark_green(&'tcx self, tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
+                let qcx = QueryCtxt { tcx, queries: self };
+                tcx.dep_graph.try_mark_green(qcx, dep_node).is_some()
+            }
+
+            $($(#[$attr])*
+            #[inline(always)]
+            #[tracing::instrument(level = "trace", skip(self, tcx))]
+            fn $name(
+                &'tcx self,
+                tcx: TyCtxt<$tcx>,
+                span: Span,
+                key: query_keys::$name<$tcx>,
+                mode: QueryMode,
+            ) -> Option<query_stored::$name<$tcx>> {
+                opt_remap_env_constness!([$($modifiers)*][key]);
+                let qcx = QueryCtxt { tcx, queries: self };
+                get_query::<queries::$name<$tcx>, _>(qcx, span, key, mode)
+            })*
+        }
+    };
+}
diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/profiling_support.rs
new file mode 100644
index 000000000..551f09420
--- /dev/null
+++ b/compiler/rustc_query_impl/src/profiling_support.rs
@@ -0,0 +1,324 @@
+use measureme::{StringComponent, StringId};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::profiling::SelfProfiler;
+use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE};
+use rustc_hir::definitions::DefPathData;
+use rustc_middle::ty::{TyCtxt, WithOptConstParam};
+use rustc_query_system::query::QueryCache;
+use std::fmt::Debug;
+use std::io::Write;
+
+struct QueryKeyStringCache {
+    def_id_cache: FxHashMap<DefId, StringId>,
+}
+
+impl QueryKeyStringCache {
+    fn new() -> QueryKeyStringCache {
+        QueryKeyStringCache { def_id_cache: Default::default() }
+    }
+}
+
+struct QueryKeyStringBuilder<'p, 'c, 'tcx> {
+    profiler: &'p SelfProfiler,
+    tcx: TyCtxt<'tcx>,
+    string_cache: &'c mut QueryKeyStringCache,
+}
+
+impl<'p, 'c, 'tcx> QueryKeyStringBuilder<'p, 'c, 'tcx> {
+    fn new(
+        profiler: &'p SelfProfiler,
+        tcx: TyCtxt<'tcx>,
+        string_cache: &'c mut QueryKeyStringCache,
+    ) -> QueryKeyStringBuilder<'p, 'c, 'tcx> {
+        QueryKeyStringBuilder { profiler, tcx, string_cache }
+    }
+
+    // The current implementation is rather crude. In the future it might be a
+    // good idea to base this on `ty::print` in order to get nicer and more
+    // efficient query keys.
+    fn def_id_to_string_id(&mut self, def_id: DefId) -> StringId {
+        if let Some(&string_id) = self.string_cache.def_id_cache.get(&def_id) {
+            return string_id;
+        }
+
+        let def_key = self.tcx.def_key(def_id);
+
+        let (parent_string_id, start_index) = match def_key.parent {
+            Some(parent_index) => {
+                let parent_def_id = DefId { index: parent_index, krate: def_id.krate };
+
+                (self.def_id_to_string_id(parent_def_id), 0)
+            }
+            None => (StringId::INVALID, 2),
+        };
+
+        let dis_buffer = &mut [0u8; 16];
+        let crate_name;
+        let other_name;
+        let name;
+        let dis;
+        let end_index;
+
+        match def_key.disambiguated_data.data {
+            DefPathData::CrateRoot => {
+                crate_name = self.tcx.crate_name(def_id.krate);
+                name = crate_name.as_str();
+                dis = "";
+                end_index = 3;
+            }
+            other => {
+                other_name = other.to_string();
+                name = other_name.as_str();
+                if def_key.disambiguated_data.disambiguator == 0 {
+                    dis = "";
+                    end_index = 3;
+                } else {
+                    write!(&mut dis_buffer[..], "[{}]", def_key.disambiguated_data.disambiguator)
+                        .unwrap();
+                    let end_of_dis = dis_buffer.iter().position(|&c| c == b']').unwrap();
+                    dis = std::str::from_utf8(&dis_buffer[..end_of_dis + 1]).unwrap();
+                    end_index = 4;
+                }
+            }
+        }
+
+        let components = [
+            StringComponent::Ref(parent_string_id),
+            StringComponent::Value("::"),
+            StringComponent::Value(name),
+            StringComponent::Value(dis),
+        ];
+
+        let string_id = self.profiler.alloc_string(&components[start_index..end_index]);
+
+        self.string_cache.def_id_cache.insert(def_id, string_id);
+
+        string_id
+    }
+}
+
+trait IntoSelfProfilingString {
+    fn to_self_profile_string(&self, builder: &mut QueryKeyStringBuilder<'_, '_, '_>) -> StringId;
+}
+
+// The default implementation of `IntoSelfProfilingString` just uses `Debug`
+// which is slow and causes lots of duplication of string data.
+// The specialized impls below take care of making the `DefId` case more
+// efficient.
+impl<T: Debug> IntoSelfProfilingString for T {
+    default fn to_self_profile_string(
+        &self,
+        builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+    ) -> StringId {
+        let s = format!("{:?}", self);
+        builder.profiler.alloc_string(&s[..])
+    }
+}
+
+impl<T: SpecIntoSelfProfilingString> IntoSelfProfilingString for T {
+    fn to_self_profile_string(&self, builder: &mut QueryKeyStringBuilder<'_, '_, '_>) -> StringId {
+        self.spec_to_self_profile_string(builder)
+    }
+}
+
+#[rustc_specialization_trait]
+trait SpecIntoSelfProfilingString: Debug {
+    fn spec_to_self_profile_string(
+        &self,
+        builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+    ) -> StringId;
+}
+
+impl SpecIntoSelfProfilingString for DefId {
+    fn spec_to_self_profile_string(
+        &self,
+        builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+    ) -> StringId {
+        builder.def_id_to_string_id(*self)
+    }
+}
+
+impl SpecIntoSelfProfilingString for CrateNum {
+    fn spec_to_self_profile_string(
+        &self,
+        builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+    ) -> StringId {
+        builder.def_id_to_string_id(self.as_def_id())
+    }
+}
+
+impl SpecIntoSelfProfilingString for DefIndex {
+    fn spec_to_self_profile_string(
+        &self,
+        builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+    ) -> StringId {
+        builder.def_id_to_string_id(DefId { krate: LOCAL_CRATE, index: *self })
+    }
+}
+
+impl SpecIntoSelfProfilingString for LocalDefId {
+    fn spec_to_self_profile_string(
+        &self,
+        builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+    ) -> StringId {
+        builder.def_id_to_string_id(DefId { krate: LOCAL_CRATE, index: self.local_def_index })
+    }
+}
+
+impl<T: SpecIntoSelfProfilingString> SpecIntoSelfProfilingString for WithOptConstParam<T> {
+    fn spec_to_self_profile_string(
+        &self,
+        builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+    ) -> StringId {
+        // We print `WithOptConstParam` values as tuples to make them shorter
+        // and more readable, without losing information:
+        //
+        // "WithOptConstParam { did: foo::bar, const_param_did: Some(foo::baz) }"
+        // becomes "(foo::bar, foo::baz)" and
+        // "WithOptConstParam { did: foo::bar, const_param_did: None }"
+        // becomes "(foo::bar, _)".
+
+        let did = StringComponent::Ref(self.did.to_self_profile_string(builder));
+
+        let const_param_did = if let Some(const_param_did) = self.const_param_did {
+            let const_param_did = builder.def_id_to_string_id(const_param_did);
+            StringComponent::Ref(const_param_did)
+        } else {
+            StringComponent::Value("_")
+        };
+
+        let components = [
+            StringComponent::Value("("),
+            did,
+            StringComponent::Value(", "),
+            const_param_did,
+            StringComponent::Value(")"),
+        ];
+
+        builder.profiler.alloc_string(&components[..])
+    }
+}
+
+impl<T0, T1> SpecIntoSelfProfilingString for (T0, T1)
+where
+    T0: SpecIntoSelfProfilingString,
+    T1: SpecIntoSelfProfilingString,
+{
+    fn spec_to_self_profile_string(
+        &self,
+        builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+    ) -> StringId {
+        let val0 = self.0.to_self_profile_string(builder);
+        let val1 = self.1.to_self_profile_string(builder);
+
+        let components = &[
+            StringComponent::Value("("),
+            StringComponent::Ref(val0),
+            StringComponent::Value(","),
+            StringComponent::Ref(val1),
+            StringComponent::Value(")"),
+        ];
+
+        builder.profiler.alloc_string(components)
+    }
+}
+
+/// Allocate the self-profiling query strings for a single query cache. This
+/// method is called from `alloc_self_profile_query_strings` which knows all
+/// the queries via macro magic.
+fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>(
+    tcx: TyCtxt<'tcx>,
+    query_name: &'static str,
+    query_cache: &C,
+    string_cache: &mut QueryKeyStringCache,
+) where
+    C: QueryCache,
+    C::Key: Debug + Clone,
+{
+    tcx.prof.with_profiler(|profiler| {
+        let event_id_builder = profiler.event_id_builder();
+
+        // Walk the entire query cache and allocate the appropriate
+        // string representations. Each cache entry is uniquely
+        // identified by its dep_node_index.
+        if profiler.query_key_recording_enabled() {
+            let mut query_string_builder = QueryKeyStringBuilder::new(profiler, tcx, string_cache);
+
+            let query_name = profiler.get_or_alloc_cached_string(query_name);
+
+            // Since building the string representation of query keys might
+            // need to invoke queries itself, we cannot keep the query caches
+            // locked while doing so. Instead we copy out the
+            // `(query_key, dep_node_index)` pairs and release the lock again.
+            let mut query_keys_and_indices = Vec::new();
+            query_cache.iter(&mut |k, _, i| query_keys_and_indices.push((k.clone(), i)));
+
+            // Now actually allocate the strings. If allocating the strings
+            // generates new entries in the query cache, we'll miss them but
+            // we don't actually care.
+            for (query_key, dep_node_index) in query_keys_and_indices {
+                // Translate the DepNodeIndex into a QueryInvocationId
+                let query_invocation_id = dep_node_index.into();
+
+                // Create the string version of the query-key
+                let query_key = query_key.to_self_profile_string(&mut query_string_builder);
+                let event_id = event_id_builder.from_label_and_arg(query_name, query_key);
+
+                // Doing this in bulk might be a good idea:
+                profiler.map_query_invocation_id_to_string(
+                    query_invocation_id,
+                    event_id.to_string_id(),
+                );
+            }
+        } else {
+            // In this branch we don't allocate query keys
+            let query_name = profiler.get_or_alloc_cached_string(query_name);
+            let event_id = event_id_builder.from_label(query_name).to_string_id();
+
+            // FIXME(eddyb) make this O(1) by using a pre-cached query name `EventId`,
+            // instead of passing the `DepNodeIndex` to `finish_with_query_invocation_id`,
+            // when recording the event in the first place.
+            let mut query_invocation_ids = Vec::new();
+            query_cache.iter(&mut |_, _, i| {
+                query_invocation_ids.push(i.into());
+            });
+
+            profiler.bulk_map_query_invocation_id_to_single_string(
+                query_invocation_ids.into_iter(),
+                event_id,
+            );
+        }
+    });
+}
+
+/// All self-profiling events generated by the query engine use
+/// virtual `StringId`s for their `event_id`. This method makes all
+/// those virtual `StringId`s point to actual strings.
+///
+/// If we are recording only summary data, the ids will point to
+/// just the query names. If we are recording query keys too, we
+/// allocate the corresponding strings here.
+pub fn alloc_self_profile_query_strings(tcx: TyCtxt<'_>) {
+    if !tcx.prof.enabled() {
+        return;
+    }
+
+    let mut string_cache = QueryKeyStringCache::new();
+
+    macro_rules! alloc_once {
+        (<$tcx:tt>
+            $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($K:ty) -> $V:ty,)*
+        ) => {
+            $({
+                alloc_self_profile_query_strings_for_query_cache(
+                    tcx,
+                    stringify!($name),
+                    &tcx.query_caches.$name,
+                    &mut string_cache,
+                );
+            })*
+        }
+    }
+
+    rustc_query_append! { [alloc_once!][<'tcx>] }
+}
diff --git a/compiler/rustc_query_impl/src/values.rs b/compiler/rustc_query_impl/src/values.rs
new file mode 100644
index 000000000..718a2971c
--- /dev/null
+++ b/compiler/rustc_query_impl/src/values.rs
@@ -0,0 +1,45 @@
+use super::QueryCtxt;
+use rustc_middle::ty::{self, AdtSizedConstraint, Ty};
+
+pub(super) trait Value<'tcx>: Sized {
+    fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self;
+}
+
+impl<'tcx, T> Value<'tcx> for T {
+    default fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> T {
+        tcx.sess.abort_if_errors();
+        bug!("Value::from_cycle_error called without errors");
+    }
+}
+
+impl<'tcx> Value<'tcx> for Ty<'_> {
+    fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self {
+        // SAFETY: This is never called when `Self` is not `Ty<'tcx>`.
+        // FIXME: Represent the above fact in the trait system somehow.
+        unsafe { std::mem::transmute::<Ty<'tcx>, Ty<'_>>(tcx.ty_error()) }
+    }
+}
+
+impl<'tcx> Value<'tcx> for ty::SymbolName<'_> {
+    fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self {
+        // SAFETY: This is never called when `Self` is not `SymbolName<'tcx>`.
+        // FIXME: Represent the above fact in the trait system somehow.
+        unsafe {
+            std::mem::transmute::<ty::SymbolName<'tcx>, ty::SymbolName<'_>>(ty::SymbolName::new(
+                *tcx, "<error>",
+            ))
+        }
+    }
+}
+
+impl<'tcx> Value<'tcx> for AdtSizedConstraint<'_> {
+    fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self {
+        // SAFETY: This is never called when `Self` is not `AdtSizedConstraint<'tcx>`.
+        // FIXME: Represent the above fact in the trait system somehow.
+        unsafe {
+            std::mem::transmute::<AdtSizedConstraint<'tcx>, AdtSizedConstraint<'_>>(
+                AdtSizedConstraint(tcx.intern_type_list(&[tcx.ty_error()])),
+            )
+        }
+    }
+}