summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_query_impl
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_query_impl')
-rw-r--r--compiler/rustc_query_impl/Cargo.toml26
-rw-r--r--compiler/rustc_query_impl/src/README.md3
-rw-r--r--compiler/rustc_query_impl/src/keys.rs545
-rw-r--r--compiler/rustc_query_impl/src/lib.rs63
-rw-r--r--compiler/rustc_query_impl/src/on_disk_cache.rs1012
-rw-r--r--compiler/rustc_query_impl/src/plumbing.rs558
-rw-r--r--compiler/rustc_query_impl/src/profiling_support.rs324
-rw-r--r--compiler/rustc_query_impl/src/values.rs45
8 files changed, 2576 insertions, 0 deletions
diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml
new file mode 100644
index 000000000..5673bb83b
--- /dev/null
+++ b/compiler/rustc_query_impl/Cargo.toml
@@ -0,0 +1,26 @@
+[package]
+name = "rustc_query_impl"
+version = "0.0.0"
+edition = "2021"
+
+[lib]
+doctest = false
+
+[dependencies]
+measureme = "10.0.0"
+rustc-rayon-core = { version = "0.4.0", optional = true }
+rustc_ast = { path = "../rustc_ast" }
+rustc_data_structures = { path = "../rustc_data_structures" }
+rustc_errors = { path = "../rustc_errors" }
+rustc_hir = { path = "../rustc_hir" }
+rustc_index = { path = "../rustc_index" }
+rustc_macros = { path = "../rustc_macros" }
+rustc_middle = { path = "../rustc_middle" }
+rustc_query_system = { path = "../rustc_query_system" }
+rustc_serialize = { path = "../rustc_serialize" }
+rustc_session = { path = "../rustc_session" }
+rustc_span = { path = "../rustc_span" }
+tracing = "0.1"
+
+[features]
+rustc_use_parallel_compiler = ["rustc-rayon-core", "rustc_query_system/rustc_use_parallel_compiler"]
diff --git a/compiler/rustc_query_impl/src/README.md b/compiler/rustc_query_impl/src/README.md
new file mode 100644
index 000000000..8ec07b9fd
--- /dev/null
+++ b/compiler/rustc_query_impl/src/README.md
@@ -0,0 +1,3 @@
+For more information about how the query system works, see the [rustc dev guide].
+
+[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/query.html
diff --git a/compiler/rustc_query_impl/src/keys.rs b/compiler/rustc_query_impl/src/keys.rs
new file mode 100644
index 000000000..49175e97f
--- /dev/null
+++ b/compiler/rustc_query_impl/src/keys.rs
@@ -0,0 +1,545 @@
+//! Defines the set of legal keys that can be used in queries.
+
+use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE};
+use rustc_middle::infer::canonical::Canonical;
+use rustc_middle::mir;
+use rustc_middle::traits;
+use rustc_middle::ty::fast_reject::SimplifiedType;
+use rustc_middle::ty::subst::{GenericArg, SubstsRef};
+use rustc_middle::ty::{self, layout::TyAndLayout, Ty, TyCtxt};
+use rustc_span::symbol::{Ident, Symbol};
+use rustc_span::{Span, DUMMY_SP};
+
+/// The `Key` trait controls what types can legally be used as the key
+/// for a query.
+pub trait Key {
+ /// Given an instance of this key, what crate is it referring to?
+ /// This is used to find the provider.
+ fn query_crate_is_local(&self) -> bool;
+
+ /// In the event that a cycle occurs, if no explicit span has been
+ /// given for a query with key `self`, what span should we use?
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span;
+
+ /// If the key is a [`DefId`] or `DefId`--equivalent, return that `DefId`.
+ /// Otherwise, return `None`.
+ fn key_as_def_id(&self) -> Option<DefId> {
+ None
+ }
+}
+
+impl Key for () {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for ty::InstanceDef<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ tcx.def_span(self.def_id())
+ }
+}
+
+impl<'tcx> Key for ty::Instance<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ tcx.def_span(self.def_id())
+ }
+}
+
+impl<'tcx> Key for mir::interpret::GlobalId<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.instance.default_span(tcx)
+ }
+}
+
+impl<'tcx> Key for (Ty<'tcx>, Option<ty::PolyExistentialTraitRef<'tcx>>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl Key for CrateNum {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ *self == LOCAL_CRATE
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl Key for LocalDefId {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.to_def_id().default_span(tcx)
+ }
+ fn key_as_def_id(&self) -> Option<DefId> {
+ Some(self.to_def_id())
+ }
+}
+
+impl Key for DefId {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ tcx.def_span(*self)
+ }
+ #[inline(always)]
+ fn key_as_def_id(&self) -> Option<DefId> {
+ Some(*self)
+ }
+}
+
+impl Key for ty::WithOptConstParam<LocalDefId> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.did.default_span(tcx)
+ }
+}
+
+impl Key for SimplifiedType {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl Key for (DefId, DefId) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.0.krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.1.default_span(tcx)
+ }
+}
+
+impl<'tcx> Key for (ty::Instance<'tcx>, LocalDefId) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.0.default_span(tcx)
+ }
+}
+
+impl Key for (DefId, LocalDefId) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.0.krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.1.default_span(tcx)
+ }
+}
+
+impl Key for (LocalDefId, DefId) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.0.default_span(tcx)
+ }
+}
+
+impl Key for (LocalDefId, LocalDefId) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.0.default_span(tcx)
+ }
+}
+
+impl Key for (DefId, Option<Ident>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.0.krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ tcx.def_span(self.0)
+ }
+ #[inline(always)]
+ fn key_as_def_id(&self) -> Option<DefId> {
+ Some(self.0)
+ }
+}
+
+impl Key for (DefId, LocalDefId, Ident) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.0.krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.1.default_span(tcx)
+ }
+}
+
+impl Key for (CrateNum, DefId) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.0 == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.1.default_span(tcx)
+ }
+}
+
+impl Key for (CrateNum, SimplifiedType) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.0 == LOCAL_CRATE
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl Key for (DefId, SimplifiedType) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.0.krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.0.default_span(tcx)
+ }
+}
+
+impl<'tcx> Key for SubstsRef<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for (DefId, SubstsRef<'tcx>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.0.krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.0.default_span(tcx)
+ }
+}
+
+impl<'tcx> Key for (ty::Unevaluated<'tcx, ()>, ty::Unevaluated<'tcx, ()>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ (self.0).def.did.krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ (self.0).def.did.default_span(tcx)
+ }
+}
+
+impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.0.default_span(tcx)
+ }
+}
+
+impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.1.def_id().krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ tcx.def_span(self.1.def_id())
+ }
+}
+
+impl<'tcx> Key for (ty::Const<'tcx>, mir::Field) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for mir::interpret::ConstAlloc<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for ty::PolyTraitRef<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.def_id().krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ tcx.def_span(self.def_id())
+ }
+}
+
+impl<'tcx> Key for ty::PolyExistentialTraitRef<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.def_id().krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ tcx.def_span(self.def_id())
+ }
+}
+
+impl<'tcx> Key for (ty::PolyTraitRef<'tcx>, ty::PolyTraitRef<'tcx>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.0.def_id().krate == LOCAL_CRATE
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ tcx.def_span(self.0.def_id())
+ }
+}
+
+impl<'tcx> Key for GenericArg<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for mir::ConstantKind<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for ty::Const<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for Ty<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for TyAndLayout<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for (Ty<'tcx>, Ty<'tcx>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for &'tcx ty::List<ty::Predicate<'tcx>> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for ty::ParamEnv<'tcx> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ self.value.query_crate_is_local()
+ }
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.value.default_span(tcx)
+ }
+}
+
+impl Key for Symbol {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl Key for Option<Symbol> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+ fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+/// Canonical query goals correspond to abstract trait operations that
+/// are not tied to any crate in particular.
+impl<'tcx, T> Key for Canonical<'tcx, T> {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl Key for (Symbol, u32, u32) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for (DefId, Ty<'tcx>, SubstsRef<'tcx>, ty::ParamEnv<'tcx>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for (ty::Predicate<'tcx>, traits::WellFormedLoc) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, _tcx: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for (ty::PolyFnSig<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
+
+impl<'tcx> Key for (ty::Instance<'tcx>, &'tcx ty::List<Ty<'tcx>>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, tcx: TyCtxt<'_>) -> Span {
+ self.0.default_span(tcx)
+ }
+}
+
+impl<'tcx> Key for (Ty<'tcx>, ty::ValTree<'tcx>) {
+ #[inline(always)]
+ fn query_crate_is_local(&self) -> bool {
+ true
+ }
+
+ fn default_span(&self, _: TyCtxt<'_>) -> Span {
+ DUMMY_SP
+ }
+}
diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs
new file mode 100644
index 000000000..eda61df77
--- /dev/null
+++ b/compiler/rustc_query_impl/src/lib.rs
@@ -0,0 +1,63 @@
+//! Support for serializing the dep-graph and reloading it.
+
+#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")]
+#![feature(min_specialization)]
+#![feature(never_type)]
+#![feature(once_cell)]
+#![feature(rustc_attrs)]
+#![recursion_limit = "256"]
+#![allow(rustc::potential_query_instability)]
+
+#[macro_use]
+extern crate rustc_macros;
+#[macro_use]
+extern crate rustc_middle;
+
+use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+use rustc_data_structures::sync::AtomicU64;
+use rustc_middle::arena::Arena;
+use rustc_middle::dep_graph::{self, DepKindStruct, SerializedDepNodeIndex};
+use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values};
+use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine};
+use rustc_middle::ty::{self, TyCtxt};
+use rustc_span::def_id::{LocalDefId, LOCAL_CRATE};
+use rustc_span::Span;
+
+#[macro_use]
+mod plumbing;
+pub use plumbing::QueryCtxt;
+use rustc_query_system::query::*;
+#[cfg(parallel_compiler)]
+pub use rustc_query_system::query::{deadlock, QueryContext};
+
+mod keys;
+use keys::Key;
+
+mod values;
+use self::values::Value;
+
+pub use rustc_query_system::query::QueryConfig;
+pub(crate) use rustc_query_system::query::{QueryDescription, QueryVTable};
+
+mod on_disk_cache;
+pub use on_disk_cache::OnDiskCache;
+
+mod profiling_support;
+pub use self::profiling_support::alloc_self_profile_query_strings;
+
+fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String {
+ if def_id.is_top_level_module() {
+ "top-level module".to_string()
+ } else {
+ format!("module `{}`", tcx.def_path_str(def_id.to_def_id()))
+ }
+}
+
+rustc_query_append! { [define_queries!][<'tcx>] }
+
+impl<'tcx> Queries<'tcx> {
+ // Force codegen in the dyn-trait transformation in this crate.
+ pub fn as_dyn(&'tcx self) -> &'tcx dyn QueryEngine<'tcx> {
+ self
+ }
+}
diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs
new file mode 100644
index 000000000..56fd90c98
--- /dev/null
+++ b/compiler/rustc_query_impl/src/on_disk_cache.rs
@@ -0,0 +1,1012 @@
+use crate::QueryCtxt;
+use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet};
+use rustc_data_structures::memmap::Mmap;
+use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, RwLock};
+use rustc_data_structures::unhash::UnhashMap;
+use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE};
+use rustc_hir::definitions::DefPathHash;
+use rustc_index::vec::{Idx, IndexVec};
+use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState};
+use rustc_middle::mir::{self, interpret};
+use rustc_middle::ty::codec::{RefDecodable, TyDecoder, TyEncoder};
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_query_system::dep_graph::DepContext;
+use rustc_query_system::query::{QueryCache, QueryContext, QuerySideEffects};
+use rustc_serialize::{
+ opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder},
+ Decodable, Decoder, Encodable, Encoder,
+};
+use rustc_session::Session;
+use rustc_span::hygiene::{
+ ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextData,
+};
+use rustc_span::source_map::{SourceMap, StableSourceFileId};
+use rustc_span::CachingSourceMapView;
+use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, SourceFile, Span};
+use std::io;
+use std::mem;
+
+const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE;
+
+// A normal span encoded with both location information and a `SyntaxContext`
+const TAG_FULL_SPAN: u8 = 0;
+// A partial span with no location information, encoded only with a `SyntaxContext`
+const TAG_PARTIAL_SPAN: u8 = 1;
+const TAG_RELATIVE_SPAN: u8 = 2;
+
+const TAG_SYNTAX_CONTEXT: u8 = 0;
+const TAG_EXPN_DATA: u8 = 1;
+
+/// Provides an interface to incremental compilation data cached from the
+/// previous compilation session. This data will eventually include the results
+/// of a few selected queries (like `typeck` and `mir_optimized`) and
+/// any side effects that have been emitted during a query.
+pub struct OnDiskCache<'sess> {
+ // The complete cache data in serialized form.
+ serialized_data: RwLock<Option<Mmap>>,
+
+ // Collects all `QuerySideEffects` created during the current compilation
+ // session.
+ current_side_effects: Lock<FxHashMap<DepNodeIndex, QuerySideEffects>>,
+
+ source_map: &'sess SourceMap,
+ file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
+
+ // Caches that are populated lazily during decoding.
+ file_index_to_file: Lock<FxHashMap<SourceFileIndex, Lrc<SourceFile>>>,
+
+ // A map from dep-node to the position of the cached query result in
+ // `serialized_data`.
+ query_result_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+
+ // A map from dep-node to the position of any associated `QuerySideEffects` in
+ // `serialized_data`.
+ prev_side_effects_index: FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+
+ alloc_decoding_state: AllocDecodingState,
+
+ // A map from syntax context ids to the position of their associated
+ // `SyntaxContextData`. We use a `u32` instead of a `SyntaxContext`
+ // to represent the fact that we are storing *encoded* ids. When we decode
+ // a `SyntaxContext`, a new id will be allocated from the global `HygieneData`,
+ // which will almost certainly be different than the serialized id.
+ syntax_contexts: FxHashMap<u32, AbsoluteBytePos>,
+ // A map from the `DefPathHash` of an `ExpnId` to the position
+ // of their associated `ExpnData`. Ideally, we would store a `DefId`,
+ // but we need to decode this before we've constructed a `TyCtxt` (which
+ // makes it difficult to decode a `DefId`).
+
+ // Note that these `DefPathHashes` correspond to both local and foreign
+ // `ExpnData` (e.g `ExpnData.krate` may not be `LOCAL_CRATE`). Alternatively,
+ // we could look up the `ExpnData` from the metadata of foreign crates,
+ // but it seemed easier to have `OnDiskCache` be independent of the `CStore`.
+ expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
+ // Additional information used when decoding hygiene data.
+ hygiene_context: HygieneDecodeContext,
+ // Maps `ExpnHash`es to their raw value from the *previous*
+ // compilation session. This is used as an initial 'guess' when
+ // we try to map an `ExpnHash` to its value in the current
+ // compilation session.
+ foreign_expn_data: UnhashMap<ExpnHash, u32>,
+}
+
+// This type is used only for serialization and deserialization.
+#[derive(Encodable, Decodable)]
+struct Footer {
+ file_index_to_stable_id: FxHashMap<SourceFileIndex, EncodedSourceFileId>,
+ query_result_index: EncodedDepNodeIndex,
+ side_effects_index: EncodedDepNodeIndex,
+ // The location of all allocations.
+ interpret_alloc_index: Vec<u32>,
+ // See `OnDiskCache.syntax_contexts`
+ syntax_contexts: FxHashMap<u32, AbsoluteBytePos>,
+ // See `OnDiskCache.expn_data`
+ expn_data: UnhashMap<ExpnHash, AbsoluteBytePos>,
+ foreign_expn_data: UnhashMap<ExpnHash, u32>,
+}
+
+pub type EncodedDepNodeIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>;
+
+#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Encodable, Decodable)]
+struct SourceFileIndex(u32);
+
+#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Encodable, Decodable)]
+pub struct AbsoluteBytePos(u32);
+
+impl AbsoluteBytePos {
+ fn new(pos: usize) -> AbsoluteBytePos {
+ debug_assert!(pos <= u32::MAX as usize);
+ AbsoluteBytePos(pos as u32)
+ }
+
+ fn to_usize(self) -> usize {
+ self.0 as usize
+ }
+}
+
+/// An `EncodedSourceFileId` is the same as a `StableSourceFileId` except that
+/// the source crate is represented as a [StableCrateId] instead of as a
+/// `CrateNum`. This way `EncodedSourceFileId` can be encoded and decoded
+/// without any additional context, i.e. with a simple `opaque::Decoder` (which
+/// is the only thing available when decoding the cache's [Footer].
+#[derive(Encodable, Decodable, Clone, Debug)]
+struct EncodedSourceFileId {
+ file_name_hash: u64,
+ stable_crate_id: StableCrateId,
+}
+
+impl EncodedSourceFileId {
+ fn translate(&self, tcx: TyCtxt<'_>) -> StableSourceFileId {
+ let cnum = tcx.stable_crate_id_to_crate_num(self.stable_crate_id);
+ StableSourceFileId { file_name_hash: self.file_name_hash, cnum }
+ }
+
+ fn new(tcx: TyCtxt<'_>, file: &SourceFile) -> EncodedSourceFileId {
+ let source_file_id = StableSourceFileId::new(file);
+ EncodedSourceFileId {
+ file_name_hash: source_file_id.file_name_hash,
+ stable_crate_id: tcx.stable_crate_id(source_file_id.cnum),
+ }
+ }
+}
+
+impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> {
+ /// Creates a new `OnDiskCache` instance from the serialized data in `data`.
+ fn new(sess: &'sess Session, data: Mmap, start_pos: usize) -> Self {
+ debug_assert!(sess.opts.incremental.is_some());
+
+ // Wrap in a scope so we can borrow `data`.
+ let footer: Footer = {
+ let mut decoder = MemDecoder::new(&data, start_pos);
+
+ // Decode the *position* of the footer, which can be found in the
+ // last 8 bytes of the file.
+ decoder.set_position(data.len() - IntEncodedWithFixedSize::ENCODED_SIZE);
+ let footer_pos = IntEncodedWithFixedSize::decode(&mut decoder).0 as usize;
+
+ // Decode the file footer, which contains all the lookup tables, etc.
+ decoder.set_position(footer_pos);
+
+ decode_tagged(&mut decoder, TAG_FILE_FOOTER)
+ };
+
+ Self {
+ serialized_data: RwLock::new(Some(data)),
+ file_index_to_stable_id: footer.file_index_to_stable_id,
+ file_index_to_file: Default::default(),
+ source_map: sess.source_map(),
+ current_side_effects: Default::default(),
+ query_result_index: footer.query_result_index.into_iter().collect(),
+ prev_side_effects_index: footer.side_effects_index.into_iter().collect(),
+ alloc_decoding_state: AllocDecodingState::new(footer.interpret_alloc_index),
+ syntax_contexts: footer.syntax_contexts,
+ expn_data: footer.expn_data,
+ foreign_expn_data: footer.foreign_expn_data,
+ hygiene_context: Default::default(),
+ }
+ }
+
+ fn new_empty(source_map: &'sess SourceMap) -> Self {
+ Self {
+ serialized_data: RwLock::new(None),
+ file_index_to_stable_id: Default::default(),
+ file_index_to_file: Default::default(),
+ source_map,
+ current_side_effects: Default::default(),
+ query_result_index: Default::default(),
+ prev_side_effects_index: Default::default(),
+ alloc_decoding_state: AllocDecodingState::new(Vec::new()),
+ syntax_contexts: FxHashMap::default(),
+ expn_data: UnhashMap::default(),
+ foreign_expn_data: UnhashMap::default(),
+ hygiene_context: Default::default(),
+ }
+ }
+
+ /// Execute all cache promotions and release the serialized backing Mmap.
+ ///
+ /// Cache promotions require invoking queries, which needs to read the serialized data.
+ /// In order to serialize the new on-disk cache, the former on-disk cache file needs to be
+ /// deleted, hence we won't be able to refer to its memmapped data.
+ fn drop_serialized_data(&self, tcx: TyCtxt<'_>) {
+ // Load everything into memory so we can write it out to the on-disk
+ // cache. The vast majority of cacheable query results should already
+ // be in memory, so this should be a cheap operation.
+ // Do this *before* we clone 'latest_foreign_def_path_hashes', since
+ // loading existing queries may cause us to create new DepNodes, which
+ // may in turn end up invoking `store_foreign_def_id_hash`
+ tcx.dep_graph.exec_cache_promotions(tcx);
+
+ *self.serialized_data.write() = None;
+ }
+
+ fn serialize<'tcx>(&self, tcx: TyCtxt<'tcx>, encoder: FileEncoder) -> FileEncodeResult {
+ // Serializing the `DepGraph` should not modify it.
+ tcx.dep_graph.with_ignore(|| {
+ // Allocate `SourceFileIndex`es.
+ let (file_to_file_index, file_index_to_stable_id) = {
+ let files = tcx.sess.source_map().files();
+ let mut file_to_file_index =
+ FxHashMap::with_capacity_and_hasher(files.len(), Default::default());
+ let mut file_index_to_stable_id =
+ FxHashMap::with_capacity_and_hasher(files.len(), Default::default());
+
+ for (index, file) in files.iter().enumerate() {
+ let index = SourceFileIndex(index as u32);
+ let file_ptr: *const SourceFile = &**file as *const _;
+ file_to_file_index.insert(file_ptr, index);
+ let source_file_id = EncodedSourceFileId::new(tcx, &file);
+ file_index_to_stable_id.insert(index, source_file_id);
+ }
+
+ (file_to_file_index, file_index_to_stable_id)
+ };
+
+ let hygiene_encode_context = HygieneEncodeContext::default();
+
+ let mut encoder = CacheEncoder {
+ tcx,
+ encoder,
+ type_shorthands: Default::default(),
+ predicate_shorthands: Default::default(),
+ interpret_allocs: Default::default(),
+ source_map: CachingSourceMapView::new(tcx.sess.source_map()),
+ file_to_file_index,
+ hygiene_context: &hygiene_encode_context,
+ };
+
+ // Encode query results.
+ let mut query_result_index = EncodedDepNodeIndex::new();
+
+ tcx.sess.time("encode_query_results", || {
+ let enc = &mut encoder;
+ let qri = &mut query_result_index;
+ QueryCtxt::from_tcx(tcx).encode_query_results(enc, qri);
+ });
+
+ // Encode side effects.
+ let side_effects_index: EncodedDepNodeIndex = self
+ .current_side_effects
+ .borrow()
+ .iter()
+ .map(|(dep_node_index, side_effects)| {
+ let pos = AbsoluteBytePos::new(encoder.position());
+ let dep_node_index = SerializedDepNodeIndex::new(dep_node_index.index());
+ encoder.encode_tagged(dep_node_index, side_effects);
+
+ (dep_node_index, pos)
+ })
+ .collect();
+
+ let interpret_alloc_index = {
+ let mut interpret_alloc_index = Vec::new();
+ let mut n = 0;
+ loop {
+ let new_n = encoder.interpret_allocs.len();
+ // If we have found new IDs, serialize those too.
+ if n == new_n {
+ // Otherwise, abort.
+ break;
+ }
+ interpret_alloc_index.reserve(new_n - n);
+ for idx in n..new_n {
+ let id = encoder.interpret_allocs[idx];
+ let pos = encoder.position() as u32;
+ interpret_alloc_index.push(pos);
+ interpret::specialized_encode_alloc_id(&mut encoder, tcx, id);
+ }
+ n = new_n;
+ }
+ interpret_alloc_index
+ };
+
+ let mut syntax_contexts = FxHashMap::default();
+ let mut expn_data = UnhashMap::default();
+ let mut foreign_expn_data = UnhashMap::default();
+
+ // Encode all hygiene data (`SyntaxContextData` and `ExpnData`) from the current
+ // session.
+
+ hygiene_encode_context.encode(
+ &mut encoder,
+ |encoder, index, ctxt_data| {
+ let pos = AbsoluteBytePos::new(encoder.position());
+ encoder.encode_tagged(TAG_SYNTAX_CONTEXT, ctxt_data);
+ syntax_contexts.insert(index, pos);
+ },
+ |encoder, expn_id, data, hash| {
+ if expn_id.krate == LOCAL_CRATE {
+ let pos = AbsoluteBytePos::new(encoder.position());
+ encoder.encode_tagged(TAG_EXPN_DATA, data);
+ expn_data.insert(hash, pos);
+ } else {
+ foreign_expn_data.insert(hash, expn_id.local_id.as_u32());
+ }
+ },
+ );
+
+ // `Encode the file footer.
+ let footer_pos = encoder.position() as u64;
+ encoder.encode_tagged(
+ TAG_FILE_FOOTER,
+ &Footer {
+ file_index_to_stable_id,
+ query_result_index,
+ side_effects_index,
+ interpret_alloc_index,
+ syntax_contexts,
+ expn_data,
+ foreign_expn_data,
+ },
+ );
+
+ // Encode the position of the footer as the last 8 bytes of the
+ // file so we know where to look for it.
+ IntEncodedWithFixedSize(footer_pos).encode(&mut encoder.encoder);
+
+ // DO NOT WRITE ANYTHING TO THE ENCODER AFTER THIS POINT! The address
+ // of the footer must be the last thing in the data stream.
+
+ encoder.finish()
+ })
+ }
+}
+
+impl<'sess> OnDiskCache<'sess> {
+ pub fn as_dyn(&self) -> &dyn rustc_middle::ty::OnDiskCache<'sess> {
+ self as _
+ }
+
+ /// Loads a `QuerySideEffects` created during the previous compilation session.
+ pub fn load_side_effects(
+ &self,
+ tcx: TyCtxt<'_>,
+ dep_node_index: SerializedDepNodeIndex,
+ ) -> QuerySideEffects {
+ let side_effects: Option<QuerySideEffects> =
+ self.load_indexed(tcx, dep_node_index, &self.prev_side_effects_index);
+
+ side_effects.unwrap_or_default()
+ }
+
+ /// Stores a `QuerySideEffects` emitted during the current compilation session.
+ /// Anything stored like this will be available via `load_side_effects` in
+ /// the next compilation session.
+ #[inline(never)]
+ #[cold]
+ pub fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
+ let mut current_side_effects = self.current_side_effects.borrow_mut();
+ let prev = current_side_effects.insert(dep_node_index, side_effects);
+ debug_assert!(prev.is_none());
+ }
+
+ /// Returns the cached query result if there is something in the cache for
+ /// the given `SerializedDepNodeIndex`; otherwise returns `None`.
+ pub fn try_load_query_result<'tcx, T>(
+ &self,
+ tcx: TyCtxt<'tcx>,
+ dep_node_index: SerializedDepNodeIndex,
+ ) -> Option<T>
+ where
+ T: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
+ {
+ self.load_indexed(tcx, dep_node_index, &self.query_result_index)
+ }
+
+ /// Stores side effect emitted during computation of an anonymous query.
+ /// Since many anonymous queries can share the same `DepNode`, we aggregate
+ /// them -- as opposed to regular queries where we assume that there is a
+ /// 1:1 relationship between query-key and `DepNode`.
+ #[inline(never)]
+ #[cold]
+ pub fn store_side_effects_for_anon_node(
+ &self,
+ dep_node_index: DepNodeIndex,
+ side_effects: QuerySideEffects,
+ ) {
+ let mut current_side_effects = self.current_side_effects.borrow_mut();
+
+ let x = current_side_effects.entry(dep_node_index).or_default();
+ x.append(side_effects);
+ }
+
+ fn load_indexed<'tcx, T>(
+ &self,
+ tcx: TyCtxt<'tcx>,
+ dep_node_index: SerializedDepNodeIndex,
+ index: &FxHashMap<SerializedDepNodeIndex, AbsoluteBytePos>,
+ ) -> Option<T>
+ where
+ T: for<'a> Decodable<CacheDecoder<'a, 'tcx>>,
+ {
+ let pos = index.get(&dep_node_index).cloned()?;
+
+ self.with_decoder(tcx, pos, |decoder| Some(decode_tagged(decoder, dep_node_index)))
+ }
+
+ fn with_decoder<'a, 'tcx, T, F: for<'s> FnOnce(&mut CacheDecoder<'s, 'tcx>) -> T>(
+ &'sess self,
+ tcx: TyCtxt<'tcx>,
+ pos: AbsoluteBytePos,
+ f: F,
+ ) -> T
+ where
+ T: Decodable<CacheDecoder<'a, 'tcx>>,
+ {
+ let serialized_data = self.serialized_data.read();
+ let mut decoder = CacheDecoder {
+ tcx,
+ opaque: MemDecoder::new(serialized_data.as_deref().unwrap_or(&[]), pos.to_usize()),
+ source_map: self.source_map,
+ file_index_to_file: &self.file_index_to_file,
+ file_index_to_stable_id: &self.file_index_to_stable_id,
+ alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(),
+ syntax_contexts: &self.syntax_contexts,
+ expn_data: &self.expn_data,
+ foreign_expn_data: &self.foreign_expn_data,
+ hygiene_context: &self.hygiene_context,
+ };
+ f(&mut decoder)
+ }
+}
+
+//- DECODING -------------------------------------------------------------------
+
+/// A decoder that can read from the incremental compilation cache. It is similar to the one
+/// we use for crate metadata decoding in that it can rebase spans and eventually
+/// will also handle things that contain `Ty` instances.
+pub struct CacheDecoder<'a, 'tcx> {
+ tcx: TyCtxt<'tcx>,
+ opaque: MemDecoder<'a>,
+ source_map: &'a SourceMap,
+ file_index_to_file: &'a Lock<FxHashMap<SourceFileIndex, Lrc<SourceFile>>>,
+ file_index_to_stable_id: &'a FxHashMap<SourceFileIndex, EncodedSourceFileId>,
+ alloc_decoding_session: AllocDecodingSession<'a>,
+ syntax_contexts: &'a FxHashMap<u32, AbsoluteBytePos>,
+ expn_data: &'a UnhashMap<ExpnHash, AbsoluteBytePos>,
+ foreign_expn_data: &'a UnhashMap<ExpnHash, u32>,
+ hygiene_context: &'a HygieneDecodeContext,
+}
+
+impl<'a, 'tcx> CacheDecoder<'a, 'tcx> {
+ fn file_index_to_file(&self, index: SourceFileIndex) -> Lrc<SourceFile> {
+ let CacheDecoder {
+ tcx,
+ ref file_index_to_file,
+ ref file_index_to_stable_id,
+ ref source_map,
+ ..
+ } = *self;
+
+ file_index_to_file
+ .borrow_mut()
+ .entry(index)
+ .or_insert_with(|| {
+ let stable_id = file_index_to_stable_id[&index].translate(tcx);
+
+ // If this `SourceFile` is from a foreign crate, then make sure
+ // that we've imported all of the source files from that crate.
+ // This has usually already been done during macro invocation.
+ // However, when encoding query results like `TypeckResults`,
+ // we might encode an `AdtDef` for a foreign type (because it
+ // was referenced in the body of the function). There is no guarantee
+ // that we will load the source files from that crate during macro
+ // expansion, so we use `import_source_files` to ensure that the foreign
+ // source files are actually imported before we call `source_file_by_stable_id`.
+ if stable_id.cnum != LOCAL_CRATE {
+ self.tcx.cstore_untracked().import_source_files(self.tcx.sess, stable_id.cnum);
+ }
+
+ source_map
+ .source_file_by_stable_id(stable_id)
+ .expect("failed to lookup `SourceFile` in new context")
+ })
+ .clone()
+ }
+}
+
+trait DecoderWithPosition: Decoder {
+ fn position(&self) -> usize;
+}
+
+impl<'a> DecoderWithPosition for MemDecoder<'a> {
+ fn position(&self) -> usize {
+ self.position()
+ }
+}
+
+impl<'a, 'tcx> DecoderWithPosition for CacheDecoder<'a, 'tcx> {
+ fn position(&self) -> usize {
+ self.opaque.position()
+ }
+}
+
+// Decodes something that was encoded with `encode_tagged()` and verify that the
+// tag matches and the correct amount of bytes was read.
+fn decode_tagged<D, T, V>(decoder: &mut D, expected_tag: T) -> V
+where
+ T: Decodable<D> + Eq + std::fmt::Debug,
+ V: Decodable<D>,
+ D: DecoderWithPosition,
+{
+ let start_pos = decoder.position();
+
+ let actual_tag = T::decode(decoder);
+ assert_eq!(actual_tag, expected_tag);
+ let value = V::decode(decoder);
+ let end_pos = decoder.position();
+
+ let expected_len: u64 = Decodable::decode(decoder);
+ assert_eq!((end_pos - start_pos) as u64, expected_len);
+
+ value
+}
+
+impl<'a, 'tcx> TyDecoder for CacheDecoder<'a, 'tcx> {
+ type I = TyCtxt<'tcx>;
+ const CLEAR_CROSS_CRATE: bool = false;
+
+ #[inline]
+ fn interner(&self) -> TyCtxt<'tcx> {
+ self.tcx
+ }
+
+ #[inline]
+ fn position(&self) -> usize {
+ self.opaque.position()
+ }
+
+ #[inline]
+ fn peek_byte(&self) -> u8 {
+ self.opaque.data[self.opaque.position()]
+ }
+
+ fn cached_ty_for_shorthand<F>(&mut self, shorthand: usize, or_insert_with: F) -> Ty<'tcx>
+ where
+ F: FnOnce(&mut Self) -> Ty<'tcx>,
+ {
+ let tcx = self.tcx;
+
+ let cache_key = ty::CReaderCacheKey { cnum: None, pos: shorthand };
+
+ if let Some(&ty) = tcx.ty_rcache.borrow().get(&cache_key) {
+ return ty;
+ }
+
+ let ty = or_insert_with(self);
+ // This may overwrite the entry, but it should overwrite with the same value.
+ tcx.ty_rcache.borrow_mut().insert_same(cache_key, ty);
+ ty
+ }
+
+ fn with_position<F, R>(&mut self, pos: usize, f: F) -> R
+ where
+ F: FnOnce(&mut Self) -> R,
+ {
+ debug_assert!(pos < self.opaque.data.len());
+
+ let new_opaque = MemDecoder::new(self.opaque.data, pos);
+ let old_opaque = mem::replace(&mut self.opaque, new_opaque);
+ let r = f(self);
+ self.opaque = old_opaque;
+ r
+ }
+
+ fn decode_alloc_id(&mut self) -> interpret::AllocId {
+ let alloc_decoding_session = self.alloc_decoding_session;
+ alloc_decoding_session.decode_alloc_id(self)
+ }
+}
+
+rustc_middle::implement_ty_decoder!(CacheDecoder<'a, 'tcx>);
+
+// This ensures that the `Decodable<opaque::Decoder>::decode` specialization for `Vec<u8>` is used
+// when a `CacheDecoder` is passed to `Decodable::decode`. Unfortunately, we have to manually opt
+// into specializations this way, given how `CacheDecoder` and the decoding traits currently work.
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Vec<u8> {
+ fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ Decodable::decode(&mut d.opaque)
+ }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for SyntaxContext {
+ fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ let syntax_contexts = decoder.syntax_contexts;
+ rustc_span::hygiene::decode_syntax_context(decoder, decoder.hygiene_context, |this, id| {
+ // This closure is invoked if we haven't already decoded the data for the `SyntaxContext` we are deserializing.
+ // We look up the position of the associated `SyntaxData` and decode it.
+ let pos = syntax_contexts.get(&id).unwrap();
+ this.with_position(pos.to_usize(), |decoder| {
+ let data: SyntaxContextData = decode_tagged(decoder, TAG_SYNTAX_CONTEXT);
+ data
+ })
+ })
+ }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for ExpnId {
+ fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ let hash = ExpnHash::decode(decoder);
+ if hash.is_root() {
+ return ExpnId::root();
+ }
+
+ if let Some(expn_id) = ExpnId::from_hash(hash) {
+ return expn_id;
+ }
+
+ let krate = decoder.tcx.stable_crate_id_to_crate_num(hash.stable_crate_id());
+
+ let expn_id = if krate == LOCAL_CRATE {
+ // We look up the position of the associated `ExpnData` and decode it.
+ let pos = decoder
+ .expn_data
+ .get(&hash)
+ .unwrap_or_else(|| panic!("Bad hash {:?} (map {:?})", hash, decoder.expn_data));
+
+ let data: ExpnData = decoder
+ .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA));
+ let expn_id = rustc_span::hygiene::register_local_expn_id(data, hash);
+
+ #[cfg(debug_assertions)]
+ {
+ use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
+ let local_hash: u64 = decoder.tcx.with_stable_hashing_context(|mut hcx| {
+ let mut hasher = StableHasher::new();
+ expn_id.expn_data().hash_stable(&mut hcx, &mut hasher);
+ hasher.finish()
+ });
+ debug_assert_eq!(hash.local_hash(), local_hash);
+ }
+
+ expn_id
+ } else {
+ let index_guess = decoder.foreign_expn_data[&hash];
+ decoder.tcx.cstore_untracked().expn_hash_to_expn_id(
+ decoder.tcx.sess,
+ krate,
+ index_guess,
+ hash,
+ )
+ };
+
+ debug_assert_eq!(expn_id.krate, krate);
+ expn_id
+ }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for Span {
+ fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ let ctxt = SyntaxContext::decode(decoder);
+ let parent = Option::<LocalDefId>::decode(decoder);
+ let tag: u8 = Decodable::decode(decoder);
+
+ if tag == TAG_PARTIAL_SPAN {
+ return Span::new(BytePos(0), BytePos(0), ctxt, parent);
+ } else if tag == TAG_RELATIVE_SPAN {
+ let dlo = u32::decode(decoder);
+ let dto = u32::decode(decoder);
+
+ let enclosing = decoder.tcx.source_span_untracked(parent.unwrap()).data_untracked();
+ let span = Span::new(
+ enclosing.lo + BytePos::from_u32(dlo),
+ enclosing.lo + BytePos::from_u32(dto),
+ ctxt,
+ parent,
+ );
+
+ return span;
+ } else {
+ debug_assert_eq!(tag, TAG_FULL_SPAN);
+ }
+
+ let file_lo_index = SourceFileIndex::decode(decoder);
+ let line_lo = usize::decode(decoder);
+ let col_lo = BytePos::decode(decoder);
+ let len = BytePos::decode(decoder);
+
+ let file_lo = decoder.file_index_to_file(file_lo_index);
+ let lo = file_lo.lines(|lines| lines[line_lo - 1] + col_lo);
+ let hi = lo + len;
+
+ Span::new(lo, hi, ctxt, parent)
+ }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for CrateNum {
+ fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ let stable_id = StableCrateId::decode(d);
+ let cnum = d.tcx.stable_crate_id_to_crate_num(stable_id);
+ cnum
+ }
+}
+
+// This impl makes sure that we get a runtime error when we try decode a
+// `DefIndex` that is not contained in a `DefId`. Such a case would be problematic
+// because we would not know how to transform the `DefIndex` to the current
+// context.
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefIndex {
+ fn decode(_d: &mut CacheDecoder<'a, 'tcx>) -> DefIndex {
+ panic!("trying to decode `DefIndex` outside the context of a `DefId`")
+ }
+}
+
+// Both the `CrateNum` and the `DefIndex` of a `DefId` can change in between two
+// compilation sessions. We use the `DefPathHash`, which is stable across
+// sessions, to map the old `DefId` to the new one.
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for DefId {
+ fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ // Load the `DefPathHash` which is was we encoded the `DefId` as.
+ let def_path_hash = DefPathHash::decode(d);
+
+ // Using the `DefPathHash`, we can lookup the new `DefId`.
+ // Subtle: We only encode a `DefId` as part of a query result.
+ // If we get to this point, then all of the query inputs were green,
+ // which means that the definition with this hash is guaranteed to
+ // still exist in the current compilation session.
+ d.tcx.def_path_hash_to_def_id(def_path_hash, &mut || {
+ panic!("Failed to convert DefPathHash {:?}", def_path_hash)
+ })
+ }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx FxHashSet<LocalDefId> {
+ fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ RefDecodable::decode(d)
+ }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>>
+ for &'tcx IndexVec<mir::Promoted, mir::Body<'tcx>>
+{
+ fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ RefDecodable::decode(d)
+ }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [ty::abstract_const::Node<'tcx>] {
+ fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ RefDecodable::decode(d)
+ }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [(ty::Predicate<'tcx>, Span)] {
+ fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ RefDecodable::decode(d)
+ }
+}
+
+impl<'a, 'tcx> Decodable<CacheDecoder<'a, 'tcx>> for &'tcx [rustc_ast::InlineAsmTemplatePiece] {
+ fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self {
+ RefDecodable::decode(d)
+ }
+}
+
+macro_rules! impl_ref_decoder {
+ (<$tcx:tt> $($ty:ty,)*) => {
+ $(impl<'a, $tcx> Decodable<CacheDecoder<'a, $tcx>> for &$tcx [$ty] {
+ fn decode(d: &mut CacheDecoder<'a, $tcx>) -> Self {
+ RefDecodable::decode(d)
+ }
+ })*
+ };
+}
+
+impl_ref_decoder! {<'tcx>
+ Span,
+ rustc_ast::Attribute,
+ rustc_span::symbol::Ident,
+ ty::Variance,
+ rustc_span::def_id::DefId,
+ rustc_span::def_id::LocalDefId,
+ (rustc_middle::middle::exported_symbols::ExportedSymbol<'tcx>, rustc_middle::middle::exported_symbols::SymbolExportInfo),
+}
+
+//- ENCODING -------------------------------------------------------------------
+
+/// An encoder that can write to the incremental compilation cache.
+pub struct CacheEncoder<'a, 'tcx> {
+ tcx: TyCtxt<'tcx>,
+ encoder: FileEncoder,
+ type_shorthands: FxHashMap<Ty<'tcx>, usize>,
+ predicate_shorthands: FxHashMap<ty::PredicateKind<'tcx>, usize>,
+ interpret_allocs: FxIndexSet<interpret::AllocId>,
+ source_map: CachingSourceMapView<'tcx>,
+ file_to_file_index: FxHashMap<*const SourceFile, SourceFileIndex>,
+ hygiene_context: &'a HygieneEncodeContext,
+}
+
+impl<'a, 'tcx> CacheEncoder<'a, 'tcx> {
+ fn source_file_index(&mut self, source_file: Lrc<SourceFile>) -> SourceFileIndex {
+ self.file_to_file_index[&(&*source_file as *const SourceFile)]
+ }
+
+ /// Encode something with additional information that allows to do some
+ /// sanity checks when decoding the data again. This method will first
+ /// encode the specified tag, then the given value, then the number of
+ /// bytes taken up by tag and value. On decoding, we can then verify that
+ /// we get the expected tag and read the expected number of bytes.
+ fn encode_tagged<T: Encodable<Self>, V: Encodable<Self>>(&mut self, tag: T, value: &V) {
+ let start_pos = self.position();
+
+ tag.encode(self);
+ value.encode(self);
+
+ let end_pos = self.position();
+ ((end_pos - start_pos) as u64).encode(self);
+ }
+
+ fn finish(self) -> Result<usize, io::Error> {
+ self.encoder.finish()
+ }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for SyntaxContext {
+ fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+ rustc_span::hygiene::raw_encode_syntax_context(*self, s.hygiene_context, s);
+ }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for ExpnId {
+ fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+ s.hygiene_context.schedule_expn_data_for_encoding(*self);
+ self.expn_hash().encode(s);
+ }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for Span {
+ fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+ let span_data = self.data_untracked();
+ span_data.ctxt.encode(s);
+ span_data.parent.encode(s);
+
+ if span_data.is_dummy() {
+ return TAG_PARTIAL_SPAN.encode(s);
+ }
+
+ if let Some(parent) = span_data.parent {
+ let enclosing = s.tcx.source_span(parent).data_untracked();
+ if enclosing.contains(span_data) {
+ TAG_RELATIVE_SPAN.encode(s);
+ (span_data.lo - enclosing.lo).to_u32().encode(s);
+ (span_data.hi - enclosing.lo).to_u32().encode(s);
+ return;
+ }
+ }
+
+ let pos = s.source_map.byte_pos_to_line_and_col(span_data.lo);
+ let partial_span = match &pos {
+ Some((file_lo, _, _)) => !file_lo.contains(span_data.hi),
+ None => true,
+ };
+
+ if partial_span {
+ return TAG_PARTIAL_SPAN.encode(s);
+ }
+
+ let (file_lo, line_lo, col_lo) = pos.unwrap();
+
+ let len = span_data.hi - span_data.lo;
+
+ let source_file_index = s.source_file_index(file_lo);
+
+ TAG_FULL_SPAN.encode(s);
+ source_file_index.encode(s);
+ line_lo.encode(s);
+ col_lo.encode(s);
+ len.encode(s);
+ }
+}
+
+impl<'a, 'tcx> TyEncoder for CacheEncoder<'a, 'tcx> {
+ type I = TyCtxt<'tcx>;
+ const CLEAR_CROSS_CRATE: bool = false;
+
+ fn position(&self) -> usize {
+ self.encoder.position()
+ }
+ fn type_shorthands(&mut self) -> &mut FxHashMap<Ty<'tcx>, usize> {
+ &mut self.type_shorthands
+ }
+ fn predicate_shorthands(&mut self) -> &mut FxHashMap<ty::PredicateKind<'tcx>, usize> {
+ &mut self.predicate_shorthands
+ }
+ fn encode_alloc_id(&mut self, alloc_id: &interpret::AllocId) {
+ let (index, _) = self.interpret_allocs.insert_full(*alloc_id);
+
+ index.encode(self);
+ }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for CrateNum {
+ fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+ s.tcx.stable_crate_id(*self).encode(s);
+ }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for DefId {
+ fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) {
+ s.tcx.def_path_hash(*self).encode(s);
+ }
+}
+
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for DefIndex {
+ fn encode(&self, _: &mut CacheEncoder<'a, 'tcx>) {
+ bug!("encoding `DefIndex` without context");
+ }
+}
+
+macro_rules! encoder_methods {
+ ($($name:ident($ty:ty);)*) => {
+ #[inline]
+ $(fn $name(&mut self, value: $ty) {
+ self.encoder.$name(value)
+ })*
+ }
+}
+
+impl<'a, 'tcx> Encoder for CacheEncoder<'a, 'tcx> {
+ encoder_methods! {
+ emit_usize(usize);
+ emit_u128(u128);
+ emit_u64(u64);
+ emit_u32(u32);
+ emit_u16(u16);
+ emit_u8(u8);
+
+ emit_isize(isize);
+ emit_i128(i128);
+ emit_i64(i64);
+ emit_i32(i32);
+ emit_i16(i16);
+ emit_i8(i8);
+
+ emit_bool(bool);
+ emit_f64(f64);
+ emit_f32(f32);
+ emit_char(char);
+ emit_str(&str);
+ emit_raw_bytes(&[u8]);
+ }
+}
+
+// This ensures that the `Encodable<opaque::FileEncoder>::encode` specialization for byte slices
+// is used when a `CacheEncoder` having an `opaque::FileEncoder` is passed to `Encodable::encode`.
+// Unfortunately, we have to manually opt into specializations this way, given how `CacheEncoder`
+// and the encoding traits currently work.
+impl<'a, 'tcx> Encodable<CacheEncoder<'a, 'tcx>> for [u8] {
+ fn encode(&self, e: &mut CacheEncoder<'a, 'tcx>) {
+ self.encode(&mut e.encoder);
+ }
+}
+
+pub fn encode_query_results<'a, 'tcx, CTX, Q>(
+ tcx: CTX,
+ encoder: &mut CacheEncoder<'a, 'tcx>,
+ query_result_index: &mut EncodedDepNodeIndex,
+) where
+ CTX: QueryContext + 'tcx,
+ Q: super::QueryDescription<CTX>,
+ Q::Value: Encodable<CacheEncoder<'a, 'tcx>>,
+{
+ let _timer = tcx
+ .dep_context()
+ .profiler()
+ .extra_verbose_generic_activity("encode_query_results_for", std::any::type_name::<Q>());
+
+ assert!(Q::query_state(tcx).all_inactive());
+ let cache = Q::query_cache(tcx);
+ cache.iter(&mut |key, value, dep_node| {
+ if Q::cache_on_disk(*tcx.dep_context(), &key) {
+ let dep_node = SerializedDepNodeIndex::new(dep_node.index());
+
+ // Record position of the cache entry.
+ query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.encoder.position())));
+
+ // Encode the type check tables with the `SerializedDepNodeIndex`
+ // as tag.
+ encoder.encode_tagged(dep_node, value);
+ }
+ });
+}
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs
new file mode 100644
index 000000000..eda4401c8
--- /dev/null
+++ b/compiler/rustc_query_impl/src/plumbing.rs
@@ -0,0 +1,558 @@
+//! The implementation of the query system itself. This defines the macros that
+//! generate the actual methods on tcx which find and execute the provider,
+//! manage the caches, and so forth.
+
+use crate::{on_disk_cache, Queries};
+use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex};
+use rustc_middle::ty::tls::{self, ImplicitCtxt};
+use rustc_middle::ty::TyCtxt;
+use rustc_query_system::dep_graph::HasDepContext;
+use rustc_query_system::query::{QueryContext, QueryJobId, QueryMap, QuerySideEffects};
+
+use rustc_data_structures::sync::Lock;
+use rustc_data_structures::thin_vec::ThinVec;
+use rustc_errors::{Diagnostic, Handler};
+
+use std::any::Any;
+use std::num::NonZeroU64;
+
+#[derive(Copy, Clone)]
+pub struct QueryCtxt<'tcx> {
+ pub tcx: TyCtxt<'tcx>,
+ pub queries: &'tcx Queries<'tcx>,
+}
+
+impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> {
+ type Target = TyCtxt<'tcx>;
+
+ #[inline]
+ fn deref(&self) -> &Self::Target {
+ &self.tcx
+ }
+}
+
+impl<'tcx> HasDepContext for QueryCtxt<'tcx> {
+ type DepKind = rustc_middle::dep_graph::DepKind;
+ type DepContext = TyCtxt<'tcx>;
+
+ #[inline]
+ fn dep_context(&self) -> &Self::DepContext {
+ &self.tcx
+ }
+}
+
+impl QueryContext for QueryCtxt<'_> {
+ fn next_job_id(&self) -> QueryJobId {
+ QueryJobId(
+ NonZeroU64::new(
+ self.queries.jobs.fetch_add(1, rustc_data_structures::sync::Ordering::Relaxed),
+ )
+ .unwrap(),
+ )
+ }
+
+ fn current_query_job(&self) -> Option<QueryJobId> {
+ tls::with_related_context(**self, |icx| icx.query)
+ }
+
+ fn try_collect_active_jobs(&self) -> Option<QueryMap> {
+ self.queries.try_collect_active_jobs(**self)
+ }
+
+ // Interactions with on_disk_cache
+ fn load_side_effects(&self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects {
+ self.queries
+ .on_disk_cache
+ .as_ref()
+ .map(|c| c.load_side_effects(**self, prev_dep_node_index))
+ .unwrap_or_default()
+ }
+
+ fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) {
+ if let Some(c) = self.queries.on_disk_cache.as_ref() {
+ c.store_side_effects(dep_node_index, side_effects)
+ }
+ }
+
+ fn store_side_effects_for_anon_node(
+ &self,
+ dep_node_index: DepNodeIndex,
+ side_effects: QuerySideEffects,
+ ) {
+ if let Some(c) = self.queries.on_disk_cache.as_ref() {
+ c.store_side_effects_for_anon_node(dep_node_index, side_effects)
+ }
+ }
+
+ /// Executes a job by changing the `ImplicitCtxt` to point to the
+ /// new query job while it executes. It returns the diagnostics
+ /// captured during execution and the actual result.
+ #[inline(always)]
+ fn start_query<R>(
+ &self,
+ token: QueryJobId,
+ diagnostics: Option<&Lock<ThinVec<Diagnostic>>>,
+ compute: impl FnOnce() -> R,
+ ) -> R {
+ // The `TyCtxt` stored in TLS has the same global interner lifetime
+ // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes
+ // when accessing the `ImplicitCtxt`.
+ tls::with_related_context(**self, move |current_icx| {
+ // Update the `ImplicitCtxt` to point to our new query job.
+ let new_icx = ImplicitCtxt {
+ tcx: **self,
+ query: Some(token),
+ diagnostics,
+ layout_depth: current_icx.layout_depth,
+ task_deps: current_icx.task_deps,
+ };
+
+ // Use the `ImplicitCtxt` while we execute the query.
+ tls::enter_context(&new_icx, |_| {
+ rustc_data_structures::stack::ensure_sufficient_stack(compute)
+ })
+ })
+ }
+}
+
+impl<'tcx> QueryCtxt<'tcx> {
+ #[inline]
+ pub fn from_tcx(tcx: TyCtxt<'tcx>) -> Self {
+ let queries = tcx.queries.as_any();
+ let queries = unsafe {
+ let queries = std::mem::transmute::<&dyn Any, &dyn Any>(queries);
+ let queries = queries.downcast_ref().unwrap();
+ let queries = std::mem::transmute::<&Queries<'_>, &Queries<'_>>(queries);
+ queries
+ };
+ QueryCtxt { tcx, queries }
+ }
+
+ pub(crate) fn on_disk_cache(self) -> Option<&'tcx on_disk_cache::OnDiskCache<'tcx>> {
+ self.queries.on_disk_cache.as_ref()
+ }
+
+ pub(super) fn encode_query_results(
+ self,
+ encoder: &mut on_disk_cache::CacheEncoder<'_, 'tcx>,
+ query_result_index: &mut on_disk_cache::EncodedDepNodeIndex,
+ ) {
+ macro_rules! encode_queries {
+ ($($query:ident,)*) => {
+ $(
+ on_disk_cache::encode_query_results::<_, super::queries::$query<'_>>(
+ self,
+ encoder,
+ query_result_index
+ );
+ )*
+ }
+ }
+
+ rustc_cached_queries!(encode_queries!);
+ }
+
+ pub fn try_print_query_stack(
+ self,
+ query: Option<QueryJobId>,
+ handler: &Handler,
+ num_frames: Option<usize>,
+ ) -> usize {
+ rustc_query_system::query::print_query_stack(self, query, handler, num_frames)
+ }
+}
+
+macro_rules! handle_cycle_error {
+ ([][$tcx: expr, $error:expr]) => {{
+ $error.emit();
+ Value::from_cycle_error($tcx)
+ }};
+ ([(fatal_cycle) $($rest:tt)*][$tcx:expr, $error:expr]) => {{
+ $error.emit();
+ $tcx.sess.abort_if_errors();
+ unreachable!()
+ }};
+ ([(cycle_delay_bug) $($rest:tt)*][$tcx:expr, $error:expr]) => {{
+ $error.delay_as_bug();
+ Value::from_cycle_error($tcx)
+ }};
+ ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
+ handle_cycle_error!([$($modifiers)*][$($args)*])
+ };
+}
+
+macro_rules! is_anon {
+ ([]) => {{
+ false
+ }};
+ ([(anon) $($rest:tt)*]) => {{
+ true
+ }};
+ ([$other:tt $($modifiers:tt)*]) => {
+ is_anon!([$($modifiers)*])
+ };
+}
+
+macro_rules! is_eval_always {
+ ([]) => {{
+ false
+ }};
+ ([(eval_always) $($rest:tt)*]) => {{
+ true
+ }};
+ ([$other:tt $($modifiers:tt)*]) => {
+ is_eval_always!([$($modifiers)*])
+ };
+}
+
+macro_rules! hash_result {
+ ([]) => {{
+ Some(dep_graph::hash_result)
+ }};
+ ([(no_hash) $($rest:tt)*]) => {{
+ None
+ }};
+ ([$other:tt $($modifiers:tt)*]) => {
+ hash_result!([$($modifiers)*])
+ };
+}
+
+macro_rules! get_provider {
+ ([][$tcx:expr, $name:ident, $key:expr]) => {{
+ $tcx.queries.local_providers.$name
+ }};
+ ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{
+ if $key.query_crate_is_local() {
+ $tcx.queries.local_providers.$name
+ } else {
+ $tcx.queries.extern_providers.$name
+ }
+ }};
+ ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => {
+ get_provider!([$($modifiers)*][$($args)*])
+ };
+}
+
+macro_rules! opt_remap_env_constness {
+ ([][$name:ident]) => {};
+ ([(remap_env_constness) $($rest:tt)*][$name:ident]) => {
+ let $name = $name.without_const();
+ };
+ ([$other:tt $($modifiers:tt)*][$name:ident]) => {
+ opt_remap_env_constness!([$($modifiers)*][$name])
+ };
+}
+
+macro_rules! define_queries {
+ (<$tcx:tt>
+ $($(#[$attr:meta])*
+ [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => {
+
+ define_queries_struct! {
+ tcx: $tcx,
+ input: ($(([$($modifiers)*] [$($attr)*] [$name]))*)
+ }
+
+ mod make_query {
+ use super::*;
+
+ // Create an eponymous constructor for each query.
+ $(#[allow(nonstandard_style)] $(#[$attr])*
+ pub fn $name<$tcx>(tcx: QueryCtxt<$tcx>, key: query_keys::$name<$tcx>) -> QueryStackFrame {
+ opt_remap_env_constness!([$($modifiers)*][key]);
+ let kind = dep_graph::DepKind::$name;
+ let name = stringify!($name);
+ // Disable visible paths printing for performance reasons.
+ // Showing visible path instead of any path is not that important in production.
+ let description = ty::print::with_no_visible_paths!(
+ // Force filename-line mode to avoid invoking `type_of` query.
+ ty::print::with_forced_impl_filename_line!(
+ queries::$name::describe(tcx, key)
+ )
+ );
+ let description = if tcx.sess.verbose() {
+ format!("{} [{}]", description, name)
+ } else {
+ description
+ };
+ let span = if kind == dep_graph::DepKind::def_span {
+ // The `def_span` query is used to calculate `default_span`,
+ // so exit to avoid infinite recursion.
+ None
+ } else {
+ Some(key.default_span(*tcx))
+ };
+ let def_kind = if kind == dep_graph::DepKind::opt_def_kind {
+ // Try to avoid infinite recursion.
+ None
+ } else {
+ key.key_as_def_id()
+ .and_then(|def_id| def_id.as_local())
+ .and_then(|def_id| tcx.opt_def_kind(def_id))
+ };
+ let hash = || {
+ tcx.with_stable_hashing_context(|mut hcx|{
+ let mut hasher = StableHasher::new();
+ std::mem::discriminant(&kind).hash_stable(&mut hcx, &mut hasher);
+ key.hash_stable(&mut hcx, &mut hasher);
+ hasher.finish::<u64>()
+ })
+ };
+
+ QueryStackFrame::new(name, description, span, def_kind, hash)
+ })*
+ }
+
+ #[allow(nonstandard_style)]
+ mod queries {
+ use std::marker::PhantomData;
+
+ $(pub struct $name<$tcx> {
+ data: PhantomData<&$tcx ()>
+ })*
+ }
+
+ $(impl<$tcx> QueryConfig for queries::$name<$tcx> {
+ type Key = query_keys::$name<$tcx>;
+ type Value = query_values::$name<$tcx>;
+ type Stored = query_stored::$name<$tcx>;
+ const NAME: &'static str = stringify!($name);
+ }
+
+ impl<$tcx> QueryDescription<QueryCtxt<$tcx>> for queries::$name<$tcx> {
+ rustc_query_description! { $name<$tcx> }
+
+ type Cache = query_storage::$name<$tcx>;
+
+ #[inline(always)]
+ fn query_state<'a>(tcx: QueryCtxt<$tcx>) -> &'a QueryState<Self::Key>
+ where QueryCtxt<$tcx>: 'a
+ {
+ &tcx.queries.$name
+ }
+
+ #[inline(always)]
+ fn query_cache<'a>(tcx: QueryCtxt<$tcx>) -> &'a Self::Cache
+ where 'tcx:'a
+ {
+ &tcx.query_caches.$name
+ }
+
+ #[inline]
+ fn make_vtable(tcx: QueryCtxt<'tcx>, key: &Self::Key) ->
+ QueryVTable<QueryCtxt<$tcx>, Self::Key, Self::Value>
+ {
+ let compute = get_provider!([$($modifiers)*][tcx, $name, key]);
+ let cache_on_disk = Self::cache_on_disk(tcx.tcx, key);
+ QueryVTable {
+ anon: is_anon!([$($modifiers)*]),
+ eval_always: is_eval_always!([$($modifiers)*]),
+ dep_kind: dep_graph::DepKind::$name,
+ hash_result: hash_result!([$($modifiers)*]),
+ handle_cycle_error: |tcx, mut error| handle_cycle_error!([$($modifiers)*][tcx, error]),
+ compute,
+ cache_on_disk,
+ try_load_from_disk: Self::TRY_LOAD_FROM_DISK,
+ }
+ }
+ })*
+
+ #[allow(nonstandard_style)]
+ mod query_callbacks {
+ use super::*;
+ use rustc_middle::dep_graph::DepNode;
+ use rustc_middle::ty::query::query_keys;
+ use rustc_query_system::dep_graph::DepNodeParams;
+ use rustc_query_system::query::{force_query, QueryDescription};
+ use rustc_query_system::dep_graph::FingerprintStyle;
+
+ // We use this for most things when incr. comp. is turned off.
+ pub fn Null() -> DepKindStruct {
+ DepKindStruct {
+ is_anon: false,
+ is_eval_always: false,
+ fingerprint_style: FingerprintStyle::Unit,
+ force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
+ try_load_from_on_disk_cache: None,
+ }
+ }
+
+ // We use this for the forever-red node.
+ pub fn Red() -> DepKindStruct {
+ DepKindStruct {
+ is_anon: false,
+ is_eval_always: false,
+ fingerprint_style: FingerprintStyle::Unit,
+ force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)),
+ try_load_from_on_disk_cache: None,
+ }
+ }
+
+ pub fn TraitSelect() -> DepKindStruct {
+ DepKindStruct {
+ is_anon: true,
+ is_eval_always: false,
+ fingerprint_style: FingerprintStyle::Unit,
+ force_from_dep_node: None,
+ try_load_from_on_disk_cache: None,
+ }
+ }
+
+ pub fn CompileCodegenUnit() -> DepKindStruct {
+ DepKindStruct {
+ is_anon: false,
+ is_eval_always: false,
+ fingerprint_style: FingerprintStyle::Opaque,
+ force_from_dep_node: None,
+ try_load_from_on_disk_cache: None,
+ }
+ }
+
+ pub fn CompileMonoItem() -> DepKindStruct {
+ DepKindStruct {
+ is_anon: false,
+ is_eval_always: false,
+ fingerprint_style: FingerprintStyle::Opaque,
+ force_from_dep_node: None,
+ try_load_from_on_disk_cache: None,
+ }
+ }
+
+ $(pub(crate) fn $name()-> DepKindStruct {
+ let is_anon = is_anon!([$($modifiers)*]);
+ let is_eval_always = is_eval_always!([$($modifiers)*]);
+
+ let fingerprint_style =
+ <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::fingerprint_style();
+
+ if is_anon || !fingerprint_style.reconstructible() {
+ return DepKindStruct {
+ is_anon,
+ is_eval_always,
+ fingerprint_style,
+ force_from_dep_node: None,
+ try_load_from_on_disk_cache: None,
+ }
+ }
+
+ #[inline(always)]
+ fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> Option<query_keys::$name<'tcx>> {
+ <query_keys::$name<'_> as DepNodeParams<TyCtxt<'_>>>::recover(tcx, &dep_node)
+ }
+
+ fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: DepNode) -> bool {
+ if let Some(key) = recover(tcx, dep_node) {
+ #[cfg(debug_assertions)]
+ let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered();
+ let tcx = QueryCtxt::from_tcx(tcx);
+ force_query::<queries::$name<'_>, _>(tcx, key, dep_node);
+ true
+ } else {
+ false
+ }
+ }
+
+ fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: DepNode) {
+ debug_assert!(tcx.dep_graph.is_green(&dep_node));
+
+ let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash));
+ if queries::$name::cache_on_disk(tcx, &key) {
+ let _ = tcx.$name(key);
+ }
+ }
+
+ DepKindStruct {
+ is_anon,
+ is_eval_always,
+ fingerprint_style,
+ force_from_dep_node: Some(force_from_dep_node),
+ try_load_from_on_disk_cache: Some(try_load_from_on_disk_cache),
+ }
+ })*
+ }
+
+ pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct] {
+ arena.alloc_from_iter(make_dep_kind_array!(query_callbacks))
+ }
+ }
+}
+
+// FIXME(eddyb) this macro (and others?) use `$tcx` and `'tcx` interchangeably.
+// We should either not take `$tcx` at all and use `'tcx` everywhere, or use
+// `$tcx` everywhere (even if that isn't necessary due to lack of hygiene).
+macro_rules! define_queries_struct {
+ (tcx: $tcx:tt,
+ input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => {
+ pub struct Queries<$tcx> {
+ local_providers: Box<Providers>,
+ extern_providers: Box<ExternProviders>,
+
+ pub on_disk_cache: Option<OnDiskCache<$tcx>>,
+
+ jobs: AtomicU64,
+
+ $($(#[$attr])* $name: QueryState<query_keys::$name<$tcx>>,)*
+ }
+
+ impl<$tcx> Queries<$tcx> {
+ pub fn new(
+ local_providers: Providers,
+ extern_providers: ExternProviders,
+ on_disk_cache: Option<OnDiskCache<$tcx>>,
+ ) -> Self {
+ Queries {
+ local_providers: Box::new(local_providers),
+ extern_providers: Box::new(extern_providers),
+ on_disk_cache,
+ jobs: AtomicU64::new(1),
+ $($name: Default::default()),*
+ }
+ }
+
+ pub(crate) fn try_collect_active_jobs(
+ &$tcx self,
+ tcx: TyCtxt<$tcx>,
+ ) -> Option<QueryMap> {
+ let tcx = QueryCtxt { tcx, queries: self };
+ let mut jobs = QueryMap::default();
+
+ $(
+ self.$name.try_collect_active_jobs(
+ tcx,
+ make_query::$name,
+ &mut jobs,
+ )?;
+ )*
+
+ Some(jobs)
+ }
+ }
+
+ impl<'tcx> QueryEngine<'tcx> for Queries<'tcx> {
+ fn as_any(&'tcx self) -> &'tcx dyn std::any::Any {
+ let this = unsafe { std::mem::transmute::<&Queries<'_>, &Queries<'_>>(self) };
+ this as _
+ }
+
+ fn try_mark_green(&'tcx self, tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool {
+ let qcx = QueryCtxt { tcx, queries: self };
+ tcx.dep_graph.try_mark_green(qcx, dep_node).is_some()
+ }
+
+ $($(#[$attr])*
+ #[inline(always)]
+ #[tracing::instrument(level = "trace", skip(self, tcx))]
+ fn $name(
+ &'tcx self,
+ tcx: TyCtxt<$tcx>,
+ span: Span,
+ key: query_keys::$name<$tcx>,
+ mode: QueryMode,
+ ) -> Option<query_stored::$name<$tcx>> {
+ opt_remap_env_constness!([$($modifiers)*][key]);
+ let qcx = QueryCtxt { tcx, queries: self };
+ get_query::<queries::$name<$tcx>, _>(qcx, span, key, mode)
+ })*
+ }
+ };
+}
diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/profiling_support.rs
new file mode 100644
index 000000000..551f09420
--- /dev/null
+++ b/compiler/rustc_query_impl/src/profiling_support.rs
@@ -0,0 +1,324 @@
+use measureme::{StringComponent, StringId};
+use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::profiling::SelfProfiler;
+use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE};
+use rustc_hir::definitions::DefPathData;
+use rustc_middle::ty::{TyCtxt, WithOptConstParam};
+use rustc_query_system::query::QueryCache;
+use std::fmt::Debug;
+use std::io::Write;
+
+struct QueryKeyStringCache {
+ def_id_cache: FxHashMap<DefId, StringId>,
+}
+
+impl QueryKeyStringCache {
+ fn new() -> QueryKeyStringCache {
+ QueryKeyStringCache { def_id_cache: Default::default() }
+ }
+}
+
+struct QueryKeyStringBuilder<'p, 'c, 'tcx> {
+ profiler: &'p SelfProfiler,
+ tcx: TyCtxt<'tcx>,
+ string_cache: &'c mut QueryKeyStringCache,
+}
+
+impl<'p, 'c, 'tcx> QueryKeyStringBuilder<'p, 'c, 'tcx> {
+ fn new(
+ profiler: &'p SelfProfiler,
+ tcx: TyCtxt<'tcx>,
+ string_cache: &'c mut QueryKeyStringCache,
+ ) -> QueryKeyStringBuilder<'p, 'c, 'tcx> {
+ QueryKeyStringBuilder { profiler, tcx, string_cache }
+ }
+
+ // The current implementation is rather crude. In the future it might be a
+ // good idea to base this on `ty::print` in order to get nicer and more
+ // efficient query keys.
+ fn def_id_to_string_id(&mut self, def_id: DefId) -> StringId {
+ if let Some(&string_id) = self.string_cache.def_id_cache.get(&def_id) {
+ return string_id;
+ }
+
+ let def_key = self.tcx.def_key(def_id);
+
+ let (parent_string_id, start_index) = match def_key.parent {
+ Some(parent_index) => {
+ let parent_def_id = DefId { index: parent_index, krate: def_id.krate };
+
+ (self.def_id_to_string_id(parent_def_id), 0)
+ }
+ None => (StringId::INVALID, 2),
+ };
+
+ let dis_buffer = &mut [0u8; 16];
+ let crate_name;
+ let other_name;
+ let name;
+ let dis;
+ let end_index;
+
+ match def_key.disambiguated_data.data {
+ DefPathData::CrateRoot => {
+ crate_name = self.tcx.crate_name(def_id.krate);
+ name = crate_name.as_str();
+ dis = "";
+ end_index = 3;
+ }
+ other => {
+ other_name = other.to_string();
+ name = other_name.as_str();
+ if def_key.disambiguated_data.disambiguator == 0 {
+ dis = "";
+ end_index = 3;
+ } else {
+ write!(&mut dis_buffer[..], "[{}]", def_key.disambiguated_data.disambiguator)
+ .unwrap();
+ let end_of_dis = dis_buffer.iter().position(|&c| c == b']').unwrap();
+ dis = std::str::from_utf8(&dis_buffer[..end_of_dis + 1]).unwrap();
+ end_index = 4;
+ }
+ }
+ }
+
+ let components = [
+ StringComponent::Ref(parent_string_id),
+ StringComponent::Value("::"),
+ StringComponent::Value(name),
+ StringComponent::Value(dis),
+ ];
+
+ let string_id = self.profiler.alloc_string(&components[start_index..end_index]);
+
+ self.string_cache.def_id_cache.insert(def_id, string_id);
+
+ string_id
+ }
+}
+
+trait IntoSelfProfilingString {
+ fn to_self_profile_string(&self, builder: &mut QueryKeyStringBuilder<'_, '_, '_>) -> StringId;
+}
+
+// The default implementation of `IntoSelfProfilingString` just uses `Debug`
+// which is slow and causes lots of duplication of string data.
+// The specialized impls below take care of making the `DefId` case more
+// efficient.
+impl<T: Debug> IntoSelfProfilingString for T {
+ default fn to_self_profile_string(
+ &self,
+ builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+ ) -> StringId {
+ let s = format!("{:?}", self);
+ builder.profiler.alloc_string(&s[..])
+ }
+}
+
+impl<T: SpecIntoSelfProfilingString> IntoSelfProfilingString for T {
+ fn to_self_profile_string(&self, builder: &mut QueryKeyStringBuilder<'_, '_, '_>) -> StringId {
+ self.spec_to_self_profile_string(builder)
+ }
+}
+
+#[rustc_specialization_trait]
+trait SpecIntoSelfProfilingString: Debug {
+ fn spec_to_self_profile_string(
+ &self,
+ builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+ ) -> StringId;
+}
+
+impl SpecIntoSelfProfilingString for DefId {
+ fn spec_to_self_profile_string(
+ &self,
+ builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+ ) -> StringId {
+ builder.def_id_to_string_id(*self)
+ }
+}
+
+impl SpecIntoSelfProfilingString for CrateNum {
+ fn spec_to_self_profile_string(
+ &self,
+ builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+ ) -> StringId {
+ builder.def_id_to_string_id(self.as_def_id())
+ }
+}
+
+impl SpecIntoSelfProfilingString for DefIndex {
+ fn spec_to_self_profile_string(
+ &self,
+ builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+ ) -> StringId {
+ builder.def_id_to_string_id(DefId { krate: LOCAL_CRATE, index: *self })
+ }
+}
+
+impl SpecIntoSelfProfilingString for LocalDefId {
+ fn spec_to_self_profile_string(
+ &self,
+ builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+ ) -> StringId {
+ builder.def_id_to_string_id(DefId { krate: LOCAL_CRATE, index: self.local_def_index })
+ }
+}
+
+impl<T: SpecIntoSelfProfilingString> SpecIntoSelfProfilingString for WithOptConstParam<T> {
+ fn spec_to_self_profile_string(
+ &self,
+ builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+ ) -> StringId {
+ // We print `WithOptConstParam` values as tuples to make them shorter
+ // and more readable, without losing information:
+ //
+ // "WithOptConstParam { did: foo::bar, const_param_did: Some(foo::baz) }"
+ // becomes "(foo::bar, foo::baz)" and
+ // "WithOptConstParam { did: foo::bar, const_param_did: None }"
+ // becomes "(foo::bar, _)".
+
+ let did = StringComponent::Ref(self.did.to_self_profile_string(builder));
+
+ let const_param_did = if let Some(const_param_did) = self.const_param_did {
+ let const_param_did = builder.def_id_to_string_id(const_param_did);
+ StringComponent::Ref(const_param_did)
+ } else {
+ StringComponent::Value("_")
+ };
+
+ let components = [
+ StringComponent::Value("("),
+ did,
+ StringComponent::Value(", "),
+ const_param_did,
+ StringComponent::Value(")"),
+ ];
+
+ builder.profiler.alloc_string(&components[..])
+ }
+}
+
+impl<T0, T1> SpecIntoSelfProfilingString for (T0, T1)
+where
+ T0: SpecIntoSelfProfilingString,
+ T1: SpecIntoSelfProfilingString,
+{
+ fn spec_to_self_profile_string(
+ &self,
+ builder: &mut QueryKeyStringBuilder<'_, '_, '_>,
+ ) -> StringId {
+ let val0 = self.0.to_self_profile_string(builder);
+ let val1 = self.1.to_self_profile_string(builder);
+
+ let components = &[
+ StringComponent::Value("("),
+ StringComponent::Ref(val0),
+ StringComponent::Value(","),
+ StringComponent::Ref(val1),
+ StringComponent::Value(")"),
+ ];
+
+ builder.profiler.alloc_string(components)
+ }
+}
+
+/// Allocate the self-profiling query strings for a single query cache. This
+/// method is called from `alloc_self_profile_query_strings` which knows all
+/// the queries via macro magic.
+fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>(
+ tcx: TyCtxt<'tcx>,
+ query_name: &'static str,
+ query_cache: &C,
+ string_cache: &mut QueryKeyStringCache,
+) where
+ C: QueryCache,
+ C::Key: Debug + Clone,
+{
+ tcx.prof.with_profiler(|profiler| {
+ let event_id_builder = profiler.event_id_builder();
+
+ // Walk the entire query cache and allocate the appropriate
+ // string representations. Each cache entry is uniquely
+ // identified by its dep_node_index.
+ if profiler.query_key_recording_enabled() {
+ let mut query_string_builder = QueryKeyStringBuilder::new(profiler, tcx, string_cache);
+
+ let query_name = profiler.get_or_alloc_cached_string(query_name);
+
+ // Since building the string representation of query keys might
+ // need to invoke queries itself, we cannot keep the query caches
+ // locked while doing so. Instead we copy out the
+ // `(query_key, dep_node_index)` pairs and release the lock again.
+ let mut query_keys_and_indices = Vec::new();
+ query_cache.iter(&mut |k, _, i| query_keys_and_indices.push((k.clone(), i)));
+
+ // Now actually allocate the strings. If allocating the strings
+ // generates new entries in the query cache, we'll miss them but
+ // we don't actually care.
+ for (query_key, dep_node_index) in query_keys_and_indices {
+ // Translate the DepNodeIndex into a QueryInvocationId
+ let query_invocation_id = dep_node_index.into();
+
+ // Create the string version of the query-key
+ let query_key = query_key.to_self_profile_string(&mut query_string_builder);
+ let event_id = event_id_builder.from_label_and_arg(query_name, query_key);
+
+ // Doing this in bulk might be a good idea:
+ profiler.map_query_invocation_id_to_string(
+ query_invocation_id,
+ event_id.to_string_id(),
+ );
+ }
+ } else {
+ // In this branch we don't allocate query keys
+ let query_name = profiler.get_or_alloc_cached_string(query_name);
+ let event_id = event_id_builder.from_label(query_name).to_string_id();
+
+ // FIXME(eddyb) make this O(1) by using a pre-cached query name `EventId`,
+ // instead of passing the `DepNodeIndex` to `finish_with_query_invocation_id`,
+ // when recording the event in the first place.
+ let mut query_invocation_ids = Vec::new();
+ query_cache.iter(&mut |_, _, i| {
+ query_invocation_ids.push(i.into());
+ });
+
+ profiler.bulk_map_query_invocation_id_to_single_string(
+ query_invocation_ids.into_iter(),
+ event_id,
+ );
+ }
+ });
+}
+
+/// All self-profiling events generated by the query engine use
+/// virtual `StringId`s for their `event_id`. This method makes all
+/// those virtual `StringId`s point to actual strings.
+///
+/// If we are recording only summary data, the ids will point to
+/// just the query names. If we are recording query keys too, we
+/// allocate the corresponding strings here.
+pub fn alloc_self_profile_query_strings(tcx: TyCtxt<'_>) {
+ if !tcx.prof.enabled() {
+ return;
+ }
+
+ let mut string_cache = QueryKeyStringCache::new();
+
+ macro_rules! alloc_once {
+ (<$tcx:tt>
+ $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($K:ty) -> $V:ty,)*
+ ) => {
+ $({
+ alloc_self_profile_query_strings_for_query_cache(
+ tcx,
+ stringify!($name),
+ &tcx.query_caches.$name,
+ &mut string_cache,
+ );
+ })*
+ }
+ }
+
+ rustc_query_append! { [alloc_once!][<'tcx>] }
+}
diff --git a/compiler/rustc_query_impl/src/values.rs b/compiler/rustc_query_impl/src/values.rs
new file mode 100644
index 000000000..718a2971c
--- /dev/null
+++ b/compiler/rustc_query_impl/src/values.rs
@@ -0,0 +1,45 @@
+use super::QueryCtxt;
+use rustc_middle::ty::{self, AdtSizedConstraint, Ty};
+
+pub(super) trait Value<'tcx>: Sized {
+ fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self;
+}
+
+impl<'tcx, T> Value<'tcx> for T {
+ default fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> T {
+ tcx.sess.abort_if_errors();
+ bug!("Value::from_cycle_error called without errors");
+ }
+}
+
+impl<'tcx> Value<'tcx> for Ty<'_> {
+ fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self {
+ // SAFETY: This is never called when `Self` is not `Ty<'tcx>`.
+ // FIXME: Represent the above fact in the trait system somehow.
+ unsafe { std::mem::transmute::<Ty<'tcx>, Ty<'_>>(tcx.ty_error()) }
+ }
+}
+
+impl<'tcx> Value<'tcx> for ty::SymbolName<'_> {
+ fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self {
+ // SAFETY: This is never called when `Self` is not `SymbolName<'tcx>`.
+ // FIXME: Represent the above fact in the trait system somehow.
+ unsafe {
+ std::mem::transmute::<ty::SymbolName<'tcx>, ty::SymbolName<'_>>(ty::SymbolName::new(
+ *tcx, "<error>",
+ ))
+ }
+ }
+}
+
+impl<'tcx> Value<'tcx> for AdtSizedConstraint<'_> {
+ fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self {
+ // SAFETY: This is never called when `Self` is not `AdtSizedConstraint<'tcx>`.
+ // FIXME: Represent the above fact in the trait system somehow.
+ unsafe {
+ std::mem::transmute::<AdtSizedConstraint<'tcx>, AdtSizedConstraint<'_>>(
+ AdtSizedConstraint(tcx.intern_type_list(&[tcx.ty_error()])),
+ )
+ }
+ }
+}