From 698f8c2f01ea549d77d7dc3338a12e04c11057b9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 17 Apr 2024 14:02:58 +0200 Subject: Adding upstream version 1.64.0+dfsg1. Signed-off-by: Daniel Baumann --- compiler/rustc_query_impl/Cargo.toml | 26 + compiler/rustc_query_impl/src/README.md | 3 + compiler/rustc_query_impl/src/keys.rs | 545 +++++++++++ compiler/rustc_query_impl/src/lib.rs | 63 ++ compiler/rustc_query_impl/src/on_disk_cache.rs | 1012 ++++++++++++++++++++ compiler/rustc_query_impl/src/plumbing.rs | 558 +++++++++++ compiler/rustc_query_impl/src/profiling_support.rs | 324 +++++++ compiler/rustc_query_impl/src/values.rs | 45 + 8 files changed, 2576 insertions(+) create mode 100644 compiler/rustc_query_impl/Cargo.toml create mode 100644 compiler/rustc_query_impl/src/README.md create mode 100644 compiler/rustc_query_impl/src/keys.rs create mode 100644 compiler/rustc_query_impl/src/lib.rs create mode 100644 compiler/rustc_query_impl/src/on_disk_cache.rs create mode 100644 compiler/rustc_query_impl/src/plumbing.rs create mode 100644 compiler/rustc_query_impl/src/profiling_support.rs create mode 100644 compiler/rustc_query_impl/src/values.rs (limited to 'compiler/rustc_query_impl') diff --git a/compiler/rustc_query_impl/Cargo.toml b/compiler/rustc_query_impl/Cargo.toml new file mode 100644 index 000000000..5673bb83b --- /dev/null +++ b/compiler/rustc_query_impl/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "rustc_query_impl" +version = "0.0.0" +edition = "2021" + +[lib] +doctest = false + +[dependencies] +measureme = "10.0.0" +rustc-rayon-core = { version = "0.4.0", optional = true } +rustc_ast = { path = "../rustc_ast" } +rustc_data_structures = { path = "../rustc_data_structures" } +rustc_errors = { path = "../rustc_errors" } +rustc_hir = { path = "../rustc_hir" } +rustc_index = { path = "../rustc_index" } +rustc_macros = { path = "../rustc_macros" } +rustc_middle = { path = "../rustc_middle" } +rustc_query_system = { path = "../rustc_query_system" } +rustc_serialize = { path = "../rustc_serialize" } +rustc_session = { path = "../rustc_session" } +rustc_span = { path = "../rustc_span" } +tracing = "0.1" + +[features] +rustc_use_parallel_compiler = ["rustc-rayon-core", "rustc_query_system/rustc_use_parallel_compiler"] diff --git a/compiler/rustc_query_impl/src/README.md b/compiler/rustc_query_impl/src/README.md new file mode 100644 index 000000000..8ec07b9fd --- /dev/null +++ b/compiler/rustc_query_impl/src/README.md @@ -0,0 +1,3 @@ +For more information about how the query system works, see the [rustc dev guide]. + +[rustc dev guide]: https://rustc-dev-guide.rust-lang.org/query.html diff --git a/compiler/rustc_query_impl/src/keys.rs b/compiler/rustc_query_impl/src/keys.rs new file mode 100644 index 000000000..49175e97f --- /dev/null +++ b/compiler/rustc_query_impl/src/keys.rs @@ -0,0 +1,545 @@ +//! Defines the set of legal keys that can be used in queries. + +use rustc_hir::def_id::{CrateNum, DefId, LocalDefId, LOCAL_CRATE}; +use rustc_middle::infer::canonical::Canonical; +use rustc_middle::mir; +use rustc_middle::traits; +use rustc_middle::ty::fast_reject::SimplifiedType; +use rustc_middle::ty::subst::{GenericArg, SubstsRef}; +use rustc_middle::ty::{self, layout::TyAndLayout, Ty, TyCtxt}; +use rustc_span::symbol::{Ident, Symbol}; +use rustc_span::{Span, DUMMY_SP}; + +/// The `Key` trait controls what types can legally be used as the key +/// for a query. +pub trait Key { + /// Given an instance of this key, what crate is it referring to? + /// This is used to find the provider. + fn query_crate_is_local(&self) -> bool; + + /// In the event that a cycle occurs, if no explicit span has been + /// given for a query with key `self`, what span should we use? + fn default_span(&self, tcx: TyCtxt<'_>) -> Span; + + /// If the key is a [`DefId`] or `DefId`--equivalent, return that `DefId`. + /// Otherwise, return `None`. + fn key_as_def_id(&self) -> Option { + None + } +} + +impl Key for () { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for ty::InstanceDef<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + tcx.def_span(self.def_id()) + } +} + +impl<'tcx> Key for ty::Instance<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + tcx.def_span(self.def_id()) + } +} + +impl<'tcx> Key for mir::interpret::GlobalId<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.instance.default_span(tcx) + } +} + +impl<'tcx> Key for (Ty<'tcx>, Option>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for mir::interpret::LitToConstInput<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl Key for CrateNum { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + *self == LOCAL_CRATE + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl Key for LocalDefId { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.to_def_id().default_span(tcx) + } + fn key_as_def_id(&self) -> Option { + Some(self.to_def_id()) + } +} + +impl Key for DefId { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + tcx.def_span(*self) + } + #[inline(always)] + fn key_as_def_id(&self) -> Option { + Some(*self) + } +} + +impl Key for ty::WithOptConstParam { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.did.default_span(tcx) + } +} + +impl Key for SimplifiedType { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl Key for (DefId, DefId) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.0.krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.1.default_span(tcx) + } +} + +impl<'tcx> Key for (ty::Instance<'tcx>, LocalDefId) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.0.default_span(tcx) + } +} + +impl Key for (DefId, LocalDefId) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.0.krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.1.default_span(tcx) + } +} + +impl Key for (LocalDefId, DefId) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.0.default_span(tcx) + } +} + +impl Key for (LocalDefId, LocalDefId) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.0.default_span(tcx) + } +} + +impl Key for (DefId, Option) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.0.krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + tcx.def_span(self.0) + } + #[inline(always)] + fn key_as_def_id(&self) -> Option { + Some(self.0) + } +} + +impl Key for (DefId, LocalDefId, Ident) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.0.krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.1.default_span(tcx) + } +} + +impl Key for (CrateNum, DefId) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.0 == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.1.default_span(tcx) + } +} + +impl Key for (CrateNum, SimplifiedType) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.0 == LOCAL_CRATE + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl Key for (DefId, SimplifiedType) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.0.krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.0.default_span(tcx) + } +} + +impl<'tcx> Key for SubstsRef<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for (DefId, SubstsRef<'tcx>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.0.krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.0.default_span(tcx) + } +} + +impl<'tcx> Key for (ty::Unevaluated<'tcx, ()>, ty::Unevaluated<'tcx, ()>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + (self.0).def.did.krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + (self.0).def.did.default_span(tcx) + } +} + +impl<'tcx> Key for (LocalDefId, DefId, SubstsRef<'tcx>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.0.default_span(tcx) + } +} + +impl<'tcx> Key for (ty::ParamEnv<'tcx>, ty::PolyTraitRef<'tcx>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.1.def_id().krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + tcx.def_span(self.1.def_id()) + } +} + +impl<'tcx> Key for (ty::Const<'tcx>, mir::Field) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for mir::interpret::ConstAlloc<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for ty::PolyTraitRef<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.def_id().krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + tcx.def_span(self.def_id()) + } +} + +impl<'tcx> Key for ty::PolyExistentialTraitRef<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.def_id().krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + tcx.def_span(self.def_id()) + } +} + +impl<'tcx> Key for (ty::PolyTraitRef<'tcx>, ty::PolyTraitRef<'tcx>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.0.def_id().krate == LOCAL_CRATE + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + tcx.def_span(self.0.def_id()) + } +} + +impl<'tcx> Key for GenericArg<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for mir::ConstantKind<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for ty::Const<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for Ty<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for TyAndLayout<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for (Ty<'tcx>, Ty<'tcx>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for &'tcx ty::List> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for ty::ParamEnv<'tcx> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx, T: Key> Key for ty::ParamEnvAnd<'tcx, T> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + self.value.query_crate_is_local() + } + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.value.default_span(tcx) + } +} + +impl Key for Symbol { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl Key for Option { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +/// Canonical query goals correspond to abstract trait operations that +/// are not tied to any crate in particular. +impl<'tcx, T> Key for Canonical<'tcx, T> { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl Key for (Symbol, u32, u32) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for (DefId, Ty<'tcx>, SubstsRef<'tcx>, ty::ParamEnv<'tcx>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for (ty::Predicate<'tcx>, traits::WellFormedLoc) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, _tcx: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for (ty::PolyFnSig<'tcx>, &'tcx ty::List>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} + +impl<'tcx> Key for (ty::Instance<'tcx>, &'tcx ty::List>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, tcx: TyCtxt<'_>) -> Span { + self.0.default_span(tcx) + } +} + +impl<'tcx> Key for (Ty<'tcx>, ty::ValTree<'tcx>) { + #[inline(always)] + fn query_crate_is_local(&self) -> bool { + true + } + + fn default_span(&self, _: TyCtxt<'_>) -> Span { + DUMMY_SP + } +} diff --git a/compiler/rustc_query_impl/src/lib.rs b/compiler/rustc_query_impl/src/lib.rs new file mode 100644 index 000000000..eda61df77 --- /dev/null +++ b/compiler/rustc_query_impl/src/lib.rs @@ -0,0 +1,63 @@ +//! Support for serializing the dep-graph and reloading it. + +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] +#![feature(min_specialization)] +#![feature(never_type)] +#![feature(once_cell)] +#![feature(rustc_attrs)] +#![recursion_limit = "256"] +#![allow(rustc::potential_query_instability)] + +#[macro_use] +extern crate rustc_macros; +#[macro_use] +extern crate rustc_middle; + +use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; +use rustc_data_structures::sync::AtomicU64; +use rustc_middle::arena::Arena; +use rustc_middle::dep_graph::{self, DepKindStruct, SerializedDepNodeIndex}; +use rustc_middle::ty::query::{query_keys, query_storage, query_stored, query_values}; +use rustc_middle::ty::query::{ExternProviders, Providers, QueryEngine}; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_span::def_id::{LocalDefId, LOCAL_CRATE}; +use rustc_span::Span; + +#[macro_use] +mod plumbing; +pub use plumbing::QueryCtxt; +use rustc_query_system::query::*; +#[cfg(parallel_compiler)] +pub use rustc_query_system::query::{deadlock, QueryContext}; + +mod keys; +use keys::Key; + +mod values; +use self::values::Value; + +pub use rustc_query_system::query::QueryConfig; +pub(crate) use rustc_query_system::query::{QueryDescription, QueryVTable}; + +mod on_disk_cache; +pub use on_disk_cache::OnDiskCache; + +mod profiling_support; +pub use self::profiling_support::alloc_self_profile_query_strings; + +fn describe_as_module(def_id: LocalDefId, tcx: TyCtxt<'_>) -> String { + if def_id.is_top_level_module() { + "top-level module".to_string() + } else { + format!("module `{}`", tcx.def_path_str(def_id.to_def_id())) + } +} + +rustc_query_append! { [define_queries!][<'tcx>] } + +impl<'tcx> Queries<'tcx> { + // Force codegen in the dyn-trait transformation in this crate. + pub fn as_dyn(&'tcx self) -> &'tcx dyn QueryEngine<'tcx> { + self + } +} diff --git a/compiler/rustc_query_impl/src/on_disk_cache.rs b/compiler/rustc_query_impl/src/on_disk_cache.rs new file mode 100644 index 000000000..56fd90c98 --- /dev/null +++ b/compiler/rustc_query_impl/src/on_disk_cache.rs @@ -0,0 +1,1012 @@ +use crate::QueryCtxt; +use rustc_data_structures::fx::{FxHashMap, FxHashSet, FxIndexSet}; +use rustc_data_structures::memmap::Mmap; +use rustc_data_structures::sync::{HashMapExt, Lock, Lrc, RwLock}; +use rustc_data_structures::unhash::UnhashMap; +use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, StableCrateId, LOCAL_CRATE}; +use rustc_hir::definitions::DefPathHash; +use rustc_index::vec::{Idx, IndexVec}; +use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; +use rustc_middle::mir::interpret::{AllocDecodingSession, AllocDecodingState}; +use rustc_middle::mir::{self, interpret}; +use rustc_middle::ty::codec::{RefDecodable, TyDecoder, TyEncoder}; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_query_system::dep_graph::DepContext; +use rustc_query_system::query::{QueryCache, QueryContext, QuerySideEffects}; +use rustc_serialize::{ + opaque::{FileEncodeResult, FileEncoder, IntEncodedWithFixedSize, MemDecoder}, + Decodable, Decoder, Encodable, Encoder, +}; +use rustc_session::Session; +use rustc_span::hygiene::{ + ExpnId, HygieneDecodeContext, HygieneEncodeContext, SyntaxContext, SyntaxContextData, +}; +use rustc_span::source_map::{SourceMap, StableSourceFileId}; +use rustc_span::CachingSourceMapView; +use rustc_span::{BytePos, ExpnData, ExpnHash, Pos, SourceFile, Span}; +use std::io; +use std::mem; + +const TAG_FILE_FOOTER: u128 = 0xC0FFEE_C0FFEE_C0FFEE_C0FFEE_C0FFEE; + +// A normal span encoded with both location information and a `SyntaxContext` +const TAG_FULL_SPAN: u8 = 0; +// A partial span with no location information, encoded only with a `SyntaxContext` +const TAG_PARTIAL_SPAN: u8 = 1; +const TAG_RELATIVE_SPAN: u8 = 2; + +const TAG_SYNTAX_CONTEXT: u8 = 0; +const TAG_EXPN_DATA: u8 = 1; + +/// Provides an interface to incremental compilation data cached from the +/// previous compilation session. This data will eventually include the results +/// of a few selected queries (like `typeck` and `mir_optimized`) and +/// any side effects that have been emitted during a query. +pub struct OnDiskCache<'sess> { + // The complete cache data in serialized form. + serialized_data: RwLock>, + + // Collects all `QuerySideEffects` created during the current compilation + // session. + current_side_effects: Lock>, + + source_map: &'sess SourceMap, + file_index_to_stable_id: FxHashMap, + + // Caches that are populated lazily during decoding. + file_index_to_file: Lock>>, + + // A map from dep-node to the position of the cached query result in + // `serialized_data`. + query_result_index: FxHashMap, + + // A map from dep-node to the position of any associated `QuerySideEffects` in + // `serialized_data`. + prev_side_effects_index: FxHashMap, + + alloc_decoding_state: AllocDecodingState, + + // A map from syntax context ids to the position of their associated + // `SyntaxContextData`. We use a `u32` instead of a `SyntaxContext` + // to represent the fact that we are storing *encoded* ids. When we decode + // a `SyntaxContext`, a new id will be allocated from the global `HygieneData`, + // which will almost certainly be different than the serialized id. + syntax_contexts: FxHashMap, + // A map from the `DefPathHash` of an `ExpnId` to the position + // of their associated `ExpnData`. Ideally, we would store a `DefId`, + // but we need to decode this before we've constructed a `TyCtxt` (which + // makes it difficult to decode a `DefId`). + + // Note that these `DefPathHashes` correspond to both local and foreign + // `ExpnData` (e.g `ExpnData.krate` may not be `LOCAL_CRATE`). Alternatively, + // we could look up the `ExpnData` from the metadata of foreign crates, + // but it seemed easier to have `OnDiskCache` be independent of the `CStore`. + expn_data: UnhashMap, + // Additional information used when decoding hygiene data. + hygiene_context: HygieneDecodeContext, + // Maps `ExpnHash`es to their raw value from the *previous* + // compilation session. This is used as an initial 'guess' when + // we try to map an `ExpnHash` to its value in the current + // compilation session. + foreign_expn_data: UnhashMap, +} + +// This type is used only for serialization and deserialization. +#[derive(Encodable, Decodable)] +struct Footer { + file_index_to_stable_id: FxHashMap, + query_result_index: EncodedDepNodeIndex, + side_effects_index: EncodedDepNodeIndex, + // The location of all allocations. + interpret_alloc_index: Vec, + // See `OnDiskCache.syntax_contexts` + syntax_contexts: FxHashMap, + // See `OnDiskCache.expn_data` + expn_data: UnhashMap, + foreign_expn_data: UnhashMap, +} + +pub type EncodedDepNodeIndex = Vec<(SerializedDepNodeIndex, AbsoluteBytePos)>; + +#[derive(Copy, Clone, PartialEq, Eq, Hash, Debug, Encodable, Decodable)] +struct SourceFileIndex(u32); + +#[derive(Copy, Clone, Debug, Hash, Eq, PartialEq, Encodable, Decodable)] +pub struct AbsoluteBytePos(u32); + +impl AbsoluteBytePos { + fn new(pos: usize) -> AbsoluteBytePos { + debug_assert!(pos <= u32::MAX as usize); + AbsoluteBytePos(pos as u32) + } + + fn to_usize(self) -> usize { + self.0 as usize + } +} + +/// An `EncodedSourceFileId` is the same as a `StableSourceFileId` except that +/// the source crate is represented as a [StableCrateId] instead of as a +/// `CrateNum`. This way `EncodedSourceFileId` can be encoded and decoded +/// without any additional context, i.e. with a simple `opaque::Decoder` (which +/// is the only thing available when decoding the cache's [Footer]. +#[derive(Encodable, Decodable, Clone, Debug)] +struct EncodedSourceFileId { + file_name_hash: u64, + stable_crate_id: StableCrateId, +} + +impl EncodedSourceFileId { + fn translate(&self, tcx: TyCtxt<'_>) -> StableSourceFileId { + let cnum = tcx.stable_crate_id_to_crate_num(self.stable_crate_id); + StableSourceFileId { file_name_hash: self.file_name_hash, cnum } + } + + fn new(tcx: TyCtxt<'_>, file: &SourceFile) -> EncodedSourceFileId { + let source_file_id = StableSourceFileId::new(file); + EncodedSourceFileId { + file_name_hash: source_file_id.file_name_hash, + stable_crate_id: tcx.stable_crate_id(source_file_id.cnum), + } + } +} + +impl<'sess> rustc_middle::ty::OnDiskCache<'sess> for OnDiskCache<'sess> { + /// Creates a new `OnDiskCache` instance from the serialized data in `data`. + fn new(sess: &'sess Session, data: Mmap, start_pos: usize) -> Self { + debug_assert!(sess.opts.incremental.is_some()); + + // Wrap in a scope so we can borrow `data`. + let footer: Footer = { + let mut decoder = MemDecoder::new(&data, start_pos); + + // Decode the *position* of the footer, which can be found in the + // last 8 bytes of the file. + decoder.set_position(data.len() - IntEncodedWithFixedSize::ENCODED_SIZE); + let footer_pos = IntEncodedWithFixedSize::decode(&mut decoder).0 as usize; + + // Decode the file footer, which contains all the lookup tables, etc. + decoder.set_position(footer_pos); + + decode_tagged(&mut decoder, TAG_FILE_FOOTER) + }; + + Self { + serialized_data: RwLock::new(Some(data)), + file_index_to_stable_id: footer.file_index_to_stable_id, + file_index_to_file: Default::default(), + source_map: sess.source_map(), + current_side_effects: Default::default(), + query_result_index: footer.query_result_index.into_iter().collect(), + prev_side_effects_index: footer.side_effects_index.into_iter().collect(), + alloc_decoding_state: AllocDecodingState::new(footer.interpret_alloc_index), + syntax_contexts: footer.syntax_contexts, + expn_data: footer.expn_data, + foreign_expn_data: footer.foreign_expn_data, + hygiene_context: Default::default(), + } + } + + fn new_empty(source_map: &'sess SourceMap) -> Self { + Self { + serialized_data: RwLock::new(None), + file_index_to_stable_id: Default::default(), + file_index_to_file: Default::default(), + source_map, + current_side_effects: Default::default(), + query_result_index: Default::default(), + prev_side_effects_index: Default::default(), + alloc_decoding_state: AllocDecodingState::new(Vec::new()), + syntax_contexts: FxHashMap::default(), + expn_data: UnhashMap::default(), + foreign_expn_data: UnhashMap::default(), + hygiene_context: Default::default(), + } + } + + /// Execute all cache promotions and release the serialized backing Mmap. + /// + /// Cache promotions require invoking queries, which needs to read the serialized data. + /// In order to serialize the new on-disk cache, the former on-disk cache file needs to be + /// deleted, hence we won't be able to refer to its memmapped data. + fn drop_serialized_data(&self, tcx: TyCtxt<'_>) { + // Load everything into memory so we can write it out to the on-disk + // cache. The vast majority of cacheable query results should already + // be in memory, so this should be a cheap operation. + // Do this *before* we clone 'latest_foreign_def_path_hashes', since + // loading existing queries may cause us to create new DepNodes, which + // may in turn end up invoking `store_foreign_def_id_hash` + tcx.dep_graph.exec_cache_promotions(tcx); + + *self.serialized_data.write() = None; + } + + fn serialize<'tcx>(&self, tcx: TyCtxt<'tcx>, encoder: FileEncoder) -> FileEncodeResult { + // Serializing the `DepGraph` should not modify it. + tcx.dep_graph.with_ignore(|| { + // Allocate `SourceFileIndex`es. + let (file_to_file_index, file_index_to_stable_id) = { + let files = tcx.sess.source_map().files(); + let mut file_to_file_index = + FxHashMap::with_capacity_and_hasher(files.len(), Default::default()); + let mut file_index_to_stable_id = + FxHashMap::with_capacity_and_hasher(files.len(), Default::default()); + + for (index, file) in files.iter().enumerate() { + let index = SourceFileIndex(index as u32); + let file_ptr: *const SourceFile = &**file as *const _; + file_to_file_index.insert(file_ptr, index); + let source_file_id = EncodedSourceFileId::new(tcx, &file); + file_index_to_stable_id.insert(index, source_file_id); + } + + (file_to_file_index, file_index_to_stable_id) + }; + + let hygiene_encode_context = HygieneEncodeContext::default(); + + let mut encoder = CacheEncoder { + tcx, + encoder, + type_shorthands: Default::default(), + predicate_shorthands: Default::default(), + interpret_allocs: Default::default(), + source_map: CachingSourceMapView::new(tcx.sess.source_map()), + file_to_file_index, + hygiene_context: &hygiene_encode_context, + }; + + // Encode query results. + let mut query_result_index = EncodedDepNodeIndex::new(); + + tcx.sess.time("encode_query_results", || { + let enc = &mut encoder; + let qri = &mut query_result_index; + QueryCtxt::from_tcx(tcx).encode_query_results(enc, qri); + }); + + // Encode side effects. + let side_effects_index: EncodedDepNodeIndex = self + .current_side_effects + .borrow() + .iter() + .map(|(dep_node_index, side_effects)| { + let pos = AbsoluteBytePos::new(encoder.position()); + let dep_node_index = SerializedDepNodeIndex::new(dep_node_index.index()); + encoder.encode_tagged(dep_node_index, side_effects); + + (dep_node_index, pos) + }) + .collect(); + + let interpret_alloc_index = { + let mut interpret_alloc_index = Vec::new(); + let mut n = 0; + loop { + let new_n = encoder.interpret_allocs.len(); + // If we have found new IDs, serialize those too. + if n == new_n { + // Otherwise, abort. + break; + } + interpret_alloc_index.reserve(new_n - n); + for idx in n..new_n { + let id = encoder.interpret_allocs[idx]; + let pos = encoder.position() as u32; + interpret_alloc_index.push(pos); + interpret::specialized_encode_alloc_id(&mut encoder, tcx, id); + } + n = new_n; + } + interpret_alloc_index + }; + + let mut syntax_contexts = FxHashMap::default(); + let mut expn_data = UnhashMap::default(); + let mut foreign_expn_data = UnhashMap::default(); + + // Encode all hygiene data (`SyntaxContextData` and `ExpnData`) from the current + // session. + + hygiene_encode_context.encode( + &mut encoder, + |encoder, index, ctxt_data| { + let pos = AbsoluteBytePos::new(encoder.position()); + encoder.encode_tagged(TAG_SYNTAX_CONTEXT, ctxt_data); + syntax_contexts.insert(index, pos); + }, + |encoder, expn_id, data, hash| { + if expn_id.krate == LOCAL_CRATE { + let pos = AbsoluteBytePos::new(encoder.position()); + encoder.encode_tagged(TAG_EXPN_DATA, data); + expn_data.insert(hash, pos); + } else { + foreign_expn_data.insert(hash, expn_id.local_id.as_u32()); + } + }, + ); + + // `Encode the file footer. + let footer_pos = encoder.position() as u64; + encoder.encode_tagged( + TAG_FILE_FOOTER, + &Footer { + file_index_to_stable_id, + query_result_index, + side_effects_index, + interpret_alloc_index, + syntax_contexts, + expn_data, + foreign_expn_data, + }, + ); + + // Encode the position of the footer as the last 8 bytes of the + // file so we know where to look for it. + IntEncodedWithFixedSize(footer_pos).encode(&mut encoder.encoder); + + // DO NOT WRITE ANYTHING TO THE ENCODER AFTER THIS POINT! The address + // of the footer must be the last thing in the data stream. + + encoder.finish() + }) + } +} + +impl<'sess> OnDiskCache<'sess> { + pub fn as_dyn(&self) -> &dyn rustc_middle::ty::OnDiskCache<'sess> { + self as _ + } + + /// Loads a `QuerySideEffects` created during the previous compilation session. + pub fn load_side_effects( + &self, + tcx: TyCtxt<'_>, + dep_node_index: SerializedDepNodeIndex, + ) -> QuerySideEffects { + let side_effects: Option = + self.load_indexed(tcx, dep_node_index, &self.prev_side_effects_index); + + side_effects.unwrap_or_default() + } + + /// Stores a `QuerySideEffects` emitted during the current compilation session. + /// Anything stored like this will be available via `load_side_effects` in + /// the next compilation session. + #[inline(never)] + #[cold] + pub fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) { + let mut current_side_effects = self.current_side_effects.borrow_mut(); + let prev = current_side_effects.insert(dep_node_index, side_effects); + debug_assert!(prev.is_none()); + } + + /// Returns the cached query result if there is something in the cache for + /// the given `SerializedDepNodeIndex`; otherwise returns `None`. + pub fn try_load_query_result<'tcx, T>( + &self, + tcx: TyCtxt<'tcx>, + dep_node_index: SerializedDepNodeIndex, + ) -> Option + where + T: for<'a> Decodable>, + { + self.load_indexed(tcx, dep_node_index, &self.query_result_index) + } + + /// Stores side effect emitted during computation of an anonymous query. + /// Since many anonymous queries can share the same `DepNode`, we aggregate + /// them -- as opposed to regular queries where we assume that there is a + /// 1:1 relationship between query-key and `DepNode`. + #[inline(never)] + #[cold] + pub fn store_side_effects_for_anon_node( + &self, + dep_node_index: DepNodeIndex, + side_effects: QuerySideEffects, + ) { + let mut current_side_effects = self.current_side_effects.borrow_mut(); + + let x = current_side_effects.entry(dep_node_index).or_default(); + x.append(side_effects); + } + + fn load_indexed<'tcx, T>( + &self, + tcx: TyCtxt<'tcx>, + dep_node_index: SerializedDepNodeIndex, + index: &FxHashMap, + ) -> Option + where + T: for<'a> Decodable>, + { + let pos = index.get(&dep_node_index).cloned()?; + + self.with_decoder(tcx, pos, |decoder| Some(decode_tagged(decoder, dep_node_index))) + } + + fn with_decoder<'a, 'tcx, T, F: for<'s> FnOnce(&mut CacheDecoder<'s, 'tcx>) -> T>( + &'sess self, + tcx: TyCtxt<'tcx>, + pos: AbsoluteBytePos, + f: F, + ) -> T + where + T: Decodable>, + { + let serialized_data = self.serialized_data.read(); + let mut decoder = CacheDecoder { + tcx, + opaque: MemDecoder::new(serialized_data.as_deref().unwrap_or(&[]), pos.to_usize()), + source_map: self.source_map, + file_index_to_file: &self.file_index_to_file, + file_index_to_stable_id: &self.file_index_to_stable_id, + alloc_decoding_session: self.alloc_decoding_state.new_decoding_session(), + syntax_contexts: &self.syntax_contexts, + expn_data: &self.expn_data, + foreign_expn_data: &self.foreign_expn_data, + hygiene_context: &self.hygiene_context, + }; + f(&mut decoder) + } +} + +//- DECODING ------------------------------------------------------------------- + +/// A decoder that can read from the incremental compilation cache. It is similar to the one +/// we use for crate metadata decoding in that it can rebase spans and eventually +/// will also handle things that contain `Ty` instances. +pub struct CacheDecoder<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + opaque: MemDecoder<'a>, + source_map: &'a SourceMap, + file_index_to_file: &'a Lock>>, + file_index_to_stable_id: &'a FxHashMap, + alloc_decoding_session: AllocDecodingSession<'a>, + syntax_contexts: &'a FxHashMap, + expn_data: &'a UnhashMap, + foreign_expn_data: &'a UnhashMap, + hygiene_context: &'a HygieneDecodeContext, +} + +impl<'a, 'tcx> CacheDecoder<'a, 'tcx> { + fn file_index_to_file(&self, index: SourceFileIndex) -> Lrc { + let CacheDecoder { + tcx, + ref file_index_to_file, + ref file_index_to_stable_id, + ref source_map, + .. + } = *self; + + file_index_to_file + .borrow_mut() + .entry(index) + .or_insert_with(|| { + let stable_id = file_index_to_stable_id[&index].translate(tcx); + + // If this `SourceFile` is from a foreign crate, then make sure + // that we've imported all of the source files from that crate. + // This has usually already been done during macro invocation. + // However, when encoding query results like `TypeckResults`, + // we might encode an `AdtDef` for a foreign type (because it + // was referenced in the body of the function). There is no guarantee + // that we will load the source files from that crate during macro + // expansion, so we use `import_source_files` to ensure that the foreign + // source files are actually imported before we call `source_file_by_stable_id`. + if stable_id.cnum != LOCAL_CRATE { + self.tcx.cstore_untracked().import_source_files(self.tcx.sess, stable_id.cnum); + } + + source_map + .source_file_by_stable_id(stable_id) + .expect("failed to lookup `SourceFile` in new context") + }) + .clone() + } +} + +trait DecoderWithPosition: Decoder { + fn position(&self) -> usize; +} + +impl<'a> DecoderWithPosition for MemDecoder<'a> { + fn position(&self) -> usize { + self.position() + } +} + +impl<'a, 'tcx> DecoderWithPosition for CacheDecoder<'a, 'tcx> { + fn position(&self) -> usize { + self.opaque.position() + } +} + +// Decodes something that was encoded with `encode_tagged()` and verify that the +// tag matches and the correct amount of bytes was read. +fn decode_tagged(decoder: &mut D, expected_tag: T) -> V +where + T: Decodable + Eq + std::fmt::Debug, + V: Decodable, + D: DecoderWithPosition, +{ + let start_pos = decoder.position(); + + let actual_tag = T::decode(decoder); + assert_eq!(actual_tag, expected_tag); + let value = V::decode(decoder); + let end_pos = decoder.position(); + + let expected_len: u64 = Decodable::decode(decoder); + assert_eq!((end_pos - start_pos) as u64, expected_len); + + value +} + +impl<'a, 'tcx> TyDecoder for CacheDecoder<'a, 'tcx> { + type I = TyCtxt<'tcx>; + const CLEAR_CROSS_CRATE: bool = false; + + #[inline] + fn interner(&self) -> TyCtxt<'tcx> { + self.tcx + } + + #[inline] + fn position(&self) -> usize { + self.opaque.position() + } + + #[inline] + fn peek_byte(&self) -> u8 { + self.opaque.data[self.opaque.position()] + } + + fn cached_ty_for_shorthand(&mut self, shorthand: usize, or_insert_with: F) -> Ty<'tcx> + where + F: FnOnce(&mut Self) -> Ty<'tcx>, + { + let tcx = self.tcx; + + let cache_key = ty::CReaderCacheKey { cnum: None, pos: shorthand }; + + if let Some(&ty) = tcx.ty_rcache.borrow().get(&cache_key) { + return ty; + } + + let ty = or_insert_with(self); + // This may overwrite the entry, but it should overwrite with the same value. + tcx.ty_rcache.borrow_mut().insert_same(cache_key, ty); + ty + } + + fn with_position(&mut self, pos: usize, f: F) -> R + where + F: FnOnce(&mut Self) -> R, + { + debug_assert!(pos < self.opaque.data.len()); + + let new_opaque = MemDecoder::new(self.opaque.data, pos); + let old_opaque = mem::replace(&mut self.opaque, new_opaque); + let r = f(self); + self.opaque = old_opaque; + r + } + + fn decode_alloc_id(&mut self) -> interpret::AllocId { + let alloc_decoding_session = self.alloc_decoding_session; + alloc_decoding_session.decode_alloc_id(self) + } +} + +rustc_middle::implement_ty_decoder!(CacheDecoder<'a, 'tcx>); + +// This ensures that the `Decodable::decode` specialization for `Vec` is used +// when a `CacheDecoder` is passed to `Decodable::decode`. Unfortunately, we have to manually opt +// into specializations this way, given how `CacheDecoder` and the decoding traits currently work. +impl<'a, 'tcx> Decodable> for Vec { + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + Decodable::decode(&mut d.opaque) + } +} + +impl<'a, 'tcx> Decodable> for SyntaxContext { + fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self { + let syntax_contexts = decoder.syntax_contexts; + rustc_span::hygiene::decode_syntax_context(decoder, decoder.hygiene_context, |this, id| { + // This closure is invoked if we haven't already decoded the data for the `SyntaxContext` we are deserializing. + // We look up the position of the associated `SyntaxData` and decode it. + let pos = syntax_contexts.get(&id).unwrap(); + this.with_position(pos.to_usize(), |decoder| { + let data: SyntaxContextData = decode_tagged(decoder, TAG_SYNTAX_CONTEXT); + data + }) + }) + } +} + +impl<'a, 'tcx> Decodable> for ExpnId { + fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self { + let hash = ExpnHash::decode(decoder); + if hash.is_root() { + return ExpnId::root(); + } + + if let Some(expn_id) = ExpnId::from_hash(hash) { + return expn_id; + } + + let krate = decoder.tcx.stable_crate_id_to_crate_num(hash.stable_crate_id()); + + let expn_id = if krate == LOCAL_CRATE { + // We look up the position of the associated `ExpnData` and decode it. + let pos = decoder + .expn_data + .get(&hash) + .unwrap_or_else(|| panic!("Bad hash {:?} (map {:?})", hash, decoder.expn_data)); + + let data: ExpnData = decoder + .with_position(pos.to_usize(), |decoder| decode_tagged(decoder, TAG_EXPN_DATA)); + let expn_id = rustc_span::hygiene::register_local_expn_id(data, hash); + + #[cfg(debug_assertions)] + { + use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; + let local_hash: u64 = decoder.tcx.with_stable_hashing_context(|mut hcx| { + let mut hasher = StableHasher::new(); + expn_id.expn_data().hash_stable(&mut hcx, &mut hasher); + hasher.finish() + }); + debug_assert_eq!(hash.local_hash(), local_hash); + } + + expn_id + } else { + let index_guess = decoder.foreign_expn_data[&hash]; + decoder.tcx.cstore_untracked().expn_hash_to_expn_id( + decoder.tcx.sess, + krate, + index_guess, + hash, + ) + }; + + debug_assert_eq!(expn_id.krate, krate); + expn_id + } +} + +impl<'a, 'tcx> Decodable> for Span { + fn decode(decoder: &mut CacheDecoder<'a, 'tcx>) -> Self { + let ctxt = SyntaxContext::decode(decoder); + let parent = Option::::decode(decoder); + let tag: u8 = Decodable::decode(decoder); + + if tag == TAG_PARTIAL_SPAN { + return Span::new(BytePos(0), BytePos(0), ctxt, parent); + } else if tag == TAG_RELATIVE_SPAN { + let dlo = u32::decode(decoder); + let dto = u32::decode(decoder); + + let enclosing = decoder.tcx.source_span_untracked(parent.unwrap()).data_untracked(); + let span = Span::new( + enclosing.lo + BytePos::from_u32(dlo), + enclosing.lo + BytePos::from_u32(dto), + ctxt, + parent, + ); + + return span; + } else { + debug_assert_eq!(tag, TAG_FULL_SPAN); + } + + let file_lo_index = SourceFileIndex::decode(decoder); + let line_lo = usize::decode(decoder); + let col_lo = BytePos::decode(decoder); + let len = BytePos::decode(decoder); + + let file_lo = decoder.file_index_to_file(file_lo_index); + let lo = file_lo.lines(|lines| lines[line_lo - 1] + col_lo); + let hi = lo + len; + + Span::new(lo, hi, ctxt, parent) + } +} + +impl<'a, 'tcx> Decodable> for CrateNum { + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + let stable_id = StableCrateId::decode(d); + let cnum = d.tcx.stable_crate_id_to_crate_num(stable_id); + cnum + } +} + +// This impl makes sure that we get a runtime error when we try decode a +// `DefIndex` that is not contained in a `DefId`. Such a case would be problematic +// because we would not know how to transform the `DefIndex` to the current +// context. +impl<'a, 'tcx> Decodable> for DefIndex { + fn decode(_d: &mut CacheDecoder<'a, 'tcx>) -> DefIndex { + panic!("trying to decode `DefIndex` outside the context of a `DefId`") + } +} + +// Both the `CrateNum` and the `DefIndex` of a `DefId` can change in between two +// compilation sessions. We use the `DefPathHash`, which is stable across +// sessions, to map the old `DefId` to the new one. +impl<'a, 'tcx> Decodable> for DefId { + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + // Load the `DefPathHash` which is was we encoded the `DefId` as. + let def_path_hash = DefPathHash::decode(d); + + // Using the `DefPathHash`, we can lookup the new `DefId`. + // Subtle: We only encode a `DefId` as part of a query result. + // If we get to this point, then all of the query inputs were green, + // which means that the definition with this hash is guaranteed to + // still exist in the current compilation session. + d.tcx.def_path_hash_to_def_id(def_path_hash, &mut || { + panic!("Failed to convert DefPathHash {:?}", def_path_hash) + }) + } +} + +impl<'a, 'tcx> Decodable> for &'tcx FxHashSet { + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +impl<'a, 'tcx> Decodable> + for &'tcx IndexVec> +{ + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +impl<'a, 'tcx> Decodable> for &'tcx [ty::abstract_const::Node<'tcx>] { + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +impl<'a, 'tcx> Decodable> for &'tcx [(ty::Predicate<'tcx>, Span)] { + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +impl<'a, 'tcx> Decodable> for &'tcx [rustc_ast::InlineAsmTemplatePiece] { + fn decode(d: &mut CacheDecoder<'a, 'tcx>) -> Self { + RefDecodable::decode(d) + } +} + +macro_rules! impl_ref_decoder { + (<$tcx:tt> $($ty:ty,)*) => { + $(impl<'a, $tcx> Decodable> for &$tcx [$ty] { + fn decode(d: &mut CacheDecoder<'a, $tcx>) -> Self { + RefDecodable::decode(d) + } + })* + }; +} + +impl_ref_decoder! {<'tcx> + Span, + rustc_ast::Attribute, + rustc_span::symbol::Ident, + ty::Variance, + rustc_span::def_id::DefId, + rustc_span::def_id::LocalDefId, + (rustc_middle::middle::exported_symbols::ExportedSymbol<'tcx>, rustc_middle::middle::exported_symbols::SymbolExportInfo), +} + +//- ENCODING ------------------------------------------------------------------- + +/// An encoder that can write to the incremental compilation cache. +pub struct CacheEncoder<'a, 'tcx> { + tcx: TyCtxt<'tcx>, + encoder: FileEncoder, + type_shorthands: FxHashMap, usize>, + predicate_shorthands: FxHashMap, usize>, + interpret_allocs: FxIndexSet, + source_map: CachingSourceMapView<'tcx>, + file_to_file_index: FxHashMap<*const SourceFile, SourceFileIndex>, + hygiene_context: &'a HygieneEncodeContext, +} + +impl<'a, 'tcx> CacheEncoder<'a, 'tcx> { + fn source_file_index(&mut self, source_file: Lrc) -> SourceFileIndex { + self.file_to_file_index[&(&*source_file as *const SourceFile)] + } + + /// Encode something with additional information that allows to do some + /// sanity checks when decoding the data again. This method will first + /// encode the specified tag, then the given value, then the number of + /// bytes taken up by tag and value. On decoding, we can then verify that + /// we get the expected tag and read the expected number of bytes. + fn encode_tagged, V: Encodable>(&mut self, tag: T, value: &V) { + let start_pos = self.position(); + + tag.encode(self); + value.encode(self); + + let end_pos = self.position(); + ((end_pos - start_pos) as u64).encode(self); + } + + fn finish(self) -> Result { + self.encoder.finish() + } +} + +impl<'a, 'tcx> Encodable> for SyntaxContext { + fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) { + rustc_span::hygiene::raw_encode_syntax_context(*self, s.hygiene_context, s); + } +} + +impl<'a, 'tcx> Encodable> for ExpnId { + fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) { + s.hygiene_context.schedule_expn_data_for_encoding(*self); + self.expn_hash().encode(s); + } +} + +impl<'a, 'tcx> Encodable> for Span { + fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) { + let span_data = self.data_untracked(); + span_data.ctxt.encode(s); + span_data.parent.encode(s); + + if span_data.is_dummy() { + return TAG_PARTIAL_SPAN.encode(s); + } + + if let Some(parent) = span_data.parent { + let enclosing = s.tcx.source_span(parent).data_untracked(); + if enclosing.contains(span_data) { + TAG_RELATIVE_SPAN.encode(s); + (span_data.lo - enclosing.lo).to_u32().encode(s); + (span_data.hi - enclosing.lo).to_u32().encode(s); + return; + } + } + + let pos = s.source_map.byte_pos_to_line_and_col(span_data.lo); + let partial_span = match &pos { + Some((file_lo, _, _)) => !file_lo.contains(span_data.hi), + None => true, + }; + + if partial_span { + return TAG_PARTIAL_SPAN.encode(s); + } + + let (file_lo, line_lo, col_lo) = pos.unwrap(); + + let len = span_data.hi - span_data.lo; + + let source_file_index = s.source_file_index(file_lo); + + TAG_FULL_SPAN.encode(s); + source_file_index.encode(s); + line_lo.encode(s); + col_lo.encode(s); + len.encode(s); + } +} + +impl<'a, 'tcx> TyEncoder for CacheEncoder<'a, 'tcx> { + type I = TyCtxt<'tcx>; + const CLEAR_CROSS_CRATE: bool = false; + + fn position(&self) -> usize { + self.encoder.position() + } + fn type_shorthands(&mut self) -> &mut FxHashMap, usize> { + &mut self.type_shorthands + } + fn predicate_shorthands(&mut self) -> &mut FxHashMap, usize> { + &mut self.predicate_shorthands + } + fn encode_alloc_id(&mut self, alloc_id: &interpret::AllocId) { + let (index, _) = self.interpret_allocs.insert_full(*alloc_id); + + index.encode(self); + } +} + +impl<'a, 'tcx> Encodable> for CrateNum { + fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) { + s.tcx.stable_crate_id(*self).encode(s); + } +} + +impl<'a, 'tcx> Encodable> for DefId { + fn encode(&self, s: &mut CacheEncoder<'a, 'tcx>) { + s.tcx.def_path_hash(*self).encode(s); + } +} + +impl<'a, 'tcx> Encodable> for DefIndex { + fn encode(&self, _: &mut CacheEncoder<'a, 'tcx>) { + bug!("encoding `DefIndex` without context"); + } +} + +macro_rules! encoder_methods { + ($($name:ident($ty:ty);)*) => { + #[inline] + $(fn $name(&mut self, value: $ty) { + self.encoder.$name(value) + })* + } +} + +impl<'a, 'tcx> Encoder for CacheEncoder<'a, 'tcx> { + encoder_methods! { + emit_usize(usize); + emit_u128(u128); + emit_u64(u64); + emit_u32(u32); + emit_u16(u16); + emit_u8(u8); + + emit_isize(isize); + emit_i128(i128); + emit_i64(i64); + emit_i32(i32); + emit_i16(i16); + emit_i8(i8); + + emit_bool(bool); + emit_f64(f64); + emit_f32(f32); + emit_char(char); + emit_str(&str); + emit_raw_bytes(&[u8]); + } +} + +// This ensures that the `Encodable::encode` specialization for byte slices +// is used when a `CacheEncoder` having an `opaque::FileEncoder` is passed to `Encodable::encode`. +// Unfortunately, we have to manually opt into specializations this way, given how `CacheEncoder` +// and the encoding traits currently work. +impl<'a, 'tcx> Encodable> for [u8] { + fn encode(&self, e: &mut CacheEncoder<'a, 'tcx>) { + self.encode(&mut e.encoder); + } +} + +pub fn encode_query_results<'a, 'tcx, CTX, Q>( + tcx: CTX, + encoder: &mut CacheEncoder<'a, 'tcx>, + query_result_index: &mut EncodedDepNodeIndex, +) where + CTX: QueryContext + 'tcx, + Q: super::QueryDescription, + Q::Value: Encodable>, +{ + let _timer = tcx + .dep_context() + .profiler() + .extra_verbose_generic_activity("encode_query_results_for", std::any::type_name::()); + + assert!(Q::query_state(tcx).all_inactive()); + let cache = Q::query_cache(tcx); + cache.iter(&mut |key, value, dep_node| { + if Q::cache_on_disk(*tcx.dep_context(), &key) { + let dep_node = SerializedDepNodeIndex::new(dep_node.index()); + + // Record position of the cache entry. + query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.encoder.position()))); + + // Encode the type check tables with the `SerializedDepNodeIndex` + // as tag. + encoder.encode_tagged(dep_node, value); + } + }); +} diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs new file mode 100644 index 000000000..eda4401c8 --- /dev/null +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -0,0 +1,558 @@ +//! The implementation of the query system itself. This defines the macros that +//! generate the actual methods on tcx which find and execute the provider, +//! manage the caches, and so forth. + +use crate::{on_disk_cache, Queries}; +use rustc_middle::dep_graph::{DepNodeIndex, SerializedDepNodeIndex}; +use rustc_middle::ty::tls::{self, ImplicitCtxt}; +use rustc_middle::ty::TyCtxt; +use rustc_query_system::dep_graph::HasDepContext; +use rustc_query_system::query::{QueryContext, QueryJobId, QueryMap, QuerySideEffects}; + +use rustc_data_structures::sync::Lock; +use rustc_data_structures::thin_vec::ThinVec; +use rustc_errors::{Diagnostic, Handler}; + +use std::any::Any; +use std::num::NonZeroU64; + +#[derive(Copy, Clone)] +pub struct QueryCtxt<'tcx> { + pub tcx: TyCtxt<'tcx>, + pub queries: &'tcx Queries<'tcx>, +} + +impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> { + type Target = TyCtxt<'tcx>; + + #[inline] + fn deref(&self) -> &Self::Target { + &self.tcx + } +} + +impl<'tcx> HasDepContext for QueryCtxt<'tcx> { + type DepKind = rustc_middle::dep_graph::DepKind; + type DepContext = TyCtxt<'tcx>; + + #[inline] + fn dep_context(&self) -> &Self::DepContext { + &self.tcx + } +} + +impl QueryContext for QueryCtxt<'_> { + fn next_job_id(&self) -> QueryJobId { + QueryJobId( + NonZeroU64::new( + self.queries.jobs.fetch_add(1, rustc_data_structures::sync::Ordering::Relaxed), + ) + .unwrap(), + ) + } + + fn current_query_job(&self) -> Option { + tls::with_related_context(**self, |icx| icx.query) + } + + fn try_collect_active_jobs(&self) -> Option { + self.queries.try_collect_active_jobs(**self) + } + + // Interactions with on_disk_cache + fn load_side_effects(&self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects { + self.queries + .on_disk_cache + .as_ref() + .map(|c| c.load_side_effects(**self, prev_dep_node_index)) + .unwrap_or_default() + } + + fn store_side_effects(&self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) { + if let Some(c) = self.queries.on_disk_cache.as_ref() { + c.store_side_effects(dep_node_index, side_effects) + } + } + + fn store_side_effects_for_anon_node( + &self, + dep_node_index: DepNodeIndex, + side_effects: QuerySideEffects, + ) { + if let Some(c) = self.queries.on_disk_cache.as_ref() { + c.store_side_effects_for_anon_node(dep_node_index, side_effects) + } + } + + /// Executes a job by changing the `ImplicitCtxt` to point to the + /// new query job while it executes. It returns the diagnostics + /// captured during execution and the actual result. + #[inline(always)] + fn start_query( + &self, + token: QueryJobId, + diagnostics: Option<&Lock>>, + compute: impl FnOnce() -> R, + ) -> R { + // The `TyCtxt` stored in TLS has the same global interner lifetime + // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes + // when accessing the `ImplicitCtxt`. + tls::with_related_context(**self, move |current_icx| { + // Update the `ImplicitCtxt` to point to our new query job. + let new_icx = ImplicitCtxt { + tcx: **self, + query: Some(token), + diagnostics, + layout_depth: current_icx.layout_depth, + task_deps: current_icx.task_deps, + }; + + // Use the `ImplicitCtxt` while we execute the query. + tls::enter_context(&new_icx, |_| { + rustc_data_structures::stack::ensure_sufficient_stack(compute) + }) + }) + } +} + +impl<'tcx> QueryCtxt<'tcx> { + #[inline] + pub fn from_tcx(tcx: TyCtxt<'tcx>) -> Self { + let queries = tcx.queries.as_any(); + let queries = unsafe { + let queries = std::mem::transmute::<&dyn Any, &dyn Any>(queries); + let queries = queries.downcast_ref().unwrap(); + let queries = std::mem::transmute::<&Queries<'_>, &Queries<'_>>(queries); + queries + }; + QueryCtxt { tcx, queries } + } + + pub(crate) fn on_disk_cache(self) -> Option<&'tcx on_disk_cache::OnDiskCache<'tcx>> { + self.queries.on_disk_cache.as_ref() + } + + pub(super) fn encode_query_results( + self, + encoder: &mut on_disk_cache::CacheEncoder<'_, 'tcx>, + query_result_index: &mut on_disk_cache::EncodedDepNodeIndex, + ) { + macro_rules! encode_queries { + ($($query:ident,)*) => { + $( + on_disk_cache::encode_query_results::<_, super::queries::$query<'_>>( + self, + encoder, + query_result_index + ); + )* + } + } + + rustc_cached_queries!(encode_queries!); + } + + pub fn try_print_query_stack( + self, + query: Option, + handler: &Handler, + num_frames: Option, + ) -> usize { + rustc_query_system::query::print_query_stack(self, query, handler, num_frames) + } +} + +macro_rules! handle_cycle_error { + ([][$tcx: expr, $error:expr]) => {{ + $error.emit(); + Value::from_cycle_error($tcx) + }}; + ([(fatal_cycle) $($rest:tt)*][$tcx:expr, $error:expr]) => {{ + $error.emit(); + $tcx.sess.abort_if_errors(); + unreachable!() + }}; + ([(cycle_delay_bug) $($rest:tt)*][$tcx:expr, $error:expr]) => {{ + $error.delay_as_bug(); + Value::from_cycle_error($tcx) + }}; + ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { + handle_cycle_error!([$($modifiers)*][$($args)*]) + }; +} + +macro_rules! is_anon { + ([]) => {{ + false + }}; + ([(anon) $($rest:tt)*]) => {{ + true + }}; + ([$other:tt $($modifiers:tt)*]) => { + is_anon!([$($modifiers)*]) + }; +} + +macro_rules! is_eval_always { + ([]) => {{ + false + }}; + ([(eval_always) $($rest:tt)*]) => {{ + true + }}; + ([$other:tt $($modifiers:tt)*]) => { + is_eval_always!([$($modifiers)*]) + }; +} + +macro_rules! hash_result { + ([]) => {{ + Some(dep_graph::hash_result) + }}; + ([(no_hash) $($rest:tt)*]) => {{ + None + }}; + ([$other:tt $($modifiers:tt)*]) => { + hash_result!([$($modifiers)*]) + }; +} + +macro_rules! get_provider { + ([][$tcx:expr, $name:ident, $key:expr]) => {{ + $tcx.queries.local_providers.$name + }}; + ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{ + if $key.query_crate_is_local() { + $tcx.queries.local_providers.$name + } else { + $tcx.queries.extern_providers.$name + } + }}; + ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { + get_provider!([$($modifiers)*][$($args)*]) + }; +} + +macro_rules! opt_remap_env_constness { + ([][$name:ident]) => {}; + ([(remap_env_constness) $($rest:tt)*][$name:ident]) => { + let $name = $name.without_const(); + }; + ([$other:tt $($modifiers:tt)*][$name:ident]) => { + opt_remap_env_constness!([$($modifiers)*][$name]) + }; +} + +macro_rules! define_queries { + (<$tcx:tt> + $($(#[$attr:meta])* + [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => { + + define_queries_struct! { + tcx: $tcx, + input: ($(([$($modifiers)*] [$($attr)*] [$name]))*) + } + + mod make_query { + use super::*; + + // Create an eponymous constructor for each query. + $(#[allow(nonstandard_style)] $(#[$attr])* + pub fn $name<$tcx>(tcx: QueryCtxt<$tcx>, key: query_keys::$name<$tcx>) -> QueryStackFrame { + opt_remap_env_constness!([$($modifiers)*][key]); + let kind = dep_graph::DepKind::$name; + let name = stringify!($name); + // Disable visible paths printing for performance reasons. + // Showing visible path instead of any path is not that important in production. + let description = ty::print::with_no_visible_paths!( + // Force filename-line mode to avoid invoking `type_of` query. + ty::print::with_forced_impl_filename_line!( + queries::$name::describe(tcx, key) + ) + ); + let description = if tcx.sess.verbose() { + format!("{} [{}]", description, name) + } else { + description + }; + let span = if kind == dep_graph::DepKind::def_span { + // The `def_span` query is used to calculate `default_span`, + // so exit to avoid infinite recursion. + None + } else { + Some(key.default_span(*tcx)) + }; + let def_kind = if kind == dep_graph::DepKind::opt_def_kind { + // Try to avoid infinite recursion. + None + } else { + key.key_as_def_id() + .and_then(|def_id| def_id.as_local()) + .and_then(|def_id| tcx.opt_def_kind(def_id)) + }; + let hash = || { + tcx.with_stable_hashing_context(|mut hcx|{ + let mut hasher = StableHasher::new(); + std::mem::discriminant(&kind).hash_stable(&mut hcx, &mut hasher); + key.hash_stable(&mut hcx, &mut hasher); + hasher.finish::() + }) + }; + + QueryStackFrame::new(name, description, span, def_kind, hash) + })* + } + + #[allow(nonstandard_style)] + mod queries { + use std::marker::PhantomData; + + $(pub struct $name<$tcx> { + data: PhantomData<&$tcx ()> + })* + } + + $(impl<$tcx> QueryConfig for queries::$name<$tcx> { + type Key = query_keys::$name<$tcx>; + type Value = query_values::$name<$tcx>; + type Stored = query_stored::$name<$tcx>; + const NAME: &'static str = stringify!($name); + } + + impl<$tcx> QueryDescription> for queries::$name<$tcx> { + rustc_query_description! { $name<$tcx> } + + type Cache = query_storage::$name<$tcx>; + + #[inline(always)] + fn query_state<'a>(tcx: QueryCtxt<$tcx>) -> &'a QueryState + where QueryCtxt<$tcx>: 'a + { + &tcx.queries.$name + } + + #[inline(always)] + fn query_cache<'a>(tcx: QueryCtxt<$tcx>) -> &'a Self::Cache + where 'tcx:'a + { + &tcx.query_caches.$name + } + + #[inline] + fn make_vtable(tcx: QueryCtxt<'tcx>, key: &Self::Key) -> + QueryVTable, Self::Key, Self::Value> + { + let compute = get_provider!([$($modifiers)*][tcx, $name, key]); + let cache_on_disk = Self::cache_on_disk(tcx.tcx, key); + QueryVTable { + anon: is_anon!([$($modifiers)*]), + eval_always: is_eval_always!([$($modifiers)*]), + dep_kind: dep_graph::DepKind::$name, + hash_result: hash_result!([$($modifiers)*]), + handle_cycle_error: |tcx, mut error| handle_cycle_error!([$($modifiers)*][tcx, error]), + compute, + cache_on_disk, + try_load_from_disk: Self::TRY_LOAD_FROM_DISK, + } + } + })* + + #[allow(nonstandard_style)] + mod query_callbacks { + use super::*; + use rustc_middle::dep_graph::DepNode; + use rustc_middle::ty::query::query_keys; + use rustc_query_system::dep_graph::DepNodeParams; + use rustc_query_system::query::{force_query, QueryDescription}; + use rustc_query_system::dep_graph::FingerprintStyle; + + // We use this for most things when incr. comp. is turned off. + pub fn Null() -> DepKindStruct { + DepKindStruct { + is_anon: false, + is_eval_always: false, + fingerprint_style: FingerprintStyle::Unit, + force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)), + try_load_from_on_disk_cache: None, + } + } + + // We use this for the forever-red node. + pub fn Red() -> DepKindStruct { + DepKindStruct { + is_anon: false, + is_eval_always: false, + fingerprint_style: FingerprintStyle::Unit, + force_from_dep_node: Some(|_, dep_node| bug!("force_from_dep_node: encountered {:?}", dep_node)), + try_load_from_on_disk_cache: None, + } + } + + pub fn TraitSelect() -> DepKindStruct { + DepKindStruct { + is_anon: true, + is_eval_always: false, + fingerprint_style: FingerprintStyle::Unit, + force_from_dep_node: None, + try_load_from_on_disk_cache: None, + } + } + + pub fn CompileCodegenUnit() -> DepKindStruct { + DepKindStruct { + is_anon: false, + is_eval_always: false, + fingerprint_style: FingerprintStyle::Opaque, + force_from_dep_node: None, + try_load_from_on_disk_cache: None, + } + } + + pub fn CompileMonoItem() -> DepKindStruct { + DepKindStruct { + is_anon: false, + is_eval_always: false, + fingerprint_style: FingerprintStyle::Opaque, + force_from_dep_node: None, + try_load_from_on_disk_cache: None, + } + } + + $(pub(crate) fn $name()-> DepKindStruct { + let is_anon = is_anon!([$($modifiers)*]); + let is_eval_always = is_eval_always!([$($modifiers)*]); + + let fingerprint_style = + as DepNodeParams>>::fingerprint_style(); + + if is_anon || !fingerprint_style.reconstructible() { + return DepKindStruct { + is_anon, + is_eval_always, + fingerprint_style, + force_from_dep_node: None, + try_load_from_on_disk_cache: None, + } + } + + #[inline(always)] + fn recover<'tcx>(tcx: TyCtxt<'tcx>, dep_node: DepNode) -> Option> { + as DepNodeParams>>::recover(tcx, &dep_node) + } + + fn force_from_dep_node(tcx: TyCtxt<'_>, dep_node: DepNode) -> bool { + if let Some(key) = recover(tcx, dep_node) { + #[cfg(debug_assertions)] + let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); + let tcx = QueryCtxt::from_tcx(tcx); + force_query::, _>(tcx, key, dep_node); + true + } else { + false + } + } + + fn try_load_from_on_disk_cache(tcx: TyCtxt<'_>, dep_node: DepNode) { + debug_assert!(tcx.dep_graph.is_green(&dep_node)); + + let key = recover(tcx, dep_node).unwrap_or_else(|| panic!("Failed to recover key for {:?} with hash {}", dep_node, dep_node.hash)); + if queries::$name::cache_on_disk(tcx, &key) { + let _ = tcx.$name(key); + } + } + + DepKindStruct { + is_anon, + is_eval_always, + fingerprint_style, + force_from_dep_node: Some(force_from_dep_node), + try_load_from_on_disk_cache: Some(try_load_from_on_disk_cache), + } + })* + } + + pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct] { + arena.alloc_from_iter(make_dep_kind_array!(query_callbacks)) + } + } +} + +// FIXME(eddyb) this macro (and others?) use `$tcx` and `'tcx` interchangeably. +// We should either not take `$tcx` at all and use `'tcx` everywhere, or use +// `$tcx` everywhere (even if that isn't necessary due to lack of hygiene). +macro_rules! define_queries_struct { + (tcx: $tcx:tt, + input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => { + pub struct Queries<$tcx> { + local_providers: Box, + extern_providers: Box, + + pub on_disk_cache: Option>, + + jobs: AtomicU64, + + $($(#[$attr])* $name: QueryState>,)* + } + + impl<$tcx> Queries<$tcx> { + pub fn new( + local_providers: Providers, + extern_providers: ExternProviders, + on_disk_cache: Option>, + ) -> Self { + Queries { + local_providers: Box::new(local_providers), + extern_providers: Box::new(extern_providers), + on_disk_cache, + jobs: AtomicU64::new(1), + $($name: Default::default()),* + } + } + + pub(crate) fn try_collect_active_jobs( + &$tcx self, + tcx: TyCtxt<$tcx>, + ) -> Option { + let tcx = QueryCtxt { tcx, queries: self }; + let mut jobs = QueryMap::default(); + + $( + self.$name.try_collect_active_jobs( + tcx, + make_query::$name, + &mut jobs, + )?; + )* + + Some(jobs) + } + } + + impl<'tcx> QueryEngine<'tcx> for Queries<'tcx> { + fn as_any(&'tcx self) -> &'tcx dyn std::any::Any { + let this = unsafe { std::mem::transmute::<&Queries<'_>, &Queries<'_>>(self) }; + this as _ + } + + fn try_mark_green(&'tcx self, tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool { + let qcx = QueryCtxt { tcx, queries: self }; + tcx.dep_graph.try_mark_green(qcx, dep_node).is_some() + } + + $($(#[$attr])* + #[inline(always)] + #[tracing::instrument(level = "trace", skip(self, tcx))] + fn $name( + &'tcx self, + tcx: TyCtxt<$tcx>, + span: Span, + key: query_keys::$name<$tcx>, + mode: QueryMode, + ) -> Option> { + opt_remap_env_constness!([$($modifiers)*][key]); + let qcx = QueryCtxt { tcx, queries: self }; + get_query::, _>(qcx, span, key, mode) + })* + } + }; +} diff --git a/compiler/rustc_query_impl/src/profiling_support.rs b/compiler/rustc_query_impl/src/profiling_support.rs new file mode 100644 index 000000000..551f09420 --- /dev/null +++ b/compiler/rustc_query_impl/src/profiling_support.rs @@ -0,0 +1,324 @@ +use measureme::{StringComponent, StringId}; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::profiling::SelfProfiler; +use rustc_hir::def_id::{CrateNum, DefId, DefIndex, LocalDefId, LOCAL_CRATE}; +use rustc_hir::definitions::DefPathData; +use rustc_middle::ty::{TyCtxt, WithOptConstParam}; +use rustc_query_system::query::QueryCache; +use std::fmt::Debug; +use std::io::Write; + +struct QueryKeyStringCache { + def_id_cache: FxHashMap, +} + +impl QueryKeyStringCache { + fn new() -> QueryKeyStringCache { + QueryKeyStringCache { def_id_cache: Default::default() } + } +} + +struct QueryKeyStringBuilder<'p, 'c, 'tcx> { + profiler: &'p SelfProfiler, + tcx: TyCtxt<'tcx>, + string_cache: &'c mut QueryKeyStringCache, +} + +impl<'p, 'c, 'tcx> QueryKeyStringBuilder<'p, 'c, 'tcx> { + fn new( + profiler: &'p SelfProfiler, + tcx: TyCtxt<'tcx>, + string_cache: &'c mut QueryKeyStringCache, + ) -> QueryKeyStringBuilder<'p, 'c, 'tcx> { + QueryKeyStringBuilder { profiler, tcx, string_cache } + } + + // The current implementation is rather crude. In the future it might be a + // good idea to base this on `ty::print` in order to get nicer and more + // efficient query keys. + fn def_id_to_string_id(&mut self, def_id: DefId) -> StringId { + if let Some(&string_id) = self.string_cache.def_id_cache.get(&def_id) { + return string_id; + } + + let def_key = self.tcx.def_key(def_id); + + let (parent_string_id, start_index) = match def_key.parent { + Some(parent_index) => { + let parent_def_id = DefId { index: parent_index, krate: def_id.krate }; + + (self.def_id_to_string_id(parent_def_id), 0) + } + None => (StringId::INVALID, 2), + }; + + let dis_buffer = &mut [0u8; 16]; + let crate_name; + let other_name; + let name; + let dis; + let end_index; + + match def_key.disambiguated_data.data { + DefPathData::CrateRoot => { + crate_name = self.tcx.crate_name(def_id.krate); + name = crate_name.as_str(); + dis = ""; + end_index = 3; + } + other => { + other_name = other.to_string(); + name = other_name.as_str(); + if def_key.disambiguated_data.disambiguator == 0 { + dis = ""; + end_index = 3; + } else { + write!(&mut dis_buffer[..], "[{}]", def_key.disambiguated_data.disambiguator) + .unwrap(); + let end_of_dis = dis_buffer.iter().position(|&c| c == b']').unwrap(); + dis = std::str::from_utf8(&dis_buffer[..end_of_dis + 1]).unwrap(); + end_index = 4; + } + } + } + + let components = [ + StringComponent::Ref(parent_string_id), + StringComponent::Value("::"), + StringComponent::Value(name), + StringComponent::Value(dis), + ]; + + let string_id = self.profiler.alloc_string(&components[start_index..end_index]); + + self.string_cache.def_id_cache.insert(def_id, string_id); + + string_id + } +} + +trait IntoSelfProfilingString { + fn to_self_profile_string(&self, builder: &mut QueryKeyStringBuilder<'_, '_, '_>) -> StringId; +} + +// The default implementation of `IntoSelfProfilingString` just uses `Debug` +// which is slow and causes lots of duplication of string data. +// The specialized impls below take care of making the `DefId` case more +// efficient. +impl IntoSelfProfilingString for T { + default fn to_self_profile_string( + &self, + builder: &mut QueryKeyStringBuilder<'_, '_, '_>, + ) -> StringId { + let s = format!("{:?}", self); + builder.profiler.alloc_string(&s[..]) + } +} + +impl IntoSelfProfilingString for T { + fn to_self_profile_string(&self, builder: &mut QueryKeyStringBuilder<'_, '_, '_>) -> StringId { + self.spec_to_self_profile_string(builder) + } +} + +#[rustc_specialization_trait] +trait SpecIntoSelfProfilingString: Debug { + fn spec_to_self_profile_string( + &self, + builder: &mut QueryKeyStringBuilder<'_, '_, '_>, + ) -> StringId; +} + +impl SpecIntoSelfProfilingString for DefId { + fn spec_to_self_profile_string( + &self, + builder: &mut QueryKeyStringBuilder<'_, '_, '_>, + ) -> StringId { + builder.def_id_to_string_id(*self) + } +} + +impl SpecIntoSelfProfilingString for CrateNum { + fn spec_to_self_profile_string( + &self, + builder: &mut QueryKeyStringBuilder<'_, '_, '_>, + ) -> StringId { + builder.def_id_to_string_id(self.as_def_id()) + } +} + +impl SpecIntoSelfProfilingString for DefIndex { + fn spec_to_self_profile_string( + &self, + builder: &mut QueryKeyStringBuilder<'_, '_, '_>, + ) -> StringId { + builder.def_id_to_string_id(DefId { krate: LOCAL_CRATE, index: *self }) + } +} + +impl SpecIntoSelfProfilingString for LocalDefId { + fn spec_to_self_profile_string( + &self, + builder: &mut QueryKeyStringBuilder<'_, '_, '_>, + ) -> StringId { + builder.def_id_to_string_id(DefId { krate: LOCAL_CRATE, index: self.local_def_index }) + } +} + +impl SpecIntoSelfProfilingString for WithOptConstParam { + fn spec_to_self_profile_string( + &self, + builder: &mut QueryKeyStringBuilder<'_, '_, '_>, + ) -> StringId { + // We print `WithOptConstParam` values as tuples to make them shorter + // and more readable, without losing information: + // + // "WithOptConstParam { did: foo::bar, const_param_did: Some(foo::baz) }" + // becomes "(foo::bar, foo::baz)" and + // "WithOptConstParam { did: foo::bar, const_param_did: None }" + // becomes "(foo::bar, _)". + + let did = StringComponent::Ref(self.did.to_self_profile_string(builder)); + + let const_param_did = if let Some(const_param_did) = self.const_param_did { + let const_param_did = builder.def_id_to_string_id(const_param_did); + StringComponent::Ref(const_param_did) + } else { + StringComponent::Value("_") + }; + + let components = [ + StringComponent::Value("("), + did, + StringComponent::Value(", "), + const_param_did, + StringComponent::Value(")"), + ]; + + builder.profiler.alloc_string(&components[..]) + } +} + +impl SpecIntoSelfProfilingString for (T0, T1) +where + T0: SpecIntoSelfProfilingString, + T1: SpecIntoSelfProfilingString, +{ + fn spec_to_self_profile_string( + &self, + builder: &mut QueryKeyStringBuilder<'_, '_, '_>, + ) -> StringId { + let val0 = self.0.to_self_profile_string(builder); + let val1 = self.1.to_self_profile_string(builder); + + let components = &[ + StringComponent::Value("("), + StringComponent::Ref(val0), + StringComponent::Value(","), + StringComponent::Ref(val1), + StringComponent::Value(")"), + ]; + + builder.profiler.alloc_string(components) + } +} + +/// Allocate the self-profiling query strings for a single query cache. This +/// method is called from `alloc_self_profile_query_strings` which knows all +/// the queries via macro magic. +fn alloc_self_profile_query_strings_for_query_cache<'tcx, C>( + tcx: TyCtxt<'tcx>, + query_name: &'static str, + query_cache: &C, + string_cache: &mut QueryKeyStringCache, +) where + C: QueryCache, + C::Key: Debug + Clone, +{ + tcx.prof.with_profiler(|profiler| { + let event_id_builder = profiler.event_id_builder(); + + // Walk the entire query cache and allocate the appropriate + // string representations. Each cache entry is uniquely + // identified by its dep_node_index. + if profiler.query_key_recording_enabled() { + let mut query_string_builder = QueryKeyStringBuilder::new(profiler, tcx, string_cache); + + let query_name = profiler.get_or_alloc_cached_string(query_name); + + // Since building the string representation of query keys might + // need to invoke queries itself, we cannot keep the query caches + // locked while doing so. Instead we copy out the + // `(query_key, dep_node_index)` pairs and release the lock again. + let mut query_keys_and_indices = Vec::new(); + query_cache.iter(&mut |k, _, i| query_keys_and_indices.push((k.clone(), i))); + + // Now actually allocate the strings. If allocating the strings + // generates new entries in the query cache, we'll miss them but + // we don't actually care. + for (query_key, dep_node_index) in query_keys_and_indices { + // Translate the DepNodeIndex into a QueryInvocationId + let query_invocation_id = dep_node_index.into(); + + // Create the string version of the query-key + let query_key = query_key.to_self_profile_string(&mut query_string_builder); + let event_id = event_id_builder.from_label_and_arg(query_name, query_key); + + // Doing this in bulk might be a good idea: + profiler.map_query_invocation_id_to_string( + query_invocation_id, + event_id.to_string_id(), + ); + } + } else { + // In this branch we don't allocate query keys + let query_name = profiler.get_or_alloc_cached_string(query_name); + let event_id = event_id_builder.from_label(query_name).to_string_id(); + + // FIXME(eddyb) make this O(1) by using a pre-cached query name `EventId`, + // instead of passing the `DepNodeIndex` to `finish_with_query_invocation_id`, + // when recording the event in the first place. + let mut query_invocation_ids = Vec::new(); + query_cache.iter(&mut |_, _, i| { + query_invocation_ids.push(i.into()); + }); + + profiler.bulk_map_query_invocation_id_to_single_string( + query_invocation_ids.into_iter(), + event_id, + ); + } + }); +} + +/// All self-profiling events generated by the query engine use +/// virtual `StringId`s for their `event_id`. This method makes all +/// those virtual `StringId`s point to actual strings. +/// +/// If we are recording only summary data, the ids will point to +/// just the query names. If we are recording query keys too, we +/// allocate the corresponding strings here. +pub fn alloc_self_profile_query_strings(tcx: TyCtxt<'_>) { + if !tcx.prof.enabled() { + return; + } + + let mut string_cache = QueryKeyStringCache::new(); + + macro_rules! alloc_once { + (<$tcx:tt> + $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($K:ty) -> $V:ty,)* + ) => { + $({ + alloc_self_profile_query_strings_for_query_cache( + tcx, + stringify!($name), + &tcx.query_caches.$name, + &mut string_cache, + ); + })* + } + } + + rustc_query_append! { [alloc_once!][<'tcx>] } +} diff --git a/compiler/rustc_query_impl/src/values.rs b/compiler/rustc_query_impl/src/values.rs new file mode 100644 index 000000000..718a2971c --- /dev/null +++ b/compiler/rustc_query_impl/src/values.rs @@ -0,0 +1,45 @@ +use super::QueryCtxt; +use rustc_middle::ty::{self, AdtSizedConstraint, Ty}; + +pub(super) trait Value<'tcx>: Sized { + fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self; +} + +impl<'tcx, T> Value<'tcx> for T { + default fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> T { + tcx.sess.abort_if_errors(); + bug!("Value::from_cycle_error called without errors"); + } +} + +impl<'tcx> Value<'tcx> for Ty<'_> { + fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self { + // SAFETY: This is never called when `Self` is not `Ty<'tcx>`. + // FIXME: Represent the above fact in the trait system somehow. + unsafe { std::mem::transmute::, Ty<'_>>(tcx.ty_error()) } + } +} + +impl<'tcx> Value<'tcx> for ty::SymbolName<'_> { + fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self { + // SAFETY: This is never called when `Self` is not `SymbolName<'tcx>`. + // FIXME: Represent the above fact in the trait system somehow. + unsafe { + std::mem::transmute::, ty::SymbolName<'_>>(ty::SymbolName::new( + *tcx, "", + )) + } + } +} + +impl<'tcx> Value<'tcx> for AdtSizedConstraint<'_> { + fn from_cycle_error(tcx: QueryCtxt<'tcx>) -> Self { + // SAFETY: This is never called when `Self` is not `AdtSizedConstraint<'tcx>`. + // FIXME: Represent the above fact in the trait system somehow. + unsafe { + std::mem::transmute::, AdtSizedConstraint<'_>>( + AdtSizedConstraint(tcx.intern_type_list(&[tcx.ty_error()])), + ) + } + } +} -- cgit v1.2.3