diff options
Diffstat (limited to 'compiler/rustc_query_impl/src/plumbing.rs')
| -rw-r--r-- | compiler/rustc_query_impl/src/plumbing.rs | 670 |
1 files changed, 310 insertions, 360 deletions
diff --git a/compiler/rustc_query_impl/src/plumbing.rs b/compiler/rustc_query_impl/src/plumbing.rs index afbead7d1..244f0e84b 100644 --- a/compiler/rustc_query_impl/src/plumbing.rs +++ b/compiler/rustc_query_impl/src/plumbing.rs @@ -2,35 +2,45 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. -use crate::on_disk_cache::{CacheDecoder, CacheEncoder, EncodedDepNodeIndex}; -use crate::profiling_support::QueryKeyStringCache; -use crate::{on_disk_cache, Queries}; -use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; -use rustc_data_structures::sync::{AtomicU64, Lock}; -use rustc_errors::{Diagnostic, Handler}; +use crate::rustc_middle::dep_graph::DepContext; +use crate::rustc_middle::ty::TyEncoder; +use crate::QueryConfigRestored; +use rustc_data_structures::stable_hasher::{Hash64, HashStable, StableHasher}; +use rustc_data_structures::sync::Lock; +use rustc_errors::Diagnostic; +use rustc_index::Idx; use rustc_middle::dep_graph::{ self, DepKind, DepKindStruct, DepNode, DepNodeIndex, SerializedDepNodeIndex, }; +use rustc_middle::query::on_disk_cache::AbsoluteBytePos; +use rustc_middle::query::on_disk_cache::{CacheDecoder, CacheEncoder, EncodedDepNodeIndex}; use rustc_middle::query::Key; use rustc_middle::ty::tls::{self, ImplicitCtxt}; use rustc_middle::ty::{self, TyCtxt}; use rustc_query_system::dep_graph::{DepNodeParams, HasDepContext}; use rustc_query_system::ich::StableHashingContext; use rustc_query_system::query::{ - force_query, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects, QueryStackFrame, + force_query, QueryCache, QueryConfig, QueryContext, QueryJobId, QueryMap, QuerySideEffects, + QueryStackFrame, }; use rustc_query_system::{LayoutOfDepth, QueryOverflow}; use rustc_serialize::Decodable; +use rustc_serialize::Encodable; use rustc_session::Limit; use rustc_span::def_id::LOCAL_CRATE; -use std::any::Any; use std::num::NonZeroU64; use thin_vec::ThinVec; #[derive(Copy, Clone)] pub struct QueryCtxt<'tcx> { pub tcx: TyCtxt<'tcx>, - pub queries: &'tcx Queries<'tcx>, +} + +impl<'tcx> QueryCtxt<'tcx> { + #[inline] + pub fn new(tcx: TyCtxt<'tcx>) -> Self { + QueryCtxt { tcx } + } } impl<'tcx> std::ops::Deref for QueryCtxt<'tcx> { @@ -53,44 +63,56 @@ impl<'tcx> HasDepContext for QueryCtxt<'tcx> { } impl QueryContext for QueryCtxt<'_> { + #[inline] fn next_job_id(self) -> QueryJobId { QueryJobId( NonZeroU64::new( - self.queries.jobs.fetch_add(1, rustc_data_structures::sync::Ordering::Relaxed), + self.query_system.jobs.fetch_add(1, rustc_data_structures::sync::Ordering::Relaxed), ) .unwrap(), ) } + #[inline] fn current_query_job(self) -> Option<QueryJobId> { - tls::with_related_context(*self, |icx| icx.query) + tls::with_related_context(self.tcx, |icx| icx.query) } fn try_collect_active_jobs(self) -> Option<QueryMap<DepKind>> { - self.queries.try_collect_active_jobs(*self) + let mut jobs = QueryMap::default(); + + for collect in super::TRY_COLLECT_ACTIVE_JOBS.iter() { + collect(self.tcx, &mut jobs); + } + + Some(jobs) } // Interactions with on_disk_cache fn load_side_effects(self, prev_dep_node_index: SerializedDepNodeIndex) -> QuerySideEffects { - self.queries + self.query_system .on_disk_cache .as_ref() - .map(|c| c.load_side_effects(*self, prev_dep_node_index)) + .map(|c| c.load_side_effects(self.tcx, prev_dep_node_index)) .unwrap_or_default() } + #[inline(never)] + #[cold] fn store_side_effects(self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects) { - if let Some(c) = self.queries.on_disk_cache.as_ref() { + if let Some(c) = self.query_system.on_disk_cache.as_ref() { c.store_side_effects(dep_node_index, side_effects) } } + #[inline(never)] + #[cold] fn store_side_effects_for_anon_node( self, dep_node_index: DepNodeIndex, side_effects: QuerySideEffects, ) { - if let Some(c) = self.queries.on_disk_cache.as_ref() { + if let Some(c) = self.query_system.on_disk_cache.as_ref() { c.store_side_effects_for_anon_node(dep_node_index, side_effects) } } @@ -109,14 +131,14 @@ impl QueryContext for QueryCtxt<'_> { // The `TyCtxt` stored in TLS has the same global interner lifetime // as `self`, so we use `with_related_context` to relate the 'tcx lifetimes // when accessing the `ImplicitCtxt`. - tls::with_related_context(*self, move |current_icx| { + tls::with_related_context(self.tcx, move |current_icx| { if depth_limit && !self.recursion_limit().value_within_limit(current_icx.query_depth) { self.depth_limit_error(token); } // Update the `ImplicitCtxt` to point to our new query job. let new_icx = ImplicitCtxt { - tcx: *self, + tcx: self.tcx, query: Some(token), diagnostics, query_depth: current_icx.query_depth + depth_limit as usize, @@ -152,51 +174,18 @@ impl QueryContext for QueryCtxt<'_> { } } -impl<'tcx> QueryCtxt<'tcx> { - #[inline] - pub fn from_tcx(tcx: TyCtxt<'tcx>) -> Self { - let queries = tcx.queries.as_any(); - let queries = unsafe { - let queries = std::mem::transmute::<&dyn Any, &dyn Any>(queries); - let queries = queries.downcast_ref().unwrap(); - let queries = std::mem::transmute::<&Queries<'_>, &Queries<'_>>(queries); - queries - }; - QueryCtxt { tcx, queries } - } - - pub(crate) fn on_disk_cache(self) -> Option<&'tcx on_disk_cache::OnDiskCache<'tcx>> { - self.queries.on_disk_cache.as_ref() - } - - pub(super) fn encode_query_results( - self, - encoder: &mut CacheEncoder<'_, 'tcx>, - query_result_index: &mut EncodedDepNodeIndex, - ) { - for query in &self.queries.query_structs { - if let Some(encode) = query.encode_query_results { - encode(self, encoder, query_result_index); - } - } - } - - pub fn try_print_query_stack( - self, - query: Option<QueryJobId>, - handler: &Handler, - num_frames: Option<usize>, - ) -> usize { - rustc_query_system::query::print_query_stack(self, query, handler, num_frames) - } +pub(super) fn try_mark_green<'tcx>(tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool { + tcx.dep_graph.try_mark_green(QueryCtxt::new(tcx), dep_node).is_some() } -#[derive(Clone, Copy)] -pub(crate) struct QueryStruct<'tcx> { - pub try_collect_active_jobs: fn(QueryCtxt<'tcx>, &mut QueryMap<DepKind>) -> Option<()>, - pub alloc_self_profile_query_strings: fn(TyCtxt<'tcx>, &mut QueryKeyStringCache), - pub encode_query_results: - Option<fn(QueryCtxt<'tcx>, &mut CacheEncoder<'_, 'tcx>, &mut EncodedDepNodeIndex)>, +pub(super) fn encode_all_query_results<'tcx>( + tcx: TyCtxt<'tcx>, + encoder: &mut CacheEncoder<'_, 'tcx>, + query_result_index: &mut EncodedDepNodeIndex, +) { + for encode in super::ENCODE_QUERY_RESULTS.iter().copied().filter_map(|e| e) { + encode(tcx, encoder, query_result_index); + } } macro_rules! handle_cycle_error { @@ -275,14 +264,14 @@ macro_rules! hash_result { } macro_rules! call_provider { - ([][$qcx:expr, $name:ident, $key:expr]) => {{ - ($qcx.queries.local_providers.$name)($qcx.tcx, $key) + ([][$tcx:expr, $name:ident, $key:expr]) => {{ + ($tcx.query_system.fns.local_providers.$name)($tcx, $key) }}; - ([(separate_provide_extern) $($rest:tt)*][$qcx:expr, $name:ident, $key:expr]) => {{ + ([(separate_provide_extern) $($rest:tt)*][$tcx:expr, $name:ident, $key:expr]) => {{ if let Some(key) = $key.as_local_key() { - ($qcx.queries.local_providers.$name)($qcx.tcx, key) + ($tcx.query_system.fns.local_providers.$name)($tcx, key) } else { - ($qcx.queries.extern_providers.$name)($qcx.tcx, $key) + ($tcx.query_system.fns.extern_providers.$name)($tcx, $key) } }}; ([$other:tt $($modifiers:tt)*][$($args:tt)*]) => { @@ -306,7 +295,7 @@ pub(crate) fn create_query_frame< 'tcx, K: Copy + Key + for<'a> HashStable<StableHashingContext<'a>>, >( - tcx: QueryCtxt<'tcx>, + tcx: TyCtxt<'tcx>, do_describe: fn(TyCtxt<'tcx>, K) -> String, key: K, kind: DepKind, @@ -318,7 +307,7 @@ pub(crate) fn create_query_frame< // Showing visible path instead of any path is not that important in production. ty::print::with_no_visible_paths!( // Force filename-line mode to avoid invoking `type_of` query. - ty::print::with_forced_impl_filename_line!(do_describe(tcx.tcx, key)) + ty::print::with_forced_impl_filename_line!(do_describe(tcx, key)) ) ); let description = @@ -328,7 +317,7 @@ pub(crate) fn create_query_frame< // so exit to avoid infinite recursion. None } else { - Some(key.default_span(*tcx)) + Some(key.default_span(tcx)) }; let def_id = key.key_as_def_id(); let def_kind = if kind == dep_graph::DepKind::opt_def_kind { @@ -342,7 +331,7 @@ pub(crate) fn create_query_frame< let mut hasher = StableHasher::new(); std::mem::discriminant(&kind).hash_stable(&mut hcx, &mut hasher); key.hash_stable(&mut hcx, &mut hasher); - hasher.finish::<u64>() + hasher.finish::<Hash64>() }) }; let ty_adt_id = key.ty_adt_id(); @@ -350,6 +339,34 @@ pub(crate) fn create_query_frame< QueryStackFrame::new(description, span, def_id, def_kind, kind, ty_adt_id, hash) } +pub(crate) fn encode_query_results<'a, 'tcx, Q>( + query: Q::Config, + qcx: QueryCtxt<'tcx>, + encoder: &mut CacheEncoder<'a, 'tcx>, + query_result_index: &mut EncodedDepNodeIndex, +) where + Q: super::QueryConfigRestored<'tcx>, + Q::RestoredValue: Encodable<CacheEncoder<'a, 'tcx>>, +{ + let _timer = + qcx.profiler().verbose_generic_activity_with_arg("encode_query_results_for", query.name()); + + assert!(query.query_state(qcx).all_inactive()); + let cache = query.query_cache(qcx); + cache.iter(&mut |key, value, dep_node| { + if query.cache_on_disk(qcx.tcx, &key) { + let dep_node = SerializedDepNodeIndex::new(dep_node.index()); + + // Record position of the cache entry. + query_result_index.push((dep_node, AbsoluteBytePos::new(encoder.position()))); + + // Encode the type check tables with the `SerializedDepNodeIndex` + // as tag. + encoder.encode_tagged(dep_node, &Q::restore(*value)); + } + }); +} + fn try_load_from_on_disk_cache<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) where Q: QueryConfig<QueryCtxt<'tcx>>, @@ -364,8 +381,8 @@ where } } -pub(crate) fn loadable_from_disk<'tcx>(tcx: QueryCtxt<'tcx>, id: SerializedDepNodeIndex) -> bool { - if let Some(cache) = tcx.on_disk_cache().as_ref() { +pub(crate) fn loadable_from_disk<'tcx>(tcx: TyCtxt<'tcx>, id: SerializedDepNodeIndex) -> bool { + if let Some(cache) = tcx.query_system.on_disk_cache.as_ref() { cache.loadable_from_disk(id) } else { false @@ -373,13 +390,27 @@ pub(crate) fn loadable_from_disk<'tcx>(tcx: QueryCtxt<'tcx>, id: SerializedDepNo } pub(crate) fn try_load_from_disk<'tcx, V>( - tcx: QueryCtxt<'tcx>, - id: SerializedDepNodeIndex, + tcx: TyCtxt<'tcx>, + prev_index: SerializedDepNodeIndex, + index: DepNodeIndex, ) -> Option<V> where V: for<'a> Decodable<CacheDecoder<'a, 'tcx>>, { - tcx.on_disk_cache().as_ref()?.try_load_query_result(*tcx, id) + let on_disk_cache = tcx.query_system.on_disk_cache.as_ref()?; + + let prof_timer = tcx.prof.incr_cache_loading(); + + // The call to `with_query_deserialization` enforces that no new `DepNodes` + // are created during deserialization. See the docs of that method for more + // details. + let value = tcx + .dep_graph + .with_query_deserialization(|| on_disk_cache.try_load_query_result(tcx, prev_index)); + + prof_timer.finish_with_query_invocation_id(index.into()); + + value } fn force_from_dep_node<'tcx, Q>(query: Q, tcx: TyCtxt<'tcx>, dep_node: DepNode) -> bool @@ -407,8 +438,7 @@ where if let Some(key) = Q::Key::recover(tcx, &dep_node) { #[cfg(debug_assertions)] let _guard = tracing::span!(tracing::Level::TRACE, stringify!($name), ?key).entered(); - let tcx = QueryCtxt::from_tcx(tcx); - force_query(query, tcx, key, dep_node); + force_query(query, QueryCtxt::new(tcx), key, dep_node); true } else { false @@ -417,10 +447,9 @@ where pub(crate) fn query_callback<'tcx, Q>(is_anon: bool, is_eval_always: bool) -> DepKindStruct<'tcx> where - Q: QueryConfig<QueryCtxt<'tcx>> + Default, - Q::Key: DepNodeParams<TyCtxt<'tcx>>, + Q: QueryConfigRestored<'tcx>, { - let fingerprint_style = Q::Key::fingerprint_style(); + let fingerprint_style = <Q::Config as QueryConfig<QueryCtxt<'tcx>>>::Key::fingerprint_style(); if is_anon || !fingerprint_style.reconstructible() { return DepKindStruct { @@ -436,13 +465,25 @@ where is_anon, is_eval_always, fingerprint_style, - force_from_dep_node: Some(|tcx, dep_node| force_from_dep_node(Q::default(), tcx, dep_node)), + force_from_dep_node: Some(|tcx, dep_node| { + force_from_dep_node(Q::config(tcx), tcx, dep_node) + }), try_load_from_on_disk_cache: Some(|tcx, dep_node| { - try_load_from_on_disk_cache(Q::default(), tcx, dep_node) + try_load_from_on_disk_cache(Q::config(tcx), tcx, dep_node) }), } } +macro_rules! item_if_cached { + ([] $tokens:tt) => {}; + ([(cache) $($rest:tt)*] { $($tokens:tt)* }) => { + $($tokens)* + }; + ([$other:tt $($modifiers:tt)*] $tokens:tt) => { + item_if_cached! { [$($modifiers)*] $tokens } + }; +} + macro_rules! expand_if_cached { ([], $tokens:expr) => {{ None @@ -455,168 +496,226 @@ macro_rules! expand_if_cached { }; } +/// Don't show the backtrace for query system by default +/// use `RUST_BACKTRACE=full` to show all the backtraces +#[inline(never)] +pub fn __rust_begin_short_backtrace<F, T>(f: F) -> T +where + F: FnOnce() -> T, +{ + let result = f(); + std::hint::black_box(()); + result +} + // NOTE: `$V` isn't used here, but we still need to match on it so it can be passed to other macros // invoked by `rustc_query_append`. macro_rules! define_queries { ( $($(#[$attr:meta])* [$($modifiers:tt)*] fn $name:ident($($K:tt)*) -> $V:ty,)*) => { - define_queries_struct! { - input: ($(([$($modifiers)*] [$($attr)*] [$name]))*) - } - #[allow(nonstandard_style)] - mod queries { + pub(crate) mod query_impl { $(pub mod $name { + use super::super::*; use std::marker::PhantomData; - $( - #[derive(Copy, Clone, Default)] - pub struct $name<'tcx> { - data: PhantomData<&'tcx ()> + pub mod get_query_incr { + use super::*; + + // Adding `__rust_end_short_backtrace` marker to backtraces so that we emit the frames + // when `RUST_BACKTRACE=1`, add a new mod with `$name` here is to allow duplicate naming + #[inline(never)] + pub fn __rust_end_short_backtrace<'tcx>( + tcx: TyCtxt<'tcx>, + span: Span, + key: queries::$name::Key<'tcx>, + mode: QueryMode, + ) -> Option<Erase<queries::$name::Value<'tcx>>> { + get_query_incr( + QueryType::config(tcx), + QueryCtxt::new(tcx), + span, + key, + mode + ) } - )* - } - - $(impl<'tcx> QueryConfig<QueryCtxt<'tcx>> for queries::$name<'tcx> { - type Key = query_keys::$name<'tcx>; - type Value = Erase<query_values::$name<'tcx>>; - - #[inline(always)] - fn name(self) -> &'static str { - stringify!($name) - } - - #[inline] - fn format_value(self) -> fn(&Self::Value) -> String { - |value| format!("{:?}", restore::<query_values::$name<'tcx>>(*value)) - } - - #[inline] - fn cache_on_disk(self, tcx: TyCtxt<'tcx>, key: &Self::Key) -> bool { - ::rustc_middle::query::cached::$name(tcx, key) - } - - type Cache = query_storage::$name<'tcx>; - - #[inline(always)] - fn query_state<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a QueryState<Self::Key, crate::dep_graph::DepKind> - where QueryCtxt<'tcx>: 'a - { - &tcx.queries.$name } - #[inline(always)] - fn query_cache<'a>(self, tcx: QueryCtxt<'tcx>) -> &'a Self::Cache - where 'tcx:'a - { - &tcx.query_system.caches.$name + pub mod get_query_non_incr { + use super::*; + + #[inline(never)] + pub fn __rust_end_short_backtrace<'tcx>( + tcx: TyCtxt<'tcx>, + span: Span, + key: queries::$name::Key<'tcx>, + __mode: QueryMode, + ) -> Option<Erase<queries::$name::Value<'tcx>>> { + Some(get_query_non_incr( + QueryType::config(tcx), + QueryCtxt::new(tcx), + span, + key, + )) + } } - fn execute_query(self, tcx: TyCtxt<'tcx>, key: Self::Key) -> Self::Value { - erase(tcx.$name(key)) + pub fn dynamic_query<'tcx>() -> DynamicQuery<'tcx, queries::$name::Storage<'tcx>> { + DynamicQuery { + name: stringify!($name), + eval_always: is_eval_always!([$($modifiers)*]), + dep_kind: dep_graph::DepKind::$name, + handle_cycle_error: handle_cycle_error!([$($modifiers)*]), + query_state: offset_of!(QueryStates<'tcx> => $name), + query_cache: offset_of!(QueryCaches<'tcx> => $name), + cache_on_disk: |tcx, key| ::rustc_middle::query::cached::$name(tcx, key), + execute_query: |tcx, key| erase(tcx.$name(key)), + compute: |tcx, key| { + __rust_begin_short_backtrace(|| + queries::$name::provided_to_erased( + tcx, + call_provider!([$($modifiers)*][tcx, $name, key]) + ) + ) + }, + can_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] true false), + try_load_from_disk: should_ever_cache_on_disk!([$($modifiers)*] { + |tcx, key, prev_index, index| { + if ::rustc_middle::query::cached::$name(tcx, key) { + let value = $crate::plumbing::try_load_from_disk::< + queries::$name::ProvidedValue<'tcx> + >( + tcx, + prev_index, + index, + ); + value.map(|value| queries::$name::provided_to_erased(tcx, value)) + } else { + None + } + } + } { + |_tcx, _key, _prev_index, _index| None + }), + value_from_cycle_error: |tcx, cycle| { + let result: queries::$name::Value<'tcx> = Value::from_cycle_error(tcx, cycle); + erase(result) + }, + loadable_from_disk: |_tcx, _key, _index| { + should_ever_cache_on_disk!([$($modifiers)*] { + ::rustc_middle::query::cached::$name(_tcx, _key) && + $crate::plumbing::loadable_from_disk(_tcx, _index) + } { + false + }) + }, + hash_result: hash_result!([$($modifiers)*][queries::$name::Value<'tcx>]), + format_value: |value| format!("{:?}", restore::<queries::$name::Value<'tcx>>(*value)), + } } - #[inline] - #[allow(unused_variables)] - fn compute(self, qcx: QueryCtxt<'tcx>, key: Self::Key) -> Self::Value { - query_provided_to_value::$name( - qcx.tcx, - call_provider!([$($modifiers)*][qcx, $name, key]) - ) + #[derive(Copy, Clone, Default)] + pub struct QueryType<'tcx> { + data: PhantomData<&'tcx ()> } - #[inline] - fn try_load_from_disk( - self, - _qcx: QueryCtxt<'tcx>, - _key: &Self::Key - ) -> rustc_query_system::query::TryLoadFromDisk<QueryCtxt<'tcx>, Self::Value> { - should_ever_cache_on_disk!([$($modifiers)*] { - if ::rustc_middle::query::cached::$name(_qcx.tcx, _key) { - Some(|qcx: QueryCtxt<'tcx>, dep_node| { - let value = $crate::plumbing::try_load_from_disk::<query_provided::$name<'tcx>>( - qcx, - dep_node - ); - value.map(|value| query_provided_to_value::$name(qcx.tcx, value)) - }) - } else { - None + impl<'tcx> QueryConfigRestored<'tcx> for QueryType<'tcx> { + type RestoredValue = queries::$name::Value<'tcx>; + type Config = DynamicConfig< + 'tcx, + queries::$name::Storage<'tcx>, + { is_anon!([$($modifiers)*]) }, + { depth_limit!([$($modifiers)*]) }, + { feedable!([$($modifiers)*]) }, + >; + + #[inline(always)] + fn config(tcx: TyCtxt<'tcx>) -> Self::Config { + DynamicConfig { + dynamic: &tcx.query_system.dynamic_queries.$name, } - } { - None - }) - } - - #[inline] - fn loadable_from_disk( - self, - _qcx: QueryCtxt<'tcx>, - _key: &Self::Key, - _index: SerializedDepNodeIndex, - ) -> bool { - should_ever_cache_on_disk!([$($modifiers)*] { - self.cache_on_disk(_qcx.tcx, _key) && - $crate::plumbing::loadable_from_disk(_qcx, _index) - } { - false - }) - } + } - #[inline] - fn value_from_cycle_error( - self, - tcx: TyCtxt<'tcx>, - cycle: &[QueryInfo<DepKind>], - ) -> Self::Value { - let result: query_values::$name<'tcx> = Value::from_cycle_error(tcx, cycle); - erase(result) + #[inline(always)] + fn restore(value: <Self::Config as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue { + restore::<queries::$name::Value<'tcx>>(value) + } } - #[inline(always)] - fn anon(self) -> bool { - is_anon!([$($modifiers)*]) + pub fn try_collect_active_jobs<'tcx>(tcx: TyCtxt<'tcx>, qmap: &mut QueryMap<DepKind>) { + let make_query = |tcx, key| { + let kind = rustc_middle::dep_graph::DepKind::$name; + let name = stringify!($name); + $crate::plumbing::create_query_frame(tcx, rustc_middle::query::descs::$name, key, kind, name) + }; + tcx.query_system.states.$name.try_collect_active_jobs( + tcx, + make_query, + qmap, + ).unwrap(); } - #[inline(always)] - fn eval_always(self) -> bool { - is_eval_always!([$($modifiers)*]) + pub fn alloc_self_profile_query_strings<'tcx>(tcx: TyCtxt<'tcx>, string_cache: &mut QueryKeyStringCache) { + $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache( + tcx, + stringify!($name), + &tcx.query_system.caches.$name, + string_cache, + ) } - #[inline(always)] - fn depth_limit(self) -> bool { - depth_limit!([$($modifiers)*]) - } + item_if_cached! { [$($modifiers)*] { + pub fn encode_query_results<'tcx>( + tcx: TyCtxt<'tcx>, + encoder: &mut CacheEncoder<'_, 'tcx>, + query_result_index: &mut EncodedDepNodeIndex + ) { + $crate::plumbing::encode_query_results::<query_impl::$name::QueryType<'tcx>>( + query_impl::$name::QueryType::config(tcx), + QueryCtxt::new(tcx), + encoder, + query_result_index, + ) + } + }} + })*} - #[inline(always)] - fn feedable(self) -> bool { - feedable!([$($modifiers)*]) + pub(crate) fn engine(incremental: bool) -> QueryEngine { + if incremental { + QueryEngine { + $($name: query_impl::$name::get_query_incr::__rust_end_short_backtrace,)* + } + } else { + QueryEngine { + $($name: query_impl::$name::get_query_non_incr::__rust_end_short_backtrace,)* + } } + } - #[inline(always)] - fn dep_kind(self) -> rustc_middle::dep_graph::DepKind { - dep_graph::DepKind::$name + pub fn dynamic_queries<'tcx>() -> DynamicQueries<'tcx> { + DynamicQueries { + $( + $name: query_impl::$name::dynamic_query(), + )* } + } - #[inline(always)] - fn handle_cycle_error(self) -> rustc_query_system::HandleCycleError { - handle_cycle_error!([$($modifiers)*]) - } + // These arrays are used for iteration and can't be indexed by `DepKind`. - #[inline(always)] - fn hash_result(self) -> rustc_query_system::query::HashResult<Self::Value> { - hash_result!([$($modifiers)*][query_values::$name<'tcx>]) - } - })* + const TRY_COLLECT_ACTIVE_JOBS: &[for<'tcx> fn(TyCtxt<'tcx>, &mut QueryMap<DepKind>)] = + &[$(query_impl::$name::try_collect_active_jobs),*]; - $(impl<'tcx> QueryConfigRestored<'tcx> for queries::$name<'tcx> { - type RestoredValue = query_values::$name<'tcx>; + const ALLOC_SELF_PROFILE_QUERY_STRINGS: &[ + for<'tcx> fn(TyCtxt<'tcx>, &mut QueryKeyStringCache) + ] = &[$(query_impl::$name::alloc_self_profile_query_strings),*]; - #[inline(always)] - fn restore(value: <Self as QueryConfig<QueryCtxt<'tcx>>>::Value) -> Self::RestoredValue { - restore::<query_values::$name<'tcx>>(value) - } - })* + const ENCODE_QUERY_RESULTS: &[ + Option<for<'tcx> fn( + TyCtxt<'tcx>, + &mut CacheEncoder<'_, 'tcx>, + &mut EncodedDepNodeIndex) + > + ] = &[$(expand_if_cached!([$($modifiers)*], query_impl::$name::encode_query_results)),*]; #[allow(nonstandard_style)] mod query_callbacks { @@ -676,164 +775,15 @@ macro_rules! define_queries { } $(pub(crate) fn $name<'tcx>()-> DepKindStruct<'tcx> { - $crate::plumbing::query_callback::<queries::$name<'tcx>>( + $crate::plumbing::query_callback::<query_impl::$name::QueryType<'tcx>>( is_anon!([$($modifiers)*]), is_eval_always!([$($modifiers)*]), ) })* } - mod query_structs { - use rustc_middle::ty::TyCtxt; - use $crate::plumbing::{QueryStruct, QueryCtxt}; - use $crate::profiling_support::QueryKeyStringCache; - use rustc_query_system::query::QueryMap; - use rustc_middle::dep_graph::DepKind; - - pub(super) const fn dummy_query_struct<'tcx>() -> QueryStruct<'tcx> { - fn noop_try_collect_active_jobs(_: QueryCtxt<'_>, _: &mut QueryMap<DepKind>) -> Option<()> { - None - } - fn noop_alloc_self_profile_query_strings(_: TyCtxt<'_>, _: &mut QueryKeyStringCache) {} - - QueryStruct { - try_collect_active_jobs: noop_try_collect_active_jobs, - alloc_self_profile_query_strings: noop_alloc_self_profile_query_strings, - encode_query_results: None, - } - } - - pub(super) use dummy_query_struct as Null; - pub(super) use dummy_query_struct as Red; - pub(super) use dummy_query_struct as TraitSelect; - pub(super) use dummy_query_struct as CompileCodegenUnit; - pub(super) use dummy_query_struct as CompileMonoItem; - - $( - pub(super) const fn $name<'tcx>() -> QueryStruct<'tcx> { QueryStruct { - try_collect_active_jobs: |tcx, qmap| { - let make_query = |tcx, key| { - let kind = rustc_middle::dep_graph::DepKind::$name; - let name = stringify!($name); - $crate::plumbing::create_query_frame(tcx, rustc_middle::query::descs::$name, key, kind, name) - }; - tcx.queries.$name.try_collect_active_jobs( - tcx, - make_query, - qmap, - ) - }, - alloc_self_profile_query_strings: |tcx, string_cache| { - $crate::profiling_support::alloc_self_profile_query_strings_for_query_cache( - tcx, - stringify!($name), - &tcx.query_system.caches.$name, - string_cache, - ) - }, - encode_query_results: expand_if_cached!([$($modifiers)*], |qcx, encoder, query_result_index| - $crate::on_disk_cache::encode_query_results::<super::queries::$name<'tcx>>( - super::queries::$name::default(), - qcx, - encoder, - query_result_index, - ) - ), - }})* - } - pub fn query_callbacks<'tcx>(arena: &'tcx Arena<'tcx>) -> &'tcx [DepKindStruct<'tcx>] { arena.alloc_from_iter(make_dep_kind_array!(query_callbacks)) } } } - -use crate::{ExternProviders, OnDiskCache, Providers}; - -impl<'tcx> Queries<'tcx> { - pub fn new( - local_providers: Providers, - extern_providers: ExternProviders, - on_disk_cache: Option<OnDiskCache<'tcx>>, - ) -> Self { - use crate::query_structs; - Queries { - local_providers: Box::new(local_providers), - extern_providers: Box::new(extern_providers), - query_structs: make_dep_kind_array!(query_structs).to_vec(), - on_disk_cache, - jobs: AtomicU64::new(1), - ..Queries::default() - } - } -} - -macro_rules! define_queries_struct { - ( - input: ($(([$($modifiers:tt)*] [$($attr:tt)*] [$name:ident]))*)) => { - #[derive(Default)] - pub struct Queries<'tcx> { - local_providers: Box<Providers>, - extern_providers: Box<ExternProviders>, - query_structs: Vec<$crate::plumbing::QueryStruct<'tcx>>, - pub on_disk_cache: Option<OnDiskCache<'tcx>>, - jobs: AtomicU64, - - $( - $(#[$attr])* - $name: QueryState< - <queries::$name<'tcx> as QueryConfig<QueryCtxt<'tcx>>>::Key, - rustc_middle::dep_graph::DepKind, - >, - )* - } - - impl<'tcx> Queries<'tcx> { - pub(crate) fn try_collect_active_jobs( - &'tcx self, - tcx: TyCtxt<'tcx>, - ) -> Option<QueryMap<rustc_middle::dep_graph::DepKind>> { - let tcx = QueryCtxt { tcx, queries: self }; - let mut jobs = QueryMap::default(); - - for query in &self.query_structs { - (query.try_collect_active_jobs)(tcx, &mut jobs); - } - - Some(jobs) - } - } - - impl<'tcx> QueryEngine<'tcx> for Queries<'tcx> { - fn as_any(&'tcx self) -> &'tcx dyn std::any::Any { - let this = unsafe { std::mem::transmute::<&Queries<'_>, &Queries<'_>>(self) }; - this as _ - } - - fn try_mark_green(&'tcx self, tcx: TyCtxt<'tcx>, dep_node: &dep_graph::DepNode) -> bool { - let qcx = QueryCtxt { tcx, queries: self }; - tcx.dep_graph.try_mark_green(qcx, dep_node).is_some() - } - - $($(#[$attr])* - #[inline(always)] - #[tracing::instrument(level = "trace", skip(self, tcx))] - fn $name( - &'tcx self, - tcx: TyCtxt<'tcx>, - span: Span, - key: query_keys::$name<'tcx>, - mode: QueryMode, - ) -> Option<Erase<query_values::$name<'tcx>>> { - let qcx = QueryCtxt { tcx, queries: self }; - get_query( - queries::$name::default(), - qcx, - span, - key, - mode - ) - })* - } - }; -} |