From 218caa410aa38c29984be31a5229b9fa717560ee Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 17 Apr 2024 14:19:13 +0200 Subject: Merging upstream version 1.68.2+dfsg1. Signed-off-by: Daniel Baumann --- compiler/rustc_query_system/src/query/plumbing.rs | 200 ++++++++++------------ 1 file changed, 94 insertions(+), 106 deletions(-) (limited to 'compiler/rustc_query_system/src/query/plumbing.rs') diff --git a/compiler/rustc_query_system/src/query/plumbing.rs b/compiler/rustc_query_system/src/query/plumbing.rs index 848fa67e3..b3b939eae 100644 --- a/compiler/rustc_query_system/src/query/plumbing.rs +++ b/compiler/rustc_query_system/src/query/plumbing.rs @@ -2,10 +2,9 @@ //! generate the actual methods on tcx which find and execute the provider, //! manage the caches, and so forth. -use crate::dep_graph::{DepContext, DepNode, DepNodeIndex, DepNodeParams}; +use crate::dep_graph::{DepContext, DepKind, DepNode, DepNodeIndex}; use crate::ich::StableHashingContext; use crate::query::caches::QueryCache; -use crate::query::config::QueryVTable; use crate::query::job::{report_cycle, QueryInfo, QueryJob, QueryJobId, QueryJobInfo}; use crate::query::{QueryContext, QueryMap, QuerySideEffects, QueryStackFrame}; use crate::values::Value; @@ -31,26 +30,27 @@ use thin_vec::ThinVec; use super::QueryConfig; -pub struct QueryState { +pub struct QueryState { #[cfg(parallel_compiler)] - active: Sharded>, + active: Sharded>>, #[cfg(not(parallel_compiler))] - active: Lock>, + active: Lock>>, } /// Indicates the state of a query for a given key in a query map. -enum QueryResult { +enum QueryResult { /// An already executing query. The query job can be used to await for its completion. - Started(QueryJob), + Started(QueryJob), /// The query panicked. Queries trying to wait on this will raise a fatal error which will /// silently panic. Poisoned, } -impl QueryState +impl QueryState where K: Eq + Hash + Clone + Debug, + D: DepKind, { pub fn all_inactive(&self) -> bool { #[cfg(parallel_compiler)] @@ -67,8 +67,8 @@ where pub fn try_collect_active_jobs( &self, qcx: Qcx, - make_query: fn(Qcx, K) -> QueryStackFrame, - jobs: &mut QueryMap, + make_query: fn(Qcx, K) -> QueryStackFrame, + jobs: &mut QueryMap, ) -> Option<()> { #[cfg(parallel_compiler)] { @@ -102,34 +102,34 @@ where } } -impl Default for QueryState { - fn default() -> QueryState { +impl Default for QueryState { + fn default() -> QueryState { QueryState { active: Default::default() } } } /// A type representing the responsibility to execute the job in the `job` field. /// This will poison the relevant query if dropped. -struct JobOwner<'tcx, K> +struct JobOwner<'tcx, K, D: DepKind> where K: Eq + Hash + Clone, { - state: &'tcx QueryState, + state: &'tcx QueryState, key: K, id: QueryJobId, } #[cold] #[inline(never)] -fn mk_cycle( +fn mk_cycle( qcx: Qcx, - cycle_error: CycleError, + cycle_error: CycleError, handler: HandleCycleError, cache: &dyn crate::query::QueryStorage, ) -> R where - Qcx: QueryContext, - V: std::fmt::Debug + Value, + Qcx: QueryContext + crate::query::HasDepContext, + V: std::fmt::Debug + Value, R: Clone, { let error = report_cycle(qcx.dep_context().sess(), &cycle_error); @@ -139,13 +139,13 @@ where fn handle_cycle_error( tcx: Tcx, - cycle_error: &CycleError, + cycle_error: &CycleError, mut error: DiagnosticBuilder<'_, ErrorGuaranteed>, handler: HandleCycleError, ) -> V where Tcx: DepContext, - V: Value, + V: Value, { use HandleCycleError::*; match handler { @@ -165,7 +165,7 @@ where } } -impl<'tcx, K> JobOwner<'tcx, K> +impl<'tcx, K, D: DepKind> JobOwner<'tcx, K, D> where K: Eq + Hash + Clone, { @@ -180,12 +180,12 @@ where #[inline(always)] fn try_start<'b, Qcx>( qcx: &'b Qcx, - state: &'b QueryState, + state: &'b QueryState, span: Span, key: K, - ) -> TryGetJob<'b, K> + ) -> TryGetJob<'b, K, D> where - Qcx: QueryContext, + Qcx: QueryContext + crate::query::HasDepContext, { #[cfg(parallel_compiler)] let mut state_lock = state.active.get_shard_by_value(&key).lock(); @@ -280,9 +280,10 @@ where } } -impl<'tcx, K> Drop for JobOwner<'tcx, K> +impl<'tcx, K, D> Drop for JobOwner<'tcx, K, D> where K: Eq + Hash + Clone, + D: DepKind, { #[inline(never)] #[cold] @@ -308,19 +309,20 @@ where } #[derive(Clone)] -pub(crate) struct CycleError { +pub(crate) struct CycleError { /// The query and related span that uses the cycle. - pub usage: Option<(Span, QueryStackFrame)>, - pub cycle: Vec, + pub usage: Option<(Span, QueryStackFrame)>, + pub cycle: Vec>, } /// The result of `try_start`. -enum TryGetJob<'tcx, K> +enum TryGetJob<'tcx, K, D> where K: Eq + Hash + Clone, + D: DepKind, { /// The query is not yet started. Contains a guard to the cache eventually used to start it. - NotYetStarted(JobOwner<'tcx, K>), + NotYetStarted(JobOwner<'tcx, K, D>), /// The query was already completed. /// Returns the result of the query and its dep-node index @@ -329,7 +331,7 @@ where JobCompleted(TimingGuard<'tcx>), /// Trying to execute the query resulted in a cycle. - Cycle(CycleError), + Cycle(CycleError), } /// Checks if the query is already computed and in the cache. @@ -337,9 +339,9 @@ where /// which will be used if the query is not in the cache and we need /// to compute it. #[inline] -pub fn try_get_cached<'a, Tcx, C, R, OnHit>( +pub fn try_get_cached( tcx: Tcx, - cache: &'a C, + cache: &C, key: &C::Key, // `on_hit` can be called while holding a lock to the query cache on_hit: OnHit, @@ -358,36 +360,34 @@ where }) } -fn try_execute_query( +fn try_execute_query( qcx: Qcx, - state: &QueryState, - cache: &C, + state: &QueryState, + cache: &Q::Cache, span: Span, - key: C::Key, + key: Q::Key, dep_node: Option>, - query: &QueryVTable, -) -> (C::Stored, Option) +) -> (Q::Stored, Option) where - C: QueryCache, - C::Key: Clone + DepNodeParams, - C::Value: Value, - C::Stored: Debug + std::borrow::Borrow, + Q: QueryConfig, Qcx: QueryContext, { - match JobOwner::<'_, C::Key>::try_start(&qcx, state, span, key.clone()) { + match JobOwner::<'_, Q::Key, Qcx::DepKind>::try_start(&qcx, state, span, key.clone()) { TryGetJob::NotYetStarted(job) => { - let (result, dep_node_index) = execute_job(qcx, key.clone(), dep_node, query, job.id); - if query.feedable { + let (result, dep_node_index) = + execute_job::(qcx, key.clone(), dep_node, job.id); + if Q::FEEDABLE { // We may have put a value inside the cache from inside the execution. // Verify that it has the same hash as what we have now, to ensure consistency. let _ = cache.lookup(&key, |cached_result, _| { - let hasher = query.hash_result.expect("feedable forbids no_hash"); + let hasher = Q::HASH_RESULT.expect("feedable forbids no_hash"); + let old_hash = qcx.dep_context().with_stable_hashing_context(|mut hcx| hasher(&mut hcx, cached_result.borrow())); let new_hash = qcx.dep_context().with_stable_hashing_context(|mut hcx| hasher(&mut hcx, &result)); debug_assert_eq!( old_hash, new_hash, "Computed query value for {:?}({:?}) is inconsistent with fed value,\ncomputed={:#?}\nfed={:#?}", - query.dep_kind, key, result, cached_result, + Q::DEP_KIND, key, result, cached_result, ); }); } @@ -395,7 +395,7 @@ where (result, Some(dep_node_index)) } TryGetJob::Cycle(error) => { - let result = mk_cycle(qcx, error, query.handle_cycle_error, cache); + let result = mk_cycle(qcx, error, Q::HANDLE_CYCLE_ERROR, cache); (result, None) } #[cfg(parallel_compiler)] @@ -414,16 +414,14 @@ where } } -fn execute_job( +fn execute_job( qcx: Qcx, - key: K, + key: Q::Key, mut dep_node_opt: Option>, - query: &QueryVTable, job_id: QueryJobId, -) -> (V, DepNodeIndex) +) -> (Q::Value, DepNodeIndex) where - K: Clone + DepNodeParams, - V: Debug, + Q: QueryConfig, Qcx: QueryContext, { let dep_graph = qcx.dep_context().dep_graph(); @@ -431,23 +429,23 @@ where // Fast path for when incr. comp. is off. if !dep_graph.is_fully_enabled() { let prof_timer = qcx.dep_context().profiler().query_provider(); - let result = qcx.start_query(job_id, query.depth_limit, None, || { - query.compute(*qcx.dep_context(), key) + let result = qcx.start_query(job_id, Q::DEPTH_LIMIT, None, || { + Q::compute(qcx, &key)(*qcx.dep_context(), key) }); let dep_node_index = dep_graph.next_virtual_depnode_index(); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); return (result, dep_node_index); } - if !query.anon && !query.eval_always { + if !Q::ANON && !Q::EVAL_ALWAYS { // `to_dep_node` is expensive for some `DepKind`s. let dep_node = - dep_node_opt.get_or_insert_with(|| query.to_dep_node(*qcx.dep_context(), &key)); + dep_node_opt.get_or_insert_with(|| Q::construct_dep_node(*qcx.dep_context(), &key)); // The diagnostics for this query will be promoted to the current session during // `try_mark_green()`, so we can ignore them here. if let Some(ret) = qcx.start_query(job_id, false, None, || { - try_load_from_disk_and_cache_in_memory(qcx, &key, &dep_node, query) + try_load_from_disk_and_cache_in_memory::(qcx, &key, &dep_node) }) { return ret; } @@ -457,18 +455,19 @@ where let diagnostics = Lock::new(ThinVec::new()); let (result, dep_node_index) = - qcx.start_query(job_id, query.depth_limit, Some(&diagnostics), || { - if query.anon { - return dep_graph.with_anon_task(*qcx.dep_context(), query.dep_kind, || { - query.compute(*qcx.dep_context(), key) + qcx.start_query(job_id, Q::DEPTH_LIMIT, Some(&diagnostics), || { + if Q::ANON { + return dep_graph.with_anon_task(*qcx.dep_context(), Q::DEP_KIND, || { + Q::compute(qcx, &key)(*qcx.dep_context(), key) }); } // `to_dep_node` is expensive for some `DepKind`s. let dep_node = - dep_node_opt.unwrap_or_else(|| query.to_dep_node(*qcx.dep_context(), &key)); + dep_node_opt.unwrap_or_else(|| Q::construct_dep_node(*qcx.dep_context(), &key)); - dep_graph.with_task(dep_node, *qcx.dep_context(), key, query.compute, query.hash_result) + let task = Q::compute(qcx, &key); + dep_graph.with_task(dep_node, *qcx.dep_context(), key, task, Q::HASH_RESULT) }); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -477,7 +476,7 @@ where let side_effects = QuerySideEffects { diagnostics }; if std::intrinsics::unlikely(!side_effects.is_empty()) { - if query.anon { + if Q::ANON { qcx.store_side_effects_for_anon_node(dep_node_index, side_effects); } else { qcx.store_side_effects(dep_node_index, side_effects); @@ -487,16 +486,14 @@ where (result, dep_node_index) } -fn try_load_from_disk_and_cache_in_memory( +fn try_load_from_disk_and_cache_in_memory( qcx: Qcx, - key: &K, + key: &Q::Key, dep_node: &DepNode, - query: &QueryVTable, -) -> Option<(V, DepNodeIndex)> +) -> Option<(Q::Value, DepNodeIndex)> where - K: Clone, + Q: QueryConfig, Qcx: QueryContext, - V: Debug, { // Note this function can be called concurrently from the same query // We must ensure that this is handled correctly. @@ -508,7 +505,7 @@ where // First we try to load the result from the on-disk cache. // Some things are never cached on disk. - if let Some(try_load_from_disk) = query.try_load_from_disk { + if let Some(try_load_from_disk) = Q::try_load_from_disk(qcx, &key) { let prof_timer = qcx.dep_context().profiler().incr_cache_loading(); // The call to `with_query_deserialization` enforces that no new `DepNodes` @@ -542,7 +539,7 @@ where if std::intrinsics::unlikely( try_verify || qcx.dep_context().sess().opts.unstable_opts.incremental_verify_ich, ) { - incremental_verify_ich(*qcx.dep_context(), &result, dep_node, query.hash_result); + incremental_verify_ich(*qcx.dep_context(), &result, dep_node, Q::HASH_RESULT); } return Some((result, dep_node_index)); @@ -552,8 +549,7 @@ where // can be forced from `DepNode`. debug_assert!( !qcx.dep_context().fingerprint_style(dep_node.kind).reconstructible(), - "missing on-disk cache entry for {:?}", - dep_node + "missing on-disk cache entry for {dep_node:?}" ); } @@ -562,7 +558,7 @@ where let prof_timer = qcx.dep_context().profiler().query_provider(); // The dep-graph for this computation is already in-place. - let result = dep_graph.with_ignore(|| query.compute(*qcx.dep_context(), key.clone())); + let result = dep_graph.with_ignore(|| Q::compute(qcx, key)(*qcx.dep_context(), key.clone())); prof_timer.finish_with_query_invocation_id(dep_node_index.into()); @@ -575,7 +571,7 @@ where // // See issue #82920 for an example of a miscompilation that would get turned into // an ICE by this check - incremental_verify_ich(*qcx.dep_context(), &result, dep_node, query.hash_result); + incremental_verify_ich(*qcx.dep_context(), &result, dep_node, Q::HASH_RESULT); Some((result, dep_node_index)) } @@ -592,8 +588,7 @@ where { assert!( tcx.dep_graph().is_green(dep_node), - "fingerprint for green query instance not loaded from cache: {:?}", - dep_node, + "fingerprint for green query instance not loaded from cache: {dep_node:?}", ); let new_hash = hash_result.map_or(Fingerprint::ZERO, |f| { @@ -672,16 +667,16 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: sess.emit_err(crate::error::Reentrant); } else { let run_cmd = if let Some(crate_name) = &sess.opts.crate_name { - format!("`cargo clean -p {}` or `cargo clean`", crate_name) + format!("`cargo clean -p {crate_name}` or `cargo clean`") } else { "`cargo clean`".to_string() }; sess.emit_err(crate::error::IncrementCompilation { run_cmd, - dep_node: format!("{:?}", dep_node), + dep_node: format!("{dep_node:?}"), }); - panic!("Found unstable fingerprints for {:?}: {:?}", dep_node, result); + panic!("Found unstable fingerprints for {dep_node:?}: {result:?}"); } INSIDE_VERIFY_PANIC.with(|in_panic| in_panic.set(old_in_panic)); @@ -696,23 +691,19 @@ fn incremental_verify_ich_failed(sess: &Session, dep_node: DebugArg<'_>, result: /// /// Note: The optimization is only available during incr. comp. #[inline(never)] -fn ensure_must_run( - qcx: Qcx, - key: &K, - query: &QueryVTable, -) -> (bool, Option>) +fn ensure_must_run(qcx: Qcx, key: &Q::Key) -> (bool, Option>) where - K: crate::dep_graph::DepNodeParams, + Q: QueryConfig, Qcx: QueryContext, { - if query.eval_always { + if Q::EVAL_ALWAYS { return (true, None); } // Ensuring an anonymous query makes no sense - assert!(!query.anon); + assert!(!Q::ANON); - let dep_node = query.to_dep_node(*qcx.dep_context(), key); + let dep_node = Q::construct_dep_node(*qcx.dep_context(), key); let dep_graph = qcx.dep_context().dep_graph(); match dep_graph.try_mark_green(qcx, &dep_node) { @@ -739,16 +730,15 @@ pub enum QueryMode { Ensure, } -pub fn get_query(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option +pub fn get_query(qcx: Qcx, span: Span, key: Q::Key, mode: QueryMode) -> Option where + D: DepKind, Q: QueryConfig, - Q::Key: DepNodeParams, - Q::Value: Value, + Q::Value: Value, Qcx: QueryContext, { - let query = Q::make_vtable(qcx, &key); let dep_node = if let QueryMode::Ensure = mode { - let (must_run, dep_node) = ensure_must_run(qcx, &key, &query); + let (must_run, dep_node) = ensure_must_run::(qcx, &key); if !must_run { return None; } @@ -757,14 +747,13 @@ where None }; - let (result, dep_node_index) = try_execute_query( + let (result, dep_node_index) = try_execute_query::( qcx, Q::query_state(qcx), Q::query_cache(qcx), span, key, dep_node, - &query, ); if let Some(dep_node_index) = dep_node_index { qcx.dep_context().dep_graph().read_index(dep_node_index) @@ -772,11 +761,11 @@ where Some(result) } -pub fn force_query(qcx: Qcx, key: Q::Key, dep_node: DepNode) +pub fn force_query(qcx: Qcx, key: Q::Key, dep_node: DepNode) where + D: DepKind, Q: QueryConfig, - Q::Key: DepNodeParams, - Q::Value: Value, + Q::Value: Value, Qcx: QueryContext, { // We may be concurrently trying both execute and force a query. @@ -793,9 +782,8 @@ where Err(()) => {} } - let query = Q::make_vtable(qcx, &key); let state = Q::query_state(qcx); - debug_assert!(!query.anon); + debug_assert!(!Q::ANON); - try_execute_query(qcx, state, cache, DUMMY_SP, key, Some(dep_node), &query); + try_execute_query::(qcx, state, cache, DUMMY_SP, key, Some(dep_node)); } -- cgit v1.2.3