diff options
Diffstat (limited to 'compiler/rustc_infer/src/infer/canonical/canonicalizer.rs')
| -rw-r--r-- | compiler/rustc_infer/src/infer/canonical/canonicalizer.rs | 271 |
1 files changed, 139 insertions, 132 deletions
diff --git a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs index 0e2f9ba70..5b00ef422 100644 --- a/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs +++ b/compiler/rustc_infer/src/infer/canonical/canonicalizer.rs @@ -9,11 +9,9 @@ use crate::infer::canonical::{ Canonical, CanonicalTyVarKind, CanonicalVarInfo, CanonicalVarKind, OriginalQueryValues, }; use crate::infer::InferCtxt; -use rustc_middle::ty::flags::FlagComputation; use rustc_middle::ty::fold::{TypeFoldable, TypeFolder, TypeSuperFoldable}; use rustc_middle::ty::GenericArg; use rustc_middle::ty::{self, BoundVar, InferConst, List, Ty, TyCtxt, TypeFlags, TypeVisitableExt}; -use std::sync::atomic::Ordering; use rustc_data_structures::fx::FxHashMap; use rustc_index::Idx; @@ -37,40 +35,13 @@ impl<'tcx> InferCtxt<'tcx> { /// [c]: https://rust-lang.github.io/chalk/book/canonical_queries/canonicalization.html#canonicalizing-the-query pub fn canonicalize_query<V>( &self, - value: V, - query_state: &mut OriginalQueryValues<'tcx>, - ) -> Canonical<'tcx, V> - where - V: TypeFoldable<TyCtxt<'tcx>>, - { - self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed); - - Canonicalizer::canonicalize(value, self, self.tcx, &CanonicalizeAllFreeRegions, query_state) - } - - /// Like [Self::canonicalize_query], but preserves distinct universes. For - /// example, canonicalizing `&'?0: Trait<'?1>`, where `'?0` is in `U1` and - /// `'?1` is in `U3` would be canonicalized to have `?0` in `U1` and `'?1` - /// in `U2`. - /// - /// This is used for Chalk integration. - pub fn canonicalize_query_preserving_universes<V>( - &self, - value: V, + value: ty::ParamEnvAnd<'tcx, V>, query_state: &mut OriginalQueryValues<'tcx>, - ) -> Canonical<'tcx, V> + ) -> Canonical<'tcx, ty::ParamEnvAnd<'tcx, V>> where V: TypeFoldable<TyCtxt<'tcx>>, { - self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed); - - Canonicalizer::canonicalize( - value, - self, - self.tcx, - &CanonicalizeAllFreeRegionsPreservingUniverses, - query_state, - ) + self.canonicalize_query_with_mode(value, query_state, &CanonicalizeAllFreeRegions) } /// Canonicalizes a query *response* `V`. When we canonicalize a @@ -105,7 +76,7 @@ impl<'tcx> InferCtxt<'tcx> { let mut query_state = OriginalQueryValues::default(); Canonicalizer::canonicalize( value, - self, + Some(self), self.tcx, &CanonicalizeQueryResponse, &mut query_state, @@ -119,7 +90,7 @@ impl<'tcx> InferCtxt<'tcx> { let mut query_state = OriginalQueryValues::default(); Canonicalizer::canonicalize( value, - self, + Some(self), self.tcx, &CanonicalizeUserTypeAnnotation, &mut query_state, @@ -132,21 +103,53 @@ impl<'tcx> InferCtxt<'tcx> { /// handling of `'static` regions (e.g. trait evaluation). pub fn canonicalize_query_keep_static<V>( &self, - value: V, + value: ty::ParamEnvAnd<'tcx, V>, query_state: &mut OriginalQueryValues<'tcx>, - ) -> Canonical<'tcx, V> + ) -> Canonical<'tcx, ty::ParamEnvAnd<'tcx, V>> + where + V: TypeFoldable<TyCtxt<'tcx>>, + { + self.canonicalize_query_with_mode( + value, + query_state, + &CanonicalizeFreeRegionsOtherThanStatic, + ) + } + + fn canonicalize_query_with_mode<V>( + &self, + value: ty::ParamEnvAnd<'tcx, V>, + query_state: &mut OriginalQueryValues<'tcx>, + canonicalize_region_mode: &dyn CanonicalizeMode, + ) -> Canonical<'tcx, ty::ParamEnvAnd<'tcx, V>> where V: TypeFoldable<TyCtxt<'tcx>>, { - self.tcx.sess.perf_stats.queries_canonicalized.fetch_add(1, Ordering::Relaxed); + let (param_env, value) = value.into_parts(); + let base = self.tcx.canonical_param_env_cache.get_or_insert( + self.tcx, + param_env, + query_state, + |tcx, param_env, query_state| { + Canonicalizer::canonicalize( + param_env, + None, + tcx, + &CanonicalizeFreeRegionsOtherThanStatic, + query_state, + ) + }, + ); - Canonicalizer::canonicalize( + Canonicalizer::canonicalize_with_base( + base, value, - self, + Some(self), self.tcx, - &CanonicalizeFreeRegionsOtherThanStatic, + canonicalize_region_mode, query_state, ) + .unchecked_map(|(param_env, value)| param_env.and(value)) } } @@ -176,10 +179,24 @@ impl CanonicalizeMode for CanonicalizeQueryResponse { fn canonicalize_free_region<'tcx>( &self, canonicalizer: &mut Canonicalizer<'_, 'tcx>, - r: ty::Region<'tcx>, + mut r: ty::Region<'tcx>, ) -> ty::Region<'tcx> { + let infcx = canonicalizer.infcx.unwrap(); + + if let ty::ReVar(vid) = *r { + r = infcx + .inner + .borrow_mut() + .unwrap_region_constraints() + .opportunistic_resolve_var(canonicalizer.tcx, vid); + debug!( + "canonical: region var found with vid {vid:?}, \ + opportunistically resolved to {r:?}", + ); + }; + match *r { - ty::ReFree(_) | ty::ReErased | ty::ReStatic | ty::ReEarlyBound(..) => r, + ty::ReLateParam(_) | ty::ReErased | ty::ReStatic | ty::ReEarlyParam(..) => r, ty::RePlaceholder(placeholder) => canonicalizer.canonical_var_for_region( CanonicalVarInfo { kind: CanonicalVarKind::PlaceholderRegion(placeholder) }, @@ -187,7 +204,8 @@ impl CanonicalizeMode for CanonicalizeQueryResponse { ), ty::ReVar(vid) => { - let universe = canonicalizer.region_var_universe(vid); + let universe = + infcx.inner.borrow_mut().unwrap_region_constraints().var_universe(vid); canonicalizer.canonical_var_for_region( CanonicalVarInfo { kind: CanonicalVarKind::Region(universe) }, r, @@ -202,8 +220,8 @@ impl CanonicalizeMode for CanonicalizeQueryResponse { // // rust-lang/rust#57464: `impl Trait` can leak local // scopes (in manner violating typeck). Therefore, use - // `delay_span_bug` to allow type error over an ICE. - canonicalizer.tcx.sess.delay_span_bug( + // `span_delayed_bug` to allow type error over an ICE. + canonicalizer.tcx.sess.span_delayed_bug( rustc_span::DUMMY_SP, format!("unexpected region in query response: `{r:?}`"), ); @@ -230,9 +248,13 @@ impl CanonicalizeMode for CanonicalizeUserTypeAnnotation { r: ty::Region<'tcx>, ) -> ty::Region<'tcx> { match *r { - ty::ReEarlyBound(_) | ty::ReFree(_) | ty::ReErased | ty::ReStatic | ty::ReError(_) => r, + ty::ReEarlyParam(_) + | ty::ReLateParam(_) + | ty::ReErased + | ty::ReStatic + | ty::ReError(_) => r, ty::ReVar(_) => canonicalizer.canonical_var_for_region_in_root_universe(r), - ty::RePlaceholder(..) | ty::ReLateBound(..) => { + ty::RePlaceholder(..) | ty::ReBound(..) => { // We only expect region names that the user can type. bug!("unexpected region in query response: `{:?}`", r) } @@ -268,30 +290,6 @@ impl CanonicalizeMode for CanonicalizeAllFreeRegions { } } -struct CanonicalizeAllFreeRegionsPreservingUniverses; - -impl CanonicalizeMode for CanonicalizeAllFreeRegionsPreservingUniverses { - fn canonicalize_free_region<'tcx>( - &self, - canonicalizer: &mut Canonicalizer<'_, 'tcx>, - r: ty::Region<'tcx>, - ) -> ty::Region<'tcx> { - let universe = canonicalizer.infcx.universe_of_region(r); - canonicalizer.canonical_var_for_region( - CanonicalVarInfo { kind: CanonicalVarKind::Region(universe) }, - r, - ) - } - - fn any(&self) -> bool { - true - } - - fn preserve_universes(&self) -> bool { - true - } -} - struct CanonicalizeFreeRegionsOtherThanStatic; impl CanonicalizeMode for CanonicalizeFreeRegionsOtherThanStatic { @@ -313,7 +311,8 @@ impl CanonicalizeMode for CanonicalizeFreeRegionsOtherThanStatic { } struct Canonicalizer<'cx, 'tcx> { - infcx: &'cx InferCtxt<'tcx>, + /// Set to `None` to disable the resolution of inference variables. + infcx: Option<&'cx InferCtxt<'tcx>>, tcx: TyCtxt<'tcx>, variables: SmallVec<[CanonicalVarInfo<'tcx>; 8]>, query_state: &'cx mut OriginalQueryValues<'tcx>, @@ -343,7 +342,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> { fn fold_region(&mut self, r: ty::Region<'tcx>) -> ty::Region<'tcx> { match *r { - ty::ReLateBound(index, ..) => { + ty::ReBound(index, ..) => { if index >= self.binder_index { bug!("escaping late-bound region during canonicalization"); } else { @@ -351,25 +350,12 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> { } } - ty::ReVar(vid) => { - let resolved = self - .infcx - .inner - .borrow_mut() - .unwrap_region_constraints() - .opportunistic_resolve_var(self.tcx, vid); - debug!( - "canonical: region var found with vid {vid:?}, \ - opportunistically resolved to {resolved:?}", - ); - self.canonicalize_mode.canonicalize_free_region(self, resolved) - } - ty::ReStatic - | ty::ReEarlyBound(..) + | ty::ReEarlyParam(..) | ty::ReError(_) - | ty::ReFree(_) + | ty::ReLateParam(_) | ty::RePlaceholder(..) + | ty::ReVar(_) | ty::ReErased => self.canonicalize_mode.canonicalize_free_region(self, r), } } @@ -380,14 +366,14 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> { // We need to canonicalize the *root* of our ty var. // This is so that our canonical response correctly reflects // any equated inference vars correctly! - let root_vid = self.infcx.root_var(vid); + let root_vid = self.infcx.unwrap().root_var(vid); if root_vid != vid { - t = Ty::new_var(self.infcx.tcx, root_vid); + t = Ty::new_var(self.tcx, root_vid); vid = root_vid; } debug!("canonical: type var found with vid {:?}", vid); - match self.infcx.probe_ty_var(vid) { + match self.infcx.unwrap().probe_ty_var(vid) { // `t` could be a float / int variable; canonicalize that instead. Ok(t) => { debug!("(resolved to {:?})", t); @@ -412,7 +398,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> { } ty::Infer(ty::IntVar(vid)) => { - let nt = self.infcx.opportunistic_resolve_int_var(vid); + let nt = self.infcx.unwrap().opportunistic_resolve_int_var(vid); if nt != t { return self.fold_ty(nt); } else { @@ -423,7 +409,7 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> { } } ty::Infer(ty::FloatVar(vid)) => { - let nt = self.infcx.opportunistic_resolve_float_var(vid); + let nt = self.infcx.unwrap().opportunistic_resolve_float_var(vid); if nt != t { return self.fold_ty(nt); } else { @@ -494,14 +480,14 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> { // We need to canonicalize the *root* of our const var. // This is so that our canonical response correctly reflects // any equated inference vars correctly! - let root_vid = self.infcx.root_const_var(vid); + let root_vid = self.infcx.unwrap().root_const_var(vid); if root_vid != vid { - ct = ty::Const::new_var(self.infcx.tcx, root_vid, ct.ty()); + ct = ty::Const::new_var(self.tcx, root_vid, ct.ty()); vid = root_vid; } debug!("canonical: const var found with vid {:?}", vid); - match self.infcx.probe_const_var(vid) { + match self.infcx.unwrap().probe_const_var(vid) { Ok(c) => { debug!("(resolved to {:?})", c); return self.fold_const(c); @@ -522,8 +508,8 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> { } } ty::ConstKind::Infer(InferConst::EffectVar(vid)) => { - match self.infcx.probe_effect_var(vid) { - Some(value) => return self.fold_const(value.as_const(self.infcx.tcx)), + match self.infcx.unwrap().probe_effect_var(vid) { + Some(value) => return self.fold_const(value.as_const(self.tcx)), None => { return self.canonicalize_const_var( CanonicalVarInfo { kind: CanonicalVarKind::Effect }, @@ -553,8 +539,11 @@ impl<'cx, 'tcx> TypeFolder<TyCtxt<'tcx>> for Canonicalizer<'cx, 'tcx> { _ => {} } - let flags = FlagComputation::for_const(ct); - if flags.intersects(self.needs_canonical_flags) { ct.super_fold_with(self) } else { ct } + if ct.flags().intersects(self.needs_canonical_flags) { + ct.super_fold_with(self) + } else { + ct + } } } @@ -563,7 +552,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { /// `canonicalize_query` and `canonicalize_response`. fn canonicalize<V>( value: V, - infcx: &InferCtxt<'tcx>, + infcx: Option<&InferCtxt<'tcx>>, tcx: TyCtxt<'tcx>, canonicalize_region_mode: &dyn CanonicalizeMode, query_state: &mut OriginalQueryValues<'tcx>, @@ -571,6 +560,33 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { where V: TypeFoldable<TyCtxt<'tcx>>, { + let base = Canonical { + max_universe: ty::UniverseIndex::ROOT, + variables: List::empty(), + value: (), + }; + Canonicalizer::canonicalize_with_base( + base, + value, + infcx, + tcx, + canonicalize_region_mode, + query_state, + ) + .unchecked_map(|((), val)| val) + } + + fn canonicalize_with_base<U, V>( + base: Canonical<'tcx, U>, + value: V, + infcx: Option<&InferCtxt<'tcx>>, + tcx: TyCtxt<'tcx>, + canonicalize_region_mode: &dyn CanonicalizeMode, + query_state: &mut OriginalQueryValues<'tcx>, + ) -> Canonical<'tcx, (U, V)> + where + V: TypeFoldable<TyCtxt<'tcx>>, + { let needs_canonical_flags = if canonicalize_region_mode.any() { TypeFlags::HAS_INFER | TypeFlags::HAS_PLACEHOLDER | TypeFlags::HAS_FREE_REGIONS } else { @@ -579,12 +595,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { // Fast path: nothing that needs to be canonicalized. if !value.has_type_flags(needs_canonical_flags) { - let canon_value = Canonical { - max_universe: ty::UniverseIndex::ROOT, - variables: List::empty(), - value, - }; - return canon_value; + return base.unchecked_map(|b| (b, value)); } let mut canonicalizer = Canonicalizer { @@ -592,11 +603,20 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { tcx, canonicalize_mode: canonicalize_region_mode, needs_canonical_flags, - variables: SmallVec::new(), + variables: SmallVec::from_slice(base.variables), query_state, indices: FxHashMap::default(), binder_index: ty::INNERMOST, }; + if canonicalizer.query_state.var_values.spilled() { + canonicalizer.indices = canonicalizer + .query_state + .var_values + .iter() + .enumerate() + .map(|(i, &kind)| (kind, BoundVar::new(i))) + .collect(); + } let out_value = value.fold_with(&mut canonicalizer); // Once we have canonicalized `out_value`, it should not @@ -613,7 +633,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { .max() .unwrap_or(ty::UniverseIndex::ROOT); - Canonical { max_universe, variables: canonical_variables, value: out_value } + Canonical { max_universe, variables: canonical_variables, value: (base.value, out_value) } } /// Creates a canonical variable replacing `kind` from the input, @@ -762,11 +782,6 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { ) } - /// Returns the universe in which `vid` is defined. - fn region_var_universe(&self, vid: ty::RegionVid) -> ty::UniverseIndex { - self.infcx.inner.borrow_mut().unwrap_region_constraints().var_universe(vid) - } - /// Creates a canonical variable (with the given `info`) /// representing the region `r`; return a region referencing it. fn canonical_var_for_region( @@ -776,7 +791,7 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { ) -> ty::Region<'tcx> { let var = self.canonical_var(info, r.into()); let br = ty::BoundRegion { var, kind: ty::BrAnon }; - ty::Region::new_late_bound(self.interner(), self.binder_index, br) + ty::Region::new_bound(self.interner(), self.binder_index, br) } /// Given a type variable `ty_var` of the given kind, first check @@ -784,14 +799,9 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { /// *that*. Otherwise, create a new canonical variable for /// `ty_var`. fn canonicalize_ty_var(&mut self, info: CanonicalVarInfo<'tcx>, ty_var: Ty<'tcx>) -> Ty<'tcx> { - let infcx = self.infcx; - let bound_to = infcx.shallow_resolve(ty_var); - if bound_to != ty_var { - self.fold_ty(bound_to) - } else { - let var = self.canonical_var(info, ty_var.into()); - Ty::new_bound(self.tcx, self.binder_index, var.into()) - } + debug_assert!(!self.infcx.is_some_and(|infcx| ty_var != infcx.shallow_resolve(ty_var))); + let var = self.canonical_var(info, ty_var.into()); + Ty::new_bound(self.tcx, self.binder_index, var.into()) } /// Given a type variable `const_var` of the given kind, first check @@ -803,13 +813,10 @@ impl<'cx, 'tcx> Canonicalizer<'cx, 'tcx> { info: CanonicalVarInfo<'tcx>, const_var: ty::Const<'tcx>, ) -> ty::Const<'tcx> { - let infcx = self.infcx; - let bound_to = infcx.shallow_resolve(const_var); - if bound_to != const_var { - self.fold_const(bound_to) - } else { - let var = self.canonical_var(info, const_var.into()); - ty::Const::new_bound(self.tcx, self.binder_index, var, self.fold_ty(const_var.ty())) - } + debug_assert!( + !self.infcx.is_some_and(|infcx| const_var != infcx.shallow_resolve(const_var)) + ); + let var = self.canonical_var(info, const_var.into()); + ty::Const::new_bound(self.tcx, self.binder_index, var, self.fold_ty(const_var.ty())) } } |