diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src')
35 files changed, 600 insertions, 241 deletions
diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs index 187d38b38..d9e7339f1 100644 --- a/compiler/rustc_mir_transform/src/add_retag.rs +++ b/compiler/rustc_mir_transform/src/add_retag.rs @@ -28,6 +28,7 @@ fn may_contain_reference<'tcx>(ty: Ty<'tcx>, depth: u32, tcx: TyCtxt<'tcx>) -> b // References ty::Ref(..) => true, ty::Adt(..) if ty.is_box() => true, + ty::Adt(adt, _) if Some(adt.did()) == tcx.lang_items().ptr_unique() => true, // Compound types: recurse ty::Array(ty, _) | ty::Slice(ty) => { // This does not branch so we keep the depth the same. diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs index ea0780688..4892ace53 100644 --- a/compiler/rustc_mir_transform/src/check_alignment.rs +++ b/compiler/rustc_mir_transform/src/check_alignment.rs @@ -155,7 +155,7 @@ fn insert_alignment_check<'tcx>( new_block: BasicBlock, ) { // Cast the pointer to a *const () - let const_raw_ptr = tcx.mk_ptr(TypeAndMut { ty: tcx.types.unit, mutbl: Mutability::Not }); + let const_raw_ptr = Ty::new_ptr(tcx, TypeAndMut { ty: tcx.types.unit, mutbl: Mutability::Not }); let rvalue = Rvalue::Cast(CastKind::PtrToPtr, Operand::Copy(pointer), const_raw_ptr); let thin_ptr = local_decls.push(LocalDecl::with_source_info(const_raw_ptr, source_info)).into(); block_data @@ -240,7 +240,10 @@ fn insert_alignment_check<'tcx>( required: Operand::Copy(alignment), found: Operand::Copy(addr), }), - unwind: UnwindAction::Terminate, + // This calls panic_misaligned_pointer_dereference, which is #[rustc_nounwind]. + // We never want to insert an unwind into unsafe code, because unwinding could + // make a failing UB check turn into much worse UB when we start unwinding. + unwind: UnwindAction::Unreachable, }, }); } diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index 069514d8a..70812761e 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -421,10 +421,8 @@ impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> { intravisit::walk_block(self, block); } - fn visit_anon_const(&mut self, c: &'tcx hir::AnonConst) { - if matches!(self.tcx.def_kind(c.def_id), DefKind::InlineConst) { - self.visit_body(self.tcx.hir().body(c.body)) - } + fn visit_inline_const(&mut self, c: &'tcx hir::ConstBlock) { + self.visit_body(self.tcx.hir().body(c.body)) } fn visit_fn( @@ -527,6 +525,8 @@ pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) { } let UnsafetyCheckResult { violations, unused_unsafes, .. } = tcx.unsafety_check_result(def_id); + // Only suggest wrapping the entire function body in an unsafe block once + let mut suggest_unsafe_block = true; for &UnsafetyViolation { source_info, lint_root, kind, details } in violations.iter() { let details = errors::RequiresUnsafeDetail { violation: details, span: source_info.span }; @@ -561,12 +561,29 @@ pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) { op_in_unsafe_fn_allowed, }); } - UnsafetyViolationKind::UnsafeFn => tcx.emit_spanned_lint( - UNSAFE_OP_IN_UNSAFE_FN, - lint_root, - source_info.span, - errors::UnsafeOpInUnsafeFn { details }, - ), + UnsafetyViolationKind::UnsafeFn => { + tcx.emit_spanned_lint( + UNSAFE_OP_IN_UNSAFE_FN, + lint_root, + source_info.span, + errors::UnsafeOpInUnsafeFn { + details, + suggest_unsafe_block: suggest_unsafe_block.then(|| { + let hir_id = tcx.hir().local_def_id_to_hir_id(def_id); + let fn_sig = tcx + .hir() + .fn_sig_by_hir_id(hir_id) + .expect("this violation only occurs in fn"); + let body = tcx.hir().body_owned_by(def_id); + let body_span = tcx.hir().body(body).value.span; + let start = tcx.sess.source_map().start_point(body_span).shrink_to_hi(); + let end = tcx.sess.source_map().end_point(body_span).shrink_to_lo(); + (start, end, fn_sig.span) + }), + }, + ); + suggest_unsafe_block = false; + } } } diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs index da101ca7a..e175f22d7 100644 --- a/compiler/rustc_mir_transform/src/const_goto.rs +++ b/compiler/rustc_mir_transform/src/const_goto.rs @@ -28,7 +28,9 @@ pub struct ConstGoto; impl<'tcx> MirPass<'tcx> for ConstGoto { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 4 + // This pass participates in some as-of-yet untested unsoundness found + // in https://github.com/rust-lang/rust/issues/112460 + sess.mir_opt_level() >= 2 && sess.opts.unstable_opts.unsound_mir_opts } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 1ba1951af..2f2c7357b 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -4,6 +4,7 @@ use either::Right; use rustc_const_eval::const_eval::CheckAlignment; +use rustc_const_eval::ReportErrorExt; use rustc_data_structures::fx::FxHashSet; use rustc_hir::def::DefKind; use rustc_index::bit_set::BitSet; @@ -37,6 +38,7 @@ macro_rules! throw_machine_stop_str { ($($tt:tt)*) => {{ // We make a new local type for it. The type itself does not carry any information, // but its vtable (for the `MachineStopType` trait) does. + #[derive(Debug)] struct Zst; // Printing this type shows the desired string. impl std::fmt::Display for Zst { @@ -44,7 +46,17 @@ macro_rules! throw_machine_stop_str { write!(f, $($tt)*) } } - impl rustc_middle::mir::interpret::MachineStopType for Zst {} + + impl rustc_middle::mir::interpret::MachineStopType for Zst { + fn diagnostic_message(&self) -> rustc_errors::DiagnosticMessage { + self.to_string().into() + } + + fn add_args( + self: Box<Self>, + _: &mut dyn FnMut(std::borrow::Cow<'static, str>, rustc_errors::DiagnosticArgValue<'static>), + ) {} + } throw_machine_stop!(Zst) }}; } @@ -103,7 +115,14 @@ impl<'tcx> MirPass<'tcx> for ConstProp { // That would require a uniform one-def no-mutation analysis // and RPO (or recursing when needing the value of a local). let mut optimization_finder = ConstPropagator::new(body, dummy_body, tcx); - optimization_finder.visit_body(body); + + // Traverse the body in reverse post-order, to ensure that `FullConstProp` locals are + // assigned before being read. + let rpo = body.basic_blocks.reverse_postorder().to_vec(); + for bb in rpo { + let data = &mut body.basic_blocks.as_mut_preserves_cfg()[bb]; + optimization_finder.visit_basic_block_data(bb, data); + } trace!("ConstProp done for {:?}", def_id); } @@ -367,7 +386,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { op } Err(e) => { - trace!("get_const failed: {}", e); + trace!("get_const failed: {:?}", e.into_kind().debug()); return None; } }; @@ -789,12 +808,6 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { self.tcx } - fn visit_body(&mut self, body: &mut Body<'tcx>) { - for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { - self.visit_basic_block_data(bb, data); - } - } - fn visit_operand(&mut self, operand: &mut Operand<'tcx>, location: Location) { self.super_operand(operand, location); @@ -885,14 +898,23 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { } } StatementKind::StorageLive(local) => { - let frame = self.ecx.frame_mut(); - frame.locals[local].value = - LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit)); - } - StatementKind::StorageDead(local) => { - let frame = self.ecx.frame_mut(); - frame.locals[local].value = LocalValue::Dead; + Self::remove_const(&mut self.ecx, local); } + // We do not need to mark dead locals as such. For `FullConstProp` locals, + // this allows to propagate the single assigned value in this case: + // ``` + // let x = SOME_CONST; + // if a { + // f(copy x); + // StorageDead(x); + // } else { + // g(copy x); + // StorageDead(x); + // } + // ``` + // + // This may propagate a constant where the local would be uninit or dead. + // In both cases, this does not matter, as those reads would be UB anyway. _ => {} } } diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 0fe49b8a1..759650fe4 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -9,6 +9,7 @@ use rustc_const_eval::interpret::Immediate; use rustc_const_eval::interpret::{ self, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup, }; +use rustc_const_eval::ReportErrorExt; use rustc_hir::def::DefKind; use rustc_hir::HirId; use rustc_index::bit_set::BitSet; @@ -232,7 +233,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { op } Err(e) => { - trace!("get_const failed: {}", e); + trace!("get_const failed: {:?}", e.into_kind().debug()); return None; } }; @@ -272,8 +273,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // dedicated error variants should be introduced instead. assert!( !error.kind().formatted_string(), - "const-prop encountered formatting error: {}", - error + "const-prop encountered formatting error: {error:?}", ); None } diff --git a/compiler/rustc_mir_transform/src/coverage/debug.rs b/compiler/rustc_mir_transform/src/coverage/debug.rs index 6a3d42511..7ad981441 100644 --- a/compiler/rustc_mir_transform/src/coverage/debug.rs +++ b/compiler/rustc_mir_transform/src/coverage/debug.rs @@ -277,14 +277,7 @@ impl DebugCounters { pub fn add_counter(&mut self, counter_kind: &CoverageKind, some_block_label: Option<String>) { if let Some(counters) = &mut self.some_counters { - let id: ExpressionOperandId = match *counter_kind { - CoverageKind::Counter { id, .. } => id.into(), - CoverageKind::Expression { id, .. } => id.into(), - _ => bug!( - "the given `CoverageKind` is not an counter or expression: {:?}", - counter_kind - ), - }; + let id = counter_kind.as_operand_id(); counters .try_insert(id, DebugCounter::new(counter_kind.clone(), some_block_label)) .expect("attempt to add the same counter_kind to DebugCounters more than once"); @@ -330,13 +323,7 @@ impl DebugCounters { } } - let id: ExpressionOperandId = match *counter_kind { - CoverageKind::Counter { id, .. } => id.into(), - CoverageKind::Expression { id, .. } => id.into(), - _ => { - bug!("the given `CoverageKind` is not an counter or expression: {:?}", counter_kind) - } - }; + let id = counter_kind.as_operand_id(); if self.some_counters.is_some() && (counter_format.block || !counter_format.id) { let counters = self.some_counters.as_ref().unwrap(); if let Some(DebugCounter { some_block_label: Some(block_label), .. }) = diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index ea1223fbc..d2a854b26 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -387,7 +387,7 @@ impl BasicCoverageBlockData { // If the BCB has an edge counter (to be injected into a new `BasicBlock`), it can also // have an expression (to be injected into an existing `BasicBlock` represented by this // `BasicCoverageBlock`). - if !self.counter_kind.as_ref().map_or(true, |c| c.is_expression()) { + if self.counter_kind.as_ref().is_some_and(|c| !c.is_expression()) { return Error::from_string(format!( "attempt to add an incoming edge counter from {:?} when the target BCB already \ has a `Counter`", diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index d27200419..35cf9ea5f 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -480,9 +480,10 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { fn check_invoked_macro_name_span(&mut self) { if let Some(visible_macro) = self.curr().visible_macro(self.body_span) { - if self.prev_expn_span.map_or(true, |prev_expn_span| { - self.curr().expn_span.ctxt() != prev_expn_span.ctxt() - }) { + if !self + .prev_expn_span + .is_some_and(|prev_expn_span| self.curr().expn_span.ctxt() == prev_expn_span.ctxt()) + { let merged_prefix_len = self.curr_original_span.lo() - self.curr().span.lo(); let after_macro_bang = merged_prefix_len + BytePos(visible_macro.as_str().len() as u32 + 1); diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 90b58933d..25891d3ca 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -140,7 +140,7 @@ impl<'tcx> MockBlocks<'tcx> { destination: self.dummy_place.clone(), target: Some(TEMP_BLOCK), unwind: UnwindAction::Continue, - from_hir_call: false, + call_source: CallSource::Misc, fn_span: DUMMY_SP, }, ) diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index 7adfc9dff..78fb19635 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -10,8 +10,12 @@ use rustc_middle::mir::visit::{MutVisitor, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::layout::TyAndLayout; use rustc_middle::ty::{self, Ty, TyCtxt}; -use rustc_mir_dataflow::value_analysis::{Map, State, TrackElem, ValueAnalysis, ValueOrPlace}; -use rustc_mir_dataflow::{lattice::FlatSet, Analysis, ResultsVisitor, SwitchIntEdgeEffects}; +use rustc_mir_dataflow::value_analysis::{ + Map, State, TrackElem, ValueAnalysis, ValueAnalysisWrapper, ValueOrPlace, +}; +use rustc_mir_dataflow::{ + lattice::FlatSet, Analysis, Results, ResultsVisitor, SwitchIntEdgeEffects, +}; use rustc_span::DUMMY_SP; use rustc_target::abi::{Align, FieldIdx, VariantIdx}; @@ -52,11 +56,11 @@ impl<'tcx> MirPass<'tcx> for DataflowConstProp { // Perform the actual dataflow analysis. let analysis = ConstAnalysis::new(tcx, body, map); - let results = debug_span!("analyze") + let mut results = debug_span!("analyze") .in_scope(|| analysis.wrap().into_engine(tcx, body).iterate_to_fixpoint()); // Collect results and patch the body afterwards. - let mut visitor = CollectAndPatch::new(tcx, &results.analysis.0.map); + let mut visitor = CollectAndPatch::new(tcx); debug_span!("collect").in_scope(|| results.visit_reachable_with(body, &mut visitor)); debug_span!("patch").in_scope(|| visitor.visit_body(body)); } @@ -387,9 +391,8 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } } -struct CollectAndPatch<'tcx, 'map> { +struct CollectAndPatch<'tcx> { tcx: TyCtxt<'tcx>, - map: &'map Map, /// For a given MIR location, this stores the values of the operands used by that location. In /// particular, this is before the effect, such that the operands of `_1 = _1 + _2` are @@ -400,9 +403,9 @@ struct CollectAndPatch<'tcx, 'map> { assignments: FxHashMap<Location, ScalarTy<'tcx>>, } -impl<'tcx, 'map> CollectAndPatch<'tcx, 'map> { - fn new(tcx: TyCtxt<'tcx>, map: &'map Map) -> Self { - Self { tcx, map, before_effect: FxHashMap::default(), assignments: FxHashMap::default() } +impl<'tcx> CollectAndPatch<'tcx> { + fn new(tcx: TyCtxt<'tcx>) -> Self { + Self { tcx, before_effect: FxHashMap::default(), assignments: FxHashMap::default() } } fn make_operand(&self, scalar: ScalarTy<'tcx>) -> Operand<'tcx> { @@ -414,18 +417,23 @@ impl<'tcx, 'map> CollectAndPatch<'tcx, 'map> { } } -impl<'mir, 'tcx, 'map> ResultsVisitor<'mir, 'tcx> for CollectAndPatch<'tcx, 'map> { +impl<'mir, 'tcx> + ResultsVisitor<'mir, 'tcx, Results<'tcx, ValueAnalysisWrapper<ConstAnalysis<'_, 'tcx>>>> + for CollectAndPatch<'tcx> +{ type FlowState = State<FlatSet<ScalarTy<'tcx>>>; fn visit_statement_before_primary_effect( &mut self, + results: &Results<'tcx, ValueAnalysisWrapper<ConstAnalysis<'_, 'tcx>>>, state: &Self::FlowState, statement: &'mir Statement<'tcx>, location: Location, ) { match &statement.kind { StatementKind::Assign(box (_, rvalue)) => { - OperandCollector { state, visitor: self }.visit_rvalue(rvalue, location); + OperandCollector { state, visitor: self, map: &results.analysis.0.map } + .visit_rvalue(rvalue, location); } _ => (), } @@ -433,6 +441,7 @@ impl<'mir, 'tcx, 'map> ResultsVisitor<'mir, 'tcx> for CollectAndPatch<'tcx, 'map fn visit_statement_after_primary_effect( &mut self, + results: &Results<'tcx, ValueAnalysisWrapper<ConstAnalysis<'_, 'tcx>>>, state: &Self::FlowState, statement: &'mir Statement<'tcx>, location: Location, @@ -441,30 +450,34 @@ impl<'mir, 'tcx, 'map> ResultsVisitor<'mir, 'tcx> for CollectAndPatch<'tcx, 'map StatementKind::Assign(box (_, Rvalue::Use(Operand::Constant(_)))) => { // Don't overwrite the assignment if it already uses a constant (to keep the span). } - StatementKind::Assign(box (place, _)) => match state.get(place.as_ref(), self.map) { - FlatSet::Top => (), - FlatSet::Elem(value) => { - self.assignments.insert(location, value); - } - FlatSet::Bottom => { - // This assignment is either unreachable, or an uninitialized value is assigned. + StatementKind::Assign(box (place, _)) => { + match state.get(place.as_ref(), &results.analysis.0.map) { + FlatSet::Top => (), + FlatSet::Elem(value) => { + self.assignments.insert(location, value); + } + FlatSet::Bottom => { + // This assignment is either unreachable, or an uninitialized value is assigned. + } } - }, + } _ => (), } } fn visit_terminator_before_primary_effect( &mut self, + results: &Results<'tcx, ValueAnalysisWrapper<ConstAnalysis<'_, 'tcx>>>, state: &Self::FlowState, terminator: &'mir Terminator<'tcx>, location: Location, ) { - OperandCollector { state, visitor: self }.visit_terminator(terminator, location); + OperandCollector { state, visitor: self, map: &results.analysis.0.map } + .visit_terminator(terminator, location); } } -impl<'tcx, 'map> MutVisitor<'tcx> for CollectAndPatch<'tcx, 'map> { +impl<'tcx> MutVisitor<'tcx> for CollectAndPatch<'tcx> { fn tcx<'a>(&'a self) -> TyCtxt<'tcx> { self.tcx } @@ -496,14 +509,15 @@ impl<'tcx, 'map> MutVisitor<'tcx> for CollectAndPatch<'tcx, 'map> { struct OperandCollector<'tcx, 'map, 'a> { state: &'a State<FlatSet<ScalarTy<'tcx>>>, - visitor: &'a mut CollectAndPatch<'tcx, 'map>, + visitor: &'a mut CollectAndPatch<'tcx>, + map: &'map Map, } impl<'tcx, 'map, 'a> Visitor<'tcx> for OperandCollector<'tcx, 'map, 'a> { fn visit_operand(&mut self, operand: &Operand<'tcx>, location: Location) { match operand { Operand::Copy(place) | Operand::Move(place) => { - match self.state.get(place.as_ref(), self.visitor.map) { + match self.state.get(place.as_ref(), self.map) { FlatSet::Top => (), FlatSet::Elem(value) => { self.visitor.before_effect.insert((location, *place), value); diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs index a133c9d47..e782c0373 100644 --- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs +++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs @@ -203,7 +203,12 @@ pub fn deduced_param_attrs<'tcx>( body.local_decls.iter().skip(1).take(body.arg_count).enumerate().map( |(arg_index, local_decl)| DeducedParamAttrs { read_only: !deduce_read_only.mutable_args.contains(arg_index) - && local_decl.ty.is_freeze(tcx, param_env), + // We must normalize here to reveal opaques and normalize + // their substs, otherwise we'll see exponential blow-up in + // compile times: #113372 + && tcx + .normalize_erasing_regions(param_env, local_decl.ty) + .is_freeze(tcx, param_env), }, ), ); diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index a39026751..95898b5b7 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -8,13 +8,13 @@ use rustc_middle::ty::TyCtxt; pub struct Derefer; -pub struct DerefChecker<'tcx> { +pub struct DerefChecker<'a, 'tcx> { tcx: TyCtxt<'tcx>, patcher: MirPatch<'tcx>, - local_decls: IndexVec<Local, LocalDecl<'tcx>>, + local_decls: &'a IndexVec<Local, LocalDecl<'tcx>>, } -impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> { +impl<'a, 'tcx> MutVisitor<'tcx> for DerefChecker<'a, 'tcx> { fn tcx(&self) -> TyCtxt<'tcx> { self.tcx } @@ -36,7 +36,7 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> { for (idx, (p_ref, p_elem)) in place.iter_projections().enumerate() { if !p_ref.projection.is_empty() && p_elem == ProjectionElem::Deref { - let ty = p_ref.ty(&self.local_decls, self.tcx).ty; + let ty = p_ref.ty(self.local_decls, self.tcx).ty; let temp = self.patcher.new_internal_with_info( ty, self.local_decls[p_ref.local].source_info.span, @@ -70,7 +70,7 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> { pub fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let patch = MirPatch::new(body); - let mut checker = DerefChecker { tcx, patcher: patch, local_decls: body.local_decls.clone() }; + let mut checker = DerefChecker { tcx, patcher: patch, local_decls: &body.local_decls }; for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() { checker.visit_basic_block_data(bb, data); diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 78758e2db..a31551cf6 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -37,6 +37,10 @@ //! if they do not consistently refer to the same place in memory. This is satisfied if they do //! not contain any indirection through a pointer or any indexing projections. //! +//! * `p` and `q` must have the **same type**. If we replace a local with a subtype or supertype, +//! we may end up with a differnet vtable for that local. See the `subtyping-impacts-selection` +//! tests for an example where that causes issues. +//! //! * We need to make sure that the goal of "merging the memory" is actually structurally possible //! in MIR. For example, even if all the other conditions are satisfied, there is no way to //! "merge" `_5.foo` and `_6.bar`. For now, we ensure this by requiring that both `p` and `q` are @@ -134,6 +138,7 @@ use crate::MirPass; use rustc_data_structures::fx::FxHashMap; use rustc_index::bit_set::BitSet; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; +use rustc_middle::mir::HasLocalDecls; use rustc_middle::mir::{dump_mir, PassWhere}; use rustc_middle::mir::{ traversal, Body, InlineAsmOperand, Local, LocalKind, Location, Operand, Place, Rvalue, @@ -763,12 +768,22 @@ impl<'tcx> Visitor<'tcx> for FindAssignments<'_, '_, 'tcx> { return; }; - // As described at the top of the file, we do not go near things that have their address - // taken. + // As described at the top of the file, we do not go near things that have + // their address taken. if self.borrowed.contains(src) || self.borrowed.contains(dest) { return; } + // As described at the top of this file, we do not touch locals which have + // different types. + let src_ty = self.body.local_decls()[src].ty; + let dest_ty = self.body.local_decls()[dest].ty; + if src_ty != dest_ty { + // FIXME(#112651): This can be removed afterwards. Also update the module description. + trace!("skipped `{src:?} = {dest:?}` due to subtyping: {src_ty} != {dest_ty}"); + return; + } + // Also, we need to make sure that MIR actually allows the `src` to be removed if is_local_required(src, self.body) { return; diff --git a/compiler/rustc_mir_transform/src/dump_mir.rs b/compiler/rustc_mir_transform/src/dump_mir.rs index 746e3d965..13841be49 100644 --- a/compiler/rustc_mir_transform/src/dump_mir.rs +++ b/compiler/rustc_mir_transform/src/dump_mir.rs @@ -7,7 +7,7 @@ use crate::MirPass; use rustc_middle::mir::write_mir_pretty; use rustc_middle::mir::Body; use rustc_middle::ty::TyCtxt; -use rustc_session::config::OutputType; +use rustc_session::config::{OutFileName, OutputType}; pub struct Marker(pub &'static str); @@ -20,8 +20,15 @@ impl<'tcx> MirPass<'tcx> for Marker { } pub fn emit_mir(tcx: TyCtxt<'_>) -> io::Result<()> { - let path = tcx.output_filenames(()).path(OutputType::Mir); - let mut f = io::BufWriter::new(File::create(&path)?); - write_mir_pretty(tcx, None, &mut f)?; + match tcx.output_filenames(()).path(OutputType::Mir) { + OutFileName::Stdout => { + let mut f = io::stdout(); + write_mir_pretty(tcx, None, &mut f)?; + } + OutFileName::Real(path) => { + let mut f = io::BufWriter::new(File::create(&path)?); + write_mir_pretty(tcx, None, &mut f)?; + } + } Ok(()) } diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs index f31653caa..cc0d7d51b 100644 --- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs +++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs @@ -21,7 +21,7 @@ pub fn build_ptr_tys<'tcx>( let substs = tcx.mk_substs(&[pointee.into()]); let unique_ty = tcx.type_of(unique_did).subst(tcx, substs); let nonnull_ty = tcx.type_of(nonnull_did).subst(tcx, substs); - let ptr_ty = tcx.mk_imm_ptr(pointee); + let ptr_ty = Ty::new_imm_ptr(tcx, pointee); (unique_ty, nonnull_ty, ptr_ty) } diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index fda0e1023..d5664e2b4 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -10,7 +10,6 @@ use rustc_mir_dataflow::elaborate_drops::{DropElaborator, DropFlagMode, DropStyl use rustc_mir_dataflow::impls::{MaybeInitializedPlaces, MaybeUninitializedPlaces}; use rustc_mir_dataflow::move_paths::{LookupResult, MoveData, MovePathIndex}; use rustc_mir_dataflow::on_lookup_result_bits; -use rustc_mir_dataflow::un_derefer::UnDerefer; use rustc_mir_dataflow::MoveDataParamEnv; use rustc_mir_dataflow::{on_all_children_bits, on_all_drop_children_bits}; use rustc_mir_dataflow::{Analysis, ResultsCursor}; @@ -54,20 +53,19 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops { let def_id = body.source.def_id(); let param_env = tcx.param_env_reveal_all_normalized(def_id); - let (side_table, move_data) = match MoveData::gather_moves(body, tcx, param_env) { + let move_data = match MoveData::gather_moves(body, tcx, param_env) { Ok(move_data) => move_data, Err((move_data, _)) => { tcx.sess.delay_span_bug( body.span, "No `move_errors` should be allowed in MIR borrowck", ); - (Default::default(), move_data) + move_data } }; - let un_derefer = UnDerefer { tcx: tcx, derefer_sidetable: side_table }; let elaborate_patch = { let env = MoveDataParamEnv { move_data, param_env }; - remove_dead_unwinds(tcx, body, &env, &un_derefer); + remove_dead_unwinds(tcx, body, &env); let inits = MaybeInitializedPlaces::new(tcx, body, &env) .into_engine(tcx, body) @@ -92,7 +90,6 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops { init_data: InitializationData { inits, uninits }, drop_flags, patch: MirPatch::new(body), - un_derefer: un_derefer, reachable, } .elaborate() @@ -108,7 +105,6 @@ fn remove_dead_unwinds<'tcx>( tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, env: &MoveDataParamEnv<'tcx>, - und: &UnDerefer<'tcx>, ) { debug!("remove_dead_unwinds({:?})", body.span); // We only need to do this pass once, because unwind edges can only @@ -121,9 +117,7 @@ fn remove_dead_unwinds<'tcx>( .into_results_cursor(body); for (bb, bb_data) in body.basic_blocks.iter_enumerated() { let place = match bb_data.terminator().kind { - TerminatorKind::Drop { ref place, unwind: UnwindAction::Cleanup(_), .. } => { - und.derefer(place.as_ref(), body).unwrap_or(*place) - } + TerminatorKind::Drop { place, unwind: UnwindAction::Cleanup(_), .. } => place, _ => continue, }; @@ -296,7 +290,6 @@ struct ElaborateDropsCtxt<'a, 'tcx> { init_data: InitializationData<'a, 'tcx>, drop_flags: IndexVec<MovePathIndex, Option<Local>>, patch: MirPatch<'tcx>, - un_derefer: UnDerefer<'tcx>, reachable: BitSet<BasicBlock>, } @@ -342,9 +335,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { } let terminator = data.terminator(); let place = match terminator.kind { - TerminatorKind::Drop { ref place, .. } => { - self.un_derefer.derefer(place.as_ref(), self.body).unwrap_or(*place) - } + TerminatorKind::Drop { ref place, .. } => place, _ => continue, }; @@ -401,11 +392,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { let terminator = data.terminator(); match terminator.kind { - TerminatorKind::Drop { mut place, target, unwind, replace } => { - if let Some(new_place) = self.un_derefer.derefer(place.as_ref(), self.body) { - place = new_place; - } - + TerminatorKind::Drop { place, target, unwind, replace } => { self.init_data.seek_before(loc); match self.move_data().rev_lookup.find(place.as_ref()) { LookupResult::Exact(path) => { diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs index 602e40d51..4b796d79e 100644 --- a/compiler/rustc_mir_transform/src/errors.rs +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -1,5 +1,6 @@ use rustc_errors::{ - DecorateLint, DiagnosticBuilder, DiagnosticMessage, EmissionGuarantee, Handler, IntoDiagnostic, + Applicability, DecorateLint, DiagnosticBuilder, DiagnosticMessage, EmissionGuarantee, Handler, + IntoDiagnostic, }; use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; use rustc_middle::mir::{AssertKind, UnsafetyViolationDetails}; @@ -130,6 +131,12 @@ impl RequiresUnsafeDetail { pub(crate) struct UnsafeOpInUnsafeFn { pub details: RequiresUnsafeDetail, + + /// These spans point to: + /// 1. the start of the function body + /// 2. the end of the function body + /// 3. the function signature + pub suggest_unsafe_block: Option<(Span, Span, Span)>, } impl<'a> DecorateLint<'a, ()> for UnsafeOpInUnsafeFn { @@ -138,13 +145,21 @@ impl<'a> DecorateLint<'a, ()> for UnsafeOpInUnsafeFn { self, diag: &'b mut DiagnosticBuilder<'a, ()>, ) -> &'b mut DiagnosticBuilder<'a, ()> { - let desc = diag - .handler() - .expect("lint should not yet be emitted") - .eagerly_translate_to_string(self.details.label(), [].into_iter()); + let handler = diag.handler().expect("lint should not yet be emitted"); + let desc = handler.eagerly_translate_to_string(self.details.label(), [].into_iter()); diag.set_arg("details", desc); diag.span_label(self.details.span, self.details.label()); diag.note(self.details.note()); + + if let Some((start, end, fn_sig)) = self.suggest_unsafe_block { + diag.span_note(fn_sig, crate::fluent_generated::mir_transform_note); + diag.tool_only_multipart_suggestion( + crate::fluent_generated::mir_transform_suggestion, + vec![(start, " unsafe {".into()), (end, "}".into())], + Applicability::MaybeIncorrect, + ); + } + diag } @@ -163,7 +178,14 @@ impl<'a, P: std::fmt::Debug> DecorateLint<'a, ()> for AssertLint<P> { self, diag: &'b mut DiagnosticBuilder<'a, ()>, ) -> &'b mut DiagnosticBuilder<'a, ()> { - diag.span_label(self.span(), format!("{:?}", self.panic())); + let span = self.span(); + let assert_kind = self.panic(); + let message = assert_kind.diagnostic_message(); + assert_kind.add_args(&mut |name, value| { + diag.set_arg(name, value); + }); + diag.span_label(span, message); + diag } @@ -191,7 +213,7 @@ impl<P> AssertLint<P> { AssertLint::ArithmeticOverflow(sp, _) | AssertLint::UnconditionalPanic(sp, _) => *sp, } } - pub fn panic(&self) -> &AssertKind<P> { + pub fn panic(self) -> AssertKind<P> { match self { AssertLint::ArithmeticOverflow(_, p) | AssertLint::UnconditionalPanic(_, p) => p, } diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs index 5989dbebf..0b41e57be 100644 --- a/compiler/rustc_mir_transform/src/function_item_references.rs +++ b/compiler/rustc_mir_transform/src/function_item_references.rs @@ -2,7 +2,7 @@ use itertools::Itertools; use rustc_hir::def_id::DefId; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; -use rustc_middle::ty::{self, EarlyBinder, PredicateKind, SubstsRef, Ty, TyCtxt}; +use rustc_middle::ty::{self, EarlyBinder, SubstsRef, Ty, TyCtxt}; use rustc_session::lint::builtin::FUNCTION_ITEM_REFERENCES; use rustc_span::{symbol::sym, Span}; use rustc_target::spec::abi::Abi; @@ -34,7 +34,7 @@ impl<'tcx> Visitor<'tcx> for FunctionItemRefChecker<'_, 'tcx> { destination: _, target: _, unwind: _, - from_hir_call: _, + call_source: _, fn_span: _, } = &terminator.kind { @@ -74,7 +74,7 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { let param_env = self.tcx.param_env(def_id); let bounds = param_env.caller_bounds(); for bound in bounds { - if let Some(bound_ty) = self.is_pointer_trait(&bound.kind().skip_binder()) { + if let Some(bound_ty) = self.is_pointer_trait(bound) { // Get the argument types as they appear in the function signature. let arg_defs = self.tcx.fn_sig(def_id).subst_identity().skip_binder().inputs(); for (arg_num, arg_def) in arg_defs.iter().enumerate() { @@ -83,7 +83,7 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { // If the inner type matches the type bound by `Pointer` if inner_ty == bound_ty { // Do a substitution using the parameters from the callsite - let subst_ty = EarlyBinder(inner_ty).subst(self.tcx, substs_ref); + let subst_ty = EarlyBinder::bind(inner_ty).subst(self.tcx, substs_ref); if let Some((fn_id, fn_substs)) = FunctionItemRefChecker::is_fn_ref(subst_ty) { @@ -104,8 +104,8 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { } /// If the given predicate is the trait `fmt::Pointer`, returns the bound parameter type. - fn is_pointer_trait(&self, bound: &PredicateKind<'tcx>) -> Option<Ty<'tcx>> { - if let ty::PredicateKind::Clause(ty::Clause::Trait(predicate)) = bound { + fn is_pointer_trait(&self, bound: ty::Clause<'tcx>) -> Option<Ty<'tcx>> { + if let ty::ClauseKind::Trait(predicate) = bound.kind().skip_binder() { self.tcx .is_diagnostic_item(sym::Pointer, predicate.def_id()) .then(|| predicate.trait_ref.self_ty()) diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index 89567ed0a..264bc61f1 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -230,7 +230,7 @@ struct TransformVisitor<'tcx> { // Mapping from Local to (type of local, generator struct index) // FIXME(eddyb) This should use `IndexVec<Local, Option<_>>`. - remap: FxHashMap<Local, (Ty<'tcx>, VariantIdx, usize)>, + remap: FxHashMap<Local, (Ty<'tcx>, VariantIdx, FieldIdx)>, // A map from a suspension point in a block to the locals which have live storage at that point storage_liveness: IndexVec<BasicBlock, Option<BitSet<Local>>>, @@ -295,11 +295,11 @@ impl<'tcx> TransformVisitor<'tcx> { } // Create a Place referencing a generator struct field - fn make_field(&self, variant_index: VariantIdx, idx: usize, ty: Ty<'tcx>) -> Place<'tcx> { + fn make_field(&self, variant_index: VariantIdx, idx: FieldIdx, ty: Ty<'tcx>) -> Place<'tcx> { let self_place = Place::from(SELF_ARG); let base = self.tcx.mk_place_downcast_unnamed(self_place, variant_index); let mut projection = base.projection.to_vec(); - projection.push(ProjectionElem::Field(FieldIdx::new(idx), ty)); + projection.push(ProjectionElem::Field(idx, ty)); Place { local: base.local, projection: self.tcx.mk_place_elems(&projection) } } @@ -413,8 +413,11 @@ impl<'tcx> MutVisitor<'tcx> for TransformVisitor<'tcx> { fn make_generator_state_argument_indirect<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let gen_ty = body.local_decls.raw[1].ty; - let ref_gen_ty = - tcx.mk_ref(tcx.lifetimes.re_erased, ty::TypeAndMut { ty: gen_ty, mutbl: Mutability::Mut }); + let ref_gen_ty = Ty::new_ref( + tcx, + tcx.lifetimes.re_erased, + ty::TypeAndMut { ty: gen_ty, mutbl: Mutability::Mut }, + ); // Replace the by value generator argument body.local_decls.raw[1].ty = ref_gen_ty; @@ -429,7 +432,7 @@ fn make_generator_state_argument_pinned<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body let pin_did = tcx.require_lang_item(LangItem::Pin, Some(body.span)); let pin_adt_ref = tcx.adt_def(pin_did); let substs = tcx.mk_substs(&[ref_gen_ty.into()]); - let pin_ref_gen_ty = tcx.mk_adt(pin_adt_ref, substs); + let pin_ref_gen_ty = Ty::new_adt(tcx, pin_adt_ref, substs); // Replace the by ref generator argument body.local_decls.raw[1].ty = pin_ref_gen_ty; @@ -481,7 +484,7 @@ fn replace_local<'tcx>( /// still using the `ResumeTy` indirection for the time being, and that indirection /// is removed here. After this transform, the generator body only knows about `&mut Context<'_>`. fn transform_async_context<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let context_mut_ref = tcx.mk_task_context(); + let context_mut_ref = Ty::new_task_context(tcx); // replace the type of the `resume` argument replace_resume_ty_local(tcx, body, Local::new(2), context_mut_ref); @@ -597,16 +600,15 @@ fn locals_live_across_suspend_points<'tcx>( let borrowed_locals_results = MaybeBorrowedLocals.into_engine(tcx, body_ref).pass_name("generator").iterate_to_fixpoint(); - let mut borrowed_locals_cursor = - rustc_mir_dataflow::ResultsCursor::new(body_ref, &borrowed_locals_results); + let mut borrowed_locals_cursor = borrowed_locals_results.cloned_results_cursor(body_ref); // Calculate the MIR locals that we actually need to keep storage around // for. - let requires_storage_results = MaybeRequiresStorage::new(body, &borrowed_locals_results) - .into_engine(tcx, body_ref) - .iterate_to_fixpoint(); - let mut requires_storage_cursor = - rustc_mir_dataflow::ResultsCursor::new(body_ref, &requires_storage_results); + let mut requires_storage_results = + MaybeRequiresStorage::new(borrowed_locals_results.cloned_results_cursor(body)) + .into_engine(tcx, body_ref) + .iterate_to_fixpoint(); + let mut requires_storage_cursor = requires_storage_results.as_results_cursor(body_ref); // Calculate the liveness of MIR locals ignoring borrows. let mut liveness = MaybeLiveLocals @@ -747,7 +749,7 @@ fn compute_storage_conflicts<'mir, 'tcx>( body: &'mir Body<'tcx>, saved_locals: &GeneratorSavedLocals, always_live_locals: BitSet<Local>, - requires_storage: rustc_mir_dataflow::Results<'tcx, MaybeRequiresStorage<'mir, 'tcx>>, + mut requires_storage: rustc_mir_dataflow::Results<'tcx, MaybeRequiresStorage<'_, 'mir, 'tcx>>, ) -> BitMatrix<GeneratorSavedLocal, GeneratorSavedLocal> { assert_eq!(body.local_decls.len(), saved_locals.domain_size()); @@ -802,13 +804,14 @@ struct StorageConflictVisitor<'mir, 'tcx, 's> { local_conflicts: BitMatrix<Local, Local>, } -impl<'mir, 'tcx> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx> +impl<'mir, 'tcx, R> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx, R> for StorageConflictVisitor<'mir, 'tcx, '_> { type FlowState = BitSet<Local>; fn visit_statement_before_primary_effect( &mut self, + _results: &R, state: &Self::FlowState, _statement: &'mir Statement<'tcx>, loc: Location, @@ -818,6 +821,7 @@ impl<'mir, 'tcx> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx> fn visit_terminator_before_primary_effect( &mut self, + _results: &R, state: &Self::FlowState, _terminator: &'mir Terminator<'tcx>, loc: Location, @@ -903,7 +907,7 @@ fn compute_layout<'tcx>( liveness: LivenessInfo, body: &Body<'tcx>, ) -> ( - FxHashMap<Local, (Ty<'tcx>, VariantIdx, usize)>, + FxHashMap<Local, (Ty<'tcx>, VariantIdx, FieldIdx)>, GeneratorLayout<'tcx>, IndexVec<BasicBlock, Option<BitSet<Local>>>, ) { @@ -981,6 +985,7 @@ fn compute_layout<'tcx>( // just use the first one here. That's fine; fields do not move // around inside generators, so it doesn't matter which variant // index we access them by. + let idx = FieldIdx::from_usize(idx); remap.entry(locals[saved_local]).or_insert((tys[saved_local].ty, variant_index, idx)); } variant_fields.push(fields); @@ -989,8 +994,23 @@ fn compute_layout<'tcx>( debug!("generator variant_fields = {:?}", variant_fields); debug!("generator storage_conflicts = {:#?}", storage_conflicts); - let layout = - GeneratorLayout { field_tys: tys, variant_fields, variant_source_info, storage_conflicts }; + let mut field_names = IndexVec::from_elem(None, &tys); + for var in &body.var_debug_info { + let VarDebugInfoContents::Place(place) = &var.value else { continue }; + let Some(local) = place.as_local() else { continue }; + let Some(&(_, variant, field)) = remap.get(&local) else { continue }; + + let saved_local = variant_fields[variant][field]; + field_names.get_or_insert_with(saved_local, || var.name); + } + + let layout = GeneratorLayout { + field_tys: tys, + field_names, + variant_fields, + variant_source_info, + storage_conflicts, + }; debug!(?layout); (remap, layout, storage_liveness) @@ -1113,13 +1133,13 @@ fn create_generator_drop_shim<'tcx>( } // Replace the return variable - body.local_decls[RETURN_PLACE] = LocalDecl::with_source_info(tcx.mk_unit(), source_info); + body.local_decls[RETURN_PLACE] = LocalDecl::with_source_info(Ty::new_unit(tcx), source_info); make_generator_state_argument_indirect(tcx, &mut body); // Change the generator argument from &mut to *mut body.local_decls[SELF_ARG] = LocalDecl::with_source_info( - tcx.mk_ptr(ty::TypeAndMut { ty: gen_ty, mutbl: hir::Mutability::Mut }), + Ty::new_ptr(tcx, ty::TypeAndMut { ty: gen_ty, mutbl: hir::Mutability::Mut }), source_info, ); @@ -1476,7 +1496,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform { let state_substs = tcx.mk_substs(&[yield_ty.into(), body.return_ty().into()]); (state_adt_ref, state_substs) }; - let ret_ty = tcx.mk_adt(state_adt_ref, state_substs); + let ret_ty = Ty::new_adt(tcx, state_adt_ref, state_substs); // We rename RETURN_PLACE which has type mir.return_ty to new_ret_local // RETURN_PLACE then is a fresh unused local with type ret_ty. @@ -1492,8 +1512,11 @@ impl<'tcx> MirPass<'tcx> for StateTransform { // case there is no `Assign` to it that the transform can turn into a store to the generator // state. After the yield the slot in the generator state would then be uninitialized. let resume_local = Local::new(2); - let resume_ty = - if is_async_kind { tcx.mk_task_context() } else { body.local_decls[resume_local].ty }; + let resume_ty = if is_async_kind { + Ty::new_task_context(tcx) + } else { + body.local_decls[resume_local].ty + }; let new_resume_local = replace_local(resume_local, resume_ty, body, tcx); // When first entering the generator, move the resume argument into its new local. @@ -1691,7 +1714,7 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> { destination, target: Some(_), unwind: _, - from_hir_call: _, + call_source: _, fn_span: _, } => { self.check_assigned_place(*destination, |this| { @@ -1807,7 +1830,7 @@ fn check_must_not_suspend_ty<'tcx>( let mut has_emitted = false; for &(predicate, _) in tcx.explicit_item_bounds(def).skip_binder() { // We only look at the `DefId`, so it is safe to skip the binder here. - if let ty::PredicateKind::Clause(ty::Clause::Trait(ref poly_trait_predicate)) = + if let ty::ClauseKind::Trait(ref poly_trait_predicate) = predicate.kind().skip_binder() { let def_id = poly_trait_predicate.trait_ref.def_id; diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 233de6dc6..b6578cb25 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -145,13 +145,16 @@ impl<'tcx> Inliner<'tcx> { Ok(new_blocks) => { debug!("inlined {}", callsite.callee); self.changed = true; + + self.history.push(callsite.callee.def_id()); + self.process_blocks(caller_body, new_blocks); + self.history.pop(); + inlined_count += 1; if inlined_count == inline_limit { + debug!("inline count reached"); return; } - self.history.push(callsite.callee.def_id()); - self.process_blocks(caller_body, new_blocks); - self.history.pop(); } } } @@ -192,7 +195,7 @@ impl<'tcx> Inliner<'tcx> { let Ok(callee_body) = callsite.callee.try_subst_mir_and_normalize_erasing_regions( self.tcx, self.param_env, - ty::EarlyBinder(callee_body.clone()), + ty::EarlyBinder::bind(callee_body.clone()), ) else { return Err("failed to normalize callee body"); }; @@ -459,7 +462,7 @@ impl<'tcx> Inliner<'tcx> { // If the place doesn't actually need dropping, treat it like a regular goto. let ty = callsite .callee - .subst_mir(self.tcx, ty::EarlyBinder(&place.ty(callee_body, tcx).ty)); + .subst_mir(self.tcx, ty::EarlyBinder::bind(&place.ty(callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) && let UnwindAction::Cleanup(unwind) = unwind { work_list.push(unwind); } @@ -480,11 +483,12 @@ impl<'tcx> Inliner<'tcx> { // Abort if type validation found anything fishy. checker.validation?; + // N.B. We still apply our cost threshold to #[inline(always)] functions. + // That attribute is often applied to very large functions that exceed LLVM's (very + // generous) inlining threshold. Such functions are very poor MIR inlining candidates. + // Always inlining #[inline(always)] functions in MIR, on net, slows down the compiler. let cost = checker.cost; - if let InlineAttr::Always = callee_attrs.inline { - debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost); - Ok(()) - } else if cost <= threshold { + if cost <= threshold { debug!("INLINING {:?} [cost={} <= threshold={}]", callsite, cost, threshold); Ok(()) } else { @@ -522,7 +526,7 @@ impl<'tcx> Inliner<'tcx> { trace!("creating temp for return destination"); let dest = Rvalue::Ref( self.tcx.lifetimes.re_erased, - BorrowKind::Mut { allow_two_phase_borrow: false }, + BorrowKind::Mut { kind: MutBorrowKind::Default }, destination, ); let dest_ty = dest.ty(caller_body, self.tcx); @@ -794,7 +798,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { // If the place doesn't actually need dropping, treat it like a regular goto. let ty = self .instance - .subst_mir(tcx, ty::EarlyBinder(&place.ty(self.callee_body, tcx).ty)); + .subst_mir(tcx, ty::EarlyBinder::bind(&place.ty(self.callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) { self.cost += CALL_PENALTY; if let UnwindAction::Cleanup(_) = unwind { @@ -805,7 +809,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { - let fn_ty = self.instance.subst_mir(tcx, ty::EarlyBinder(&f.literal.ty())); + let fn_ty = self.instance.subst_mir(tcx, ty::EarlyBinder::bind(&f.literal.ty())); self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { // Don't give intrinsics the extra penalty for calls INSTR_COST @@ -839,22 +843,20 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { /// to normalization failure. fn visit_projection_elem( &mut self, - local: Local, - proj_base: &[PlaceElem<'tcx>], + place_ref: PlaceRef<'tcx>, elem: PlaceElem<'tcx>, context: PlaceContext, location: Location, ) { if let ProjectionElem::Field(f, ty) = elem { - let parent = Place { local, projection: self.tcx.mk_place_elems(proj_base) }; - let parent_ty = parent.ty(&self.callee_body.local_decls, self.tcx); + let parent_ty = place_ref.ty(&self.callee_body.local_decls, self.tcx); let check_equal = |this: &mut Self, f_ty| { // Fast path if there is nothing to substitute. if ty == f_ty { return; } - let ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder(&ty)); - let f_ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder(&f_ty)); + let ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder::bind(&ty)); + let f_ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder::bind(&f_ty)); if ty == f_ty { return; } @@ -935,7 +937,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } - self.super_projection_elem(local, proj_base, elem, context, location); + self.super_projection_elem(place_ref, elem, context, location); } } diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 1ccf06f61..8a10445f8 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -47,7 +47,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( let Ok(substs) = caller.try_subst_mir_and_normalize_erasing_regions( tcx, param_env, - ty::EarlyBinder(substs), + ty::EarlyBinder::bind(substs), ) else { trace!(?caller, ?param_env, ?substs, "cannot normalize, skipping"); continue; diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs index 430a6f6ce..8ed4706e1 100644 --- a/compiler/rustc_mir_transform/src/large_enums.rs +++ b/compiler/rustc_mir_transform/src/large_enums.rs @@ -141,7 +141,7 @@ impl EnumSizeOpt { self.candidate(tcx, param_env, ty, &mut alloc_cache)?; let alloc = tcx.global_alloc(alloc_id).unwrap_memory(); - let tmp_ty = tcx.mk_array(tcx.types.usize, num_variants as u64); + let tmp_ty = Ty::new_array(tcx, tcx.types.usize, num_variants as u64); let size_array_local = local_decls.push(LocalDecl::new(tmp_ty, span)); let store_live = Statement { @@ -208,8 +208,9 @@ impl EnumSizeOpt { ))), }; - let dst = - Place::from(local_decls.push(LocalDecl::new(tcx.mk_mut_ptr(ty), span))); + let dst = Place::from( + local_decls.push(LocalDecl::new(Ty::new_mut_ptr(tcx, ty), span)), + ); let dst_ptr = Statement { source_info, @@ -219,7 +220,7 @@ impl EnumSizeOpt { ))), }; - let dst_cast_ty = tcx.mk_mut_ptr(tcx.types.u8); + let dst_cast_ty = Ty::new_mut_ptr(tcx, tcx.types.u8); let dst_cast_place = Place::from(local_decls.push(LocalDecl::new(dst_cast_ty, span))); @@ -231,8 +232,9 @@ impl EnumSizeOpt { ))), }; - let src = - Place::from(local_decls.push(LocalDecl::new(tcx.mk_imm_ptr(ty), span))); + let src = Place::from( + local_decls.push(LocalDecl::new(Ty::new_imm_ptr(tcx, ty), span)), + ); let src_ptr = Statement { source_info, @@ -242,7 +244,7 @@ impl EnumSizeOpt { ))), }; - let src_cast_ty = tcx.mk_imm_ptr(tcx.types.u8); + let src_cast_ty = Ty::new_imm_ptr(tcx, tcx.types.u8); let src_cast_place = Place::from(local_decls.push(LocalDecl::new(src_cast_ty, span))); diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index 65864dc01..fa8257cf9 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -2,7 +2,7 @@ #![deny(rustc::untranslatable_diagnostic)] #![deny(rustc::diagnostic_outside_of_impl)] #![feature(box_patterns)] -#![feature(drain_filter)] +#![feature(is_sorted)] #![feature(let_chains)] #![feature(map_try_insert)] #![feature(min_specialization)] @@ -30,8 +30,8 @@ use rustc_hir::intravisit::{self, Visitor}; use rustc_index::IndexVec; use rustc_middle::mir::visit::Visitor as _; use rustc_middle::mir::{ - traversal, AnalysisPhase, Body, ClearCrossCrate, ConstQualifs, Constant, LocalDecl, MirPass, - MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, + traversal, AnalysisPhase, Body, CallSource, ClearCrossCrate, ConstQualifs, Constant, LocalDecl, + MirPass, MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, Statement, StatementKind, TerminatorKind, START_BLOCK, }; use rustc_middle::query::Providers; @@ -84,6 +84,7 @@ mod match_branches; mod multiple_return_terminators; mod normalize_array_len; mod nrvo; +mod prettify; mod ref_prop; mod remove_noop_landing_pads; mod remove_storage_markers; @@ -188,7 +189,7 @@ fn remap_mir_for_const_eval_select<'tcx>( }; method(place) }).collect(); - terminator.kind = TerminatorKind::Call { func, args: arguments, destination, target, unwind, from_hir_call: false, fn_span }; + terminator.kind = TerminatorKind::Call { func, args: arguments, destination, target, unwind, call_source: CallSource::Misc, fn_span }; } _ => {} } @@ -557,10 +558,13 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { // inst combine is after MatchBranchSimplification to clean up Ne(_1, false) &multiple_return_terminators::MultipleReturnTerminators, &instsimplify::InstSimplify, - &separate_const_switch::SeparateConstSwitch, &simplify::SimplifyLocals::BeforeConstProp, ©_prop::CopyProp, &ref_prop::ReferencePropagation, + // Perform `SeparateConstSwitch` after SSA-based analyses, as cloning blocks may + // destroy the SSA property. It should still happen before const-propagation, so the + // latter pass will leverage the created opportunities. + &separate_const_switch::SeparateConstSwitch, &const_prop::ConstProp, &dataflow_const_prop::DataflowConstProp, // @@ -581,6 +585,9 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &large_enums::EnumSizeOpt { discrepancy: 128 }, // Some cleanup necessary at least for LLVM and potentially other codegen backends. &add_call_guards::CriticalCallEdges, + // Cleanup for human readability, off by default. + &prettify::ReorderBasicBlocks, + &prettify::ReorderLocals, // Dump the end result for testing and debugging purposes. &dump_mir::Marker("PreCodegen"), ], diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index dae01e41e..ce98e9b0c 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -82,30 +82,45 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { drop(args); terminator.kind = TerminatorKind::Goto { target }; } - sym::wrapping_add | sym::wrapping_sub | sym::wrapping_mul => { - if let Some(target) = *target { - let lhs; - let rhs; - { - let mut args = args.drain(..); - lhs = args.next().unwrap(); - rhs = args.next().unwrap(); - } - let bin_op = match intrinsic_name { - sym::wrapping_add => BinOp::Add, - sym::wrapping_sub => BinOp::Sub, - sym::wrapping_mul => BinOp::Mul, - _ => bug!("unexpected intrinsic"), - }; - block.statements.push(Statement { - source_info: terminator.source_info, - kind: StatementKind::Assign(Box::new(( - *destination, - Rvalue::BinaryOp(bin_op, Box::new((lhs, rhs))), - ))), - }); - terminator.kind = TerminatorKind::Goto { target }; + sym::wrapping_add + | sym::wrapping_sub + | sym::wrapping_mul + | sym::unchecked_add + | sym::unchecked_sub + | sym::unchecked_mul + | sym::unchecked_div + | sym::unchecked_rem + | sym::unchecked_shl + | sym::unchecked_shr => { + let target = target.unwrap(); + let lhs; + let rhs; + { + let mut args = args.drain(..); + lhs = args.next().unwrap(); + rhs = args.next().unwrap(); } + let bin_op = match intrinsic_name { + sym::wrapping_add => BinOp::Add, + sym::wrapping_sub => BinOp::Sub, + sym::wrapping_mul => BinOp::Mul, + sym::unchecked_add => BinOp::AddUnchecked, + sym::unchecked_sub => BinOp::SubUnchecked, + sym::unchecked_mul => BinOp::MulUnchecked, + sym::unchecked_div => BinOp::Div, + sym::unchecked_rem => BinOp::Rem, + sym::unchecked_shl => BinOp::ShlUnchecked, + sym::unchecked_shr => BinOp::ShrUnchecked, + _ => bug!("unexpected intrinsic"), + }; + block.statements.push(Statement { + source_info: terminator.source_info, + kind: StatementKind::Assign(Box::new(( + *destination, + Rvalue::BinaryOp(bin_op, Box::new((lhs, rhs))), + ))), + }); + terminator.kind = TerminatorKind::Goto { target }; } sym::add_with_overflow | sym::sub_with_overflow | sym::mul_with_overflow => { if let Some(target) = *target { diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs index 6e40dfa0d..b7cc0db95 100644 --- a/compiler/rustc_mir_transform/src/lower_slice_len.rs +++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs @@ -54,7 +54,7 @@ fn lower_slice_len_call<'tcx>( args, destination, target: Some(bb), - from_hir_call: true, + call_source: CallSource::Normal, .. } => { // some heuristics for fast rejection diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index 59942dc76..6eb484982 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -41,7 +41,7 @@ pub struct MatchBranchSimplification; impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 3 + sess.mir_opt_level() >= 1 } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -62,7 +62,12 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { .. } if targets.iter().len() == 1 => { let (value, target) = targets.iter().next().unwrap(); - if target == targets.otherwise() { + // We require that this block and the two possible target blocks all be + // distinct. + if target == targets.otherwise() + || bb_idx == target + || bb_idx == targets.otherwise() + { continue; } (discr, value, target, targets.otherwise()) diff --git a/compiler/rustc_mir_transform/src/normalize_array_len.rs b/compiler/rustc_mir_transform/src/normalize_array_len.rs index 3d61d33ce..6c3b7c58f 100644 --- a/compiler/rustc_mir_transform/src/normalize_array_len.rs +++ b/compiler/rustc_mir_transform/src/normalize_array_len.rs @@ -41,7 +41,7 @@ fn compute_slice_length<'tcx>( for (local, rvalue, _) in ssa.assignments(body) { match rvalue { Rvalue::Cast( - CastKind::Pointer(ty::adjustment::PointerCast::Unsize), + CastKind::PointerCoercion(ty::adjustment::PointerCoercion::Unsize), operand, cast_ty, ) => { diff --git a/compiler/rustc_mir_transform/src/prettify.rs b/compiler/rustc_mir_transform/src/prettify.rs new file mode 100644 index 000000000..745fa3084 --- /dev/null +++ b/compiler/rustc_mir_transform/src/prettify.rs @@ -0,0 +1,150 @@ +//! These two passes provide no value to the compiler, so are off at every level. +//! +//! However, they can be enabled on the command line +//! (`-Zmir-enable-passes=+ReorderBasicBlocks,+ReorderLocals`) +//! to make the MIR easier to read for humans. + +use crate::MirPass; +use rustc_index::{bit_set::BitSet, IndexSlice, IndexVec}; +use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; +use rustc_middle::mir::*; +use rustc_middle::ty::TyCtxt; +use rustc_session::Session; + +/// Rearranges the basic blocks into a *reverse post-order*. +/// +/// Thus after this pass, all the successors of a block are later than it in the +/// `IndexVec`, unless that successor is a back-edge (such as from a loop). +pub struct ReorderBasicBlocks; + +impl<'tcx> MirPass<'tcx> for ReorderBasicBlocks { + fn is_enabled(&self, _session: &Session) -> bool { + false + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let rpo: IndexVec<BasicBlock, BasicBlock> = + body.basic_blocks.reverse_postorder().iter().copied().collect(); + if rpo.iter().is_sorted() { + return; + } + + let mut updater = BasicBlockUpdater { map: rpo.invert_bijective_mapping(), tcx }; + debug_assert_eq!(updater.map[START_BLOCK], START_BLOCK); + updater.visit_body(body); + + permute(body.basic_blocks.as_mut(), &updater.map); + } +} + +/// Rearranges the locals into *use* order. +/// +/// Thus after this pass, a local with a smaller [`Location`] where it was first +/// assigned or referenced will have a smaller number. +/// +/// (Does not reorder arguments nor the [`RETURN_PLACE`].) +pub struct ReorderLocals; + +impl<'tcx> MirPass<'tcx> for ReorderLocals { + fn is_enabled(&self, _session: &Session) -> bool { + false + } + + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let mut finder = + LocalFinder { map: IndexVec::new(), seen: BitSet::new_empty(body.local_decls.len()) }; + + // We can't reorder the return place or the arguments + for local in (0..=body.arg_count).map(Local::from_usize) { + finder.track(local); + } + + for (bb, bbd) in body.basic_blocks.iter_enumerated() { + finder.visit_basic_block_data(bb, bbd); + } + + // track everything in case there are some locals that we never saw, + // such as in non-block things like debug info or in non-uses. + for local in body.local_decls.indices() { + finder.track(local); + } + + if finder.map.iter().is_sorted() { + return; + } + + let mut updater = LocalUpdater { map: finder.map.invert_bijective_mapping(), tcx }; + + for local in (0..=body.arg_count).map(Local::from_usize) { + debug_assert_eq!(updater.map[local], local); + } + + updater.visit_body_preserves_cfg(body); + + permute(&mut body.local_decls, &updater.map); + } +} + +fn permute<I: rustc_index::Idx + Ord, T>(data: &mut IndexVec<I, T>, map: &IndexSlice<I, I>) { + // FIXME: It would be nice to have a less-awkward way to apply permutations, + // but I don't know one that exists. `sort_by_cached_key` has logic for it + // internally, but not in a way that we're allowed to use here. + let mut enumerated: Vec<_> = std::mem::take(data).into_iter_enumerated().collect(); + enumerated.sort_by_key(|p| map[p.0]); + *data = enumerated.into_iter().map(|p| p.1).collect(); +} + +struct BasicBlockUpdater<'tcx> { + map: IndexVec<BasicBlock, BasicBlock>, + tcx: TyCtxt<'tcx>, +} + +impl<'tcx> MutVisitor<'tcx> for BasicBlockUpdater<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_terminator(&mut self, terminator: &mut Terminator<'tcx>, _location: Location) { + for succ in terminator.successors_mut() { + *succ = self.map[*succ]; + } + } +} + +struct LocalFinder { + map: IndexVec<Local, Local>, + seen: BitSet<Local>, +} + +impl LocalFinder { + fn track(&mut self, l: Local) { + if self.seen.insert(l) { + self.map.push(l); + } + } +} + +impl<'tcx> Visitor<'tcx> for LocalFinder { + fn visit_local(&mut self, l: Local, context: PlaceContext, _location: Location) { + // Exclude non-uses to keep `StorageLive` from controlling where we put + // a `Local`, since it might not actually be assigned until much later. + if context.is_use() { + self.track(l); + } + } +} + +struct LocalUpdater<'tcx> { + pub map: IndexVec<Local, Local>, + pub tcx: TyCtxt<'tcx>, +} + +impl<'tcx> MutVisitor<'tcx> for LocalUpdater<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_local(&mut self, l: &mut Local, _: PlaceContext, _: Location) { + *l = self.map[*l]; + } +} diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs index 1f9e521d3..283931de0 100644 --- a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs +++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs @@ -22,7 +22,7 @@ pub struct RemoveUninitDrops; impl<'tcx> MirPass<'tcx> for RemoveUninitDrops { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let param_env = tcx.param_env(body.source.def_id()); - let Ok((_,move_data)) = MoveData::gather_moves(body, tcx, param_env) else { + let Ok(move_data) = MoveData::gather_moves(body, tcx, param_env) else { // We could continue if there are move errors, but there's not much point since our // init data isn't complete. return; diff --git a/compiler/rustc_mir_transform/src/required_consts.rs b/compiler/rustc_mir_transform/src/required_consts.rs index 0ea8f2ba9..243cb4635 100644 --- a/compiler/rustc_mir_transform/src/required_consts.rs +++ b/compiler/rustc_mir_transform/src/required_consts.rs @@ -17,8 +17,8 @@ impl<'tcx> Visitor<'tcx> for RequiredConstsVisitor<'_, 'tcx> { let literal = constant.literal; match literal { ConstantKind::Ty(c) => match c.kind() { - ConstKind::Param(_) | ConstKind::Error(_) => {} - _ => bug!("only ConstKind::Param should be encountered here, got {:#?}", c), + ConstKind::Param(_) | ConstKind::Error(_) | ConstKind::Value(_) => {} + _ => bug!("only ConstKind::Param/Value should be encountered here, got {:#?}", c), }, ConstantKind::Unevaluated(..) => self.required_consts.push(*constant), ConstantKind::Val(..) => {} diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs index 2479856b7..1d8e54cdc 100644 --- a/compiler/rustc_mir_transform/src/separate_const_switch.rs +++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs @@ -46,7 +46,9 @@ pub struct SeparateConstSwitch; impl<'tcx> MirPass<'tcx> for SeparateConstSwitch { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() >= 4 + // This pass participates in some as-of-yet untested unsoundness found + // in https://github.com/rust-lang/rust/issues/112460 + sess.mir_opt_level() >= 2 && sess.opts.unstable_opts.unsound_mir_opts } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 0eb27c231..b176db3c9 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -32,13 +32,15 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' let mut result = match instance { ty::InstanceDef::Item(..) => bug!("item {:?} passed to make_shim", instance), ty::InstanceDef::VTableShim(def_id) => { - build_call_shim(tcx, instance, Some(Adjustment::Deref), CallKind::Direct(def_id)) + let adjustment = Adjustment::Deref { source: DerefSource::MutPtr }; + build_call_shim(tcx, instance, Some(adjustment), CallKind::Direct(def_id)) } ty::InstanceDef::FnPtrShim(def_id, ty) => { let trait_ = tcx.trait_of_item(def_id).unwrap(); let adjustment = match tcx.fn_trait_kind_from_def_id(trait_) { Some(ty::ClosureKind::FnOnce) => Adjustment::Identity, - Some(ty::ClosureKind::FnMut | ty::ClosureKind::Fn) => Adjustment::Deref, + Some(ty::ClosureKind::Fn) => Adjustment::Deref { source: DerefSource::ImmRef }, + Some(ty::ClosureKind::FnMut) => Adjustment::Deref { source: DerefSource::MutRef }, None => bug!("fn pointer {:?} is not an fn", ty), }; @@ -69,7 +71,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' // of this function. Is this intentional? if let Some(ty::Generator(gen_def_id, substs, _)) = ty.map(Ty::kind) { let body = tcx.optimized_mir(*gen_def_id).generator_drop().unwrap(); - let body = EarlyBinder(body.clone()).subst(tcx, substs); + let body = EarlyBinder::bind(body.clone()).subst(tcx, substs); debug!("make_shim({:?}) = {:?}", instance, body); return body; } @@ -108,15 +110,25 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' } #[derive(Copy, Clone, Debug, PartialEq)] +enum DerefSource { + /// `fn shim(&self) { inner(*self )}`. + ImmRef, + /// `fn shim(&mut self) { inner(*self )}`. + MutRef, + /// `fn shim(*mut self) { inner(*self )}`. + MutPtr, +} + +#[derive(Copy, Clone, Debug, PartialEq)] enum Adjustment { /// Pass the receiver as-is. Identity, - /// We get passed `&[mut] self` and call the target with `*self`. + /// We get passed a reference or a raw pointer to `self` and call the target with `*self`. /// /// This either copies `self` (if `Self: Copy`, eg. for function items), or moves out of it /// (for `VTableShim`, which effectively is passed `&own Self`). - Deref, + Deref { source: DerefSource }, /// We get passed `self: Self` and call the target with `&mut self`. /// @@ -188,7 +200,7 @@ fn build_drop_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, ty: Option<Ty<'tcx>>) // has been put into the body. let reborrow = Rvalue::Ref( tcx.lifetimes.re_erased, - BorrowKind::Mut { allow_two_phase_borrow: false }, + BorrowKind::Mut { kind: MutBorrowKind::Default }, tcx.mk_place_deref(dropee_ptr), ); let ref_ty = reborrow.ty(body.local_decls(), tcx); @@ -473,7 +485,7 @@ impl<'tcx> CloneShimBuilder<'tcx> { let tcx = self.tcx; // `func == Clone::clone(&ty) -> ty` - let func_ty = tcx.mk_fn_def(self.def_id, [ty]); + let func_ty = Ty::new_fn_def(tcx, self.def_id, [ty]); let func = Operand::Constant(Box::new(Constant { span: self.span, user_ty: None, @@ -482,7 +494,11 @@ impl<'tcx> CloneShimBuilder<'tcx> { let ref_loc = self.make_place( Mutability::Not, - tcx.mk_ref(tcx.lifetimes.re_erased, ty::TypeAndMut { ty, mutbl: hir::Mutability::Not }), + Ty::new_ref( + tcx, + tcx.lifetimes.re_erased, + ty::TypeAndMut { ty, mutbl: hir::Mutability::Not }, + ), ); // `let ref_loc: &ty = &src;` @@ -500,7 +516,7 @@ impl<'tcx> CloneShimBuilder<'tcx> { destination: dest, target: Some(next), unwind: UnwindAction::Cleanup(cleanup), - from_hir_call: true, + call_source: CallSource::Normal, fn_span: self.span, }, false, @@ -632,7 +648,7 @@ fn build_call_shim<'tcx>( let untuple_args = sig.inputs(); // Create substitutions for the `Self` and `Args` generic parameters of the shim body. - let arg_tup = tcx.mk_tup(untuple_args); + let arg_tup = Ty::new_tup(tcx, untuple_args); (Some([ty.into(), arg_tup.into()]), Some(untuple_args)) } else { @@ -644,8 +660,11 @@ fn build_call_shim<'tcx>( let sig = sig.map_bound(|sig| tcx.erase_late_bound_regions(sig)); assert_eq!(sig_substs.is_some(), !instance.has_polymorphic_mir_body()); - let mut sig = - if let Some(sig_substs) = sig_substs { sig.subst(tcx, &sig_substs) } else { sig.0 }; + let mut sig = if let Some(sig_substs) = sig_substs { + sig.subst(tcx, &sig_substs) + } else { + sig.subst_identity() + }; if let CallKind::Indirect(fnty) = call_kind { // `sig` determines our local decls, and thus the callee type in the `Call` terminator. This @@ -664,8 +683,12 @@ fn build_call_shim<'tcx>( let self_arg = &mut inputs_and_output[0]; *self_arg = match rcvr_adjustment.unwrap() { Adjustment::Identity => fnty, - Adjustment::Deref => tcx.mk_imm_ptr(fnty), - Adjustment::RefMut => tcx.mk_mut_ptr(fnty), + Adjustment::Deref { source } => match source { + DerefSource::ImmRef => Ty::new_imm_ref(tcx, tcx.lifetimes.re_erased, fnty), + DerefSource::MutRef => Ty::new_mut_ref(tcx, tcx.lifetimes.re_erased, fnty), + DerefSource::MutPtr => Ty::new_mut_ptr(tcx, fnty), + }, + Adjustment::RefMut => bug!("`RefMut` is never used with indirect calls: {instance:?}"), }; sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output); } @@ -677,7 +700,7 @@ fn build_call_shim<'tcx>( let mut inputs_and_output = sig.inputs_and_output.to_vec(); let self_arg = &mut inputs_and_output[0]; debug_assert!(tcx.generics_of(def_id).has_self && *self_arg == tcx.types.self_param); - *self_arg = tcx.mk_mut_ptr(*self_arg); + *self_arg = Ty::new_mut_ptr(tcx, *self_arg); sig.inputs_and_output = tcx.mk_type_list(&inputs_and_output); } @@ -696,12 +719,13 @@ fn build_call_shim<'tcx>( let rcvr = rcvr_adjustment.map(|rcvr_adjustment| match rcvr_adjustment { Adjustment::Identity => Operand::Move(rcvr_place()), - Adjustment::Deref => Operand::Move(tcx.mk_place_deref(rcvr_place())), + Adjustment::Deref { source: _ } => Operand::Move(tcx.mk_place_deref(rcvr_place())), Adjustment::RefMut => { // let rcvr = &mut rcvr; let ref_rcvr = local_decls.push( LocalDecl::new( - tcx.mk_ref( + Ty::new_ref( + tcx, tcx.lifetimes.re_erased, ty::TypeAndMut { ty: sig.inputs()[0], mutbl: hir::Mutability::Mut }, ), @@ -709,7 +733,7 @@ fn build_call_shim<'tcx>( ) .immutable(), ); - let borrow_kind = BorrowKind::Mut { allow_two_phase_borrow: false }; + let borrow_kind = BorrowKind::Mut { kind: MutBorrowKind::Default }; statements.push(Statement { source_info, kind: StatementKind::Assign(Box::new(( @@ -786,7 +810,7 @@ fn build_call_shim<'tcx>( } else { UnwindAction::Continue }, - from_hir_call: true, + call_source: CallSource::Misc, fn_span: span, }, false, @@ -927,7 +951,7 @@ fn build_fn_ptr_addr_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'t let rvalue = Rvalue::Cast( CastKind::FnPtrToPtr, Operand::Move(Place::from(Local::new(1))), - tcx.mk_imm_ptr(tcx.types.unit), + Ty::new_imm_ptr(tcx, tcx.types.unit), ); let stmt = Statement { source_info, diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index 2d7729129..94e1da8e1 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -6,7 +6,7 @@ use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_mir_dataflow::value_analysis::{excluded_locals, iter_fields}; -use rustc_target::abi::FieldIdx; +use rustc_target::abi::{FieldIdx, ReprFlags, FIRST_VARIANT}; pub struct ScalarReplacementOfAggregates; @@ -18,11 +18,17 @@ impl<'tcx> MirPass<'tcx> for ScalarReplacementOfAggregates { #[instrument(level = "debug", skip(self, tcx, body))] fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { debug!(def_id = ?body.source.def_id()); + + // Avoid query cycles (generators require optimized MIR for layout). + if tcx.type_of(body.source.def_id()).subst_identity().is_generator() { + return; + } + let mut excluded = excluded_locals(body); let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); loop { debug!(?excluded); - let escaping = escaping_locals(&excluded, body); + let escaping = escaping_locals(tcx, param_env, &excluded, body); debug!(?escaping); let replacements = compute_flattening(tcx, param_env, body, escaping); debug!(?replacements); @@ -48,11 +54,45 @@ impl<'tcx> MirPass<'tcx> for ScalarReplacementOfAggregates { /// - the locals is a union or an enum; /// - the local's address is taken, and thus the relative addresses of the fields are observable to /// client code. -fn escaping_locals(excluded: &BitSet<Local>, body: &Body<'_>) -> BitSet<Local> { +fn escaping_locals<'tcx>( + tcx: TyCtxt<'tcx>, + param_env: ty::ParamEnv<'tcx>, + excluded: &BitSet<Local>, + body: &Body<'tcx>, +) -> BitSet<Local> { + let is_excluded_ty = |ty: Ty<'tcx>| { + if ty.is_union() || ty.is_enum() { + return true; + } + if let ty::Adt(def, _substs) = ty.kind() { + if def.repr().flags.contains(ReprFlags::IS_SIMD) { + // Exclude #[repr(simd)] types so that they are not de-optimized into an array + return true; + } + // We already excluded unions and enums, so this ADT must have one variant + let variant = def.variant(FIRST_VARIANT); + if variant.fields.len() > 1 { + // If this has more than one field, it cannot be a wrapper that only provides a + // niche, so we do not want to automatically exclude it. + return false; + } + let Ok(layout) = tcx.layout_of(param_env.and(ty)) else { + // We can't get the layout + return true; + }; + if layout.layout.largest_niche().is_some() { + // This type has a niche + return true; + } + } + // Default for non-ADTs + false + }; + let mut set = BitSet::new_empty(body.local_decls.len()); set.insert_range(RETURN_PLACE..=Local::from_usize(body.arg_count)); for (local, decl) in body.local_decls().iter_enumerated() { - if decl.ty.is_union() || decl.ty.is_enum() || excluded.contains(local) { + if excluded.contains(local) || is_excluded_ty(decl.ty) { set.insert(local); } } @@ -396,13 +436,12 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { VarDebugInfoContents::Composite { ty: _, ref mut fragments } => { let mut new_fragments = Vec::new(); debug!(?fragments); - fragments - .drain_filter(|fragment| { - if let Some(repl) = + fragments.retain_mut(|fragment| { + if let Some(repl) = self.replacements.replace_place(self.tcx, fragment.contents.as_ref()) { fragment.contents = repl; - false + true } else if let Some(local) = fragment.contents.as_local() && let Some(frg) = self.gather_debug_info_fragments(local) { @@ -410,12 +449,11 @@ impl<'tcx, 'll> MutVisitor<'tcx> for ReplacementVisitor<'tcx, 'll> { f.projection.splice(0..0, fragment.projection.iter().copied()); f })); - true - } else { false + } else { + true } - }) - .for_each(drop); + }); debug!(?fragments); debug!(?new_fragments); fragments.extend(new_fragments); diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index 7a0d3a025..8dc2dfe13 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -271,6 +271,14 @@ fn compute_copy_classes(ssa: &mut SsaLocals, body: &Body<'_>) { else { continue }; let Some(rhs) = place.as_local() else { continue }; + let local_ty = body.local_decls()[local].ty; + let rhs_ty = body.local_decls()[rhs].ty; + if local_ty != rhs_ty { + // FIXME(#112651): This can be removed afterwards. + trace!("skipped `{local:?} = {rhs:?}` due to subtyping: {local_ty} != {rhs_ty}"); + continue; + } + if !ssa.is_ssa(rhs) { continue; } |