diff options
Diffstat (limited to 'compiler/rustc_mir_transform')
55 files changed, 1668 insertions, 981 deletions
diff --git a/compiler/rustc_mir_transform/Cargo.toml b/compiler/rustc_mir_transform/Cargo.toml index 962536669..eca5f98a2 100644 --- a/compiler/rustc_mir_transform/Cargo.toml +++ b/compiler/rustc_mir_transform/Cargo.toml @@ -24,6 +24,8 @@ rustc_session = { path = "../rustc_session" } rustc_target = { path = "../rustc_target" } rustc_trait_selection = { path = "../rustc_trait_selection" } rustc_span = { path = "../rustc_span" } +rustc_fluent_macro = { path = "../rustc_fluent_macro" } +rustc_macros = { path = "../rustc_macros" } [dev-dependencies] coverage_test_macros = { path = "src/coverage/test_macros" } diff --git a/compiler/rustc_mir_transform/messages.ftl b/compiler/rustc_mir_transform/messages.ftl new file mode 100644 index 000000000..b13429d12 --- /dev/null +++ b/compiler/rustc_mir_transform/messages.ftl @@ -0,0 +1,67 @@ +mir_transform_arithmetic_overflow = this arithmetic operation will overflow +mir_transform_call_to_unsafe_label = call to unsafe function +mir_transform_call_to_unsafe_note = consult the function's documentation for information on how to avoid undefined behavior +mir_transform_const_defined_here = `const` item defined here + +mir_transform_const_modify = attempting to modify a `const` item + .note = each usage of a `const` item creates a new temporary; the original `const` item will not be modified + +mir_transform_const_mut_borrow = taking a mutable reference to a `const` item + .note = each usage of a `const` item creates a new temporary + .note2 = the mutable reference will refer to this temporary, not the original `const` item + .note3 = mutable reference created due to call to this method + +mir_transform_const_ptr2int_label = cast of pointer to int +mir_transform_const_ptr2int_note = casting pointers to integers in constants +mir_transform_deref_ptr_label = dereference of raw pointer +mir_transform_deref_ptr_note = raw pointers may be null, dangling or unaligned; they can violate aliasing rules and cause data races: all of these are undefined behavior +mir_transform_ffi_unwind_call = call to {$foreign -> + [true] foreign function + *[false] function pointer + } with FFI-unwind ABI + +mir_transform_fn_item_ref = taking a reference to a function item does not give a function pointer + .suggestion = cast `{$ident}` to obtain a function pointer + +mir_transform_initializing_valid_range_label = initializing type with `rustc_layout_scalar_valid_range` attr +mir_transform_initializing_valid_range_note = initializing a layout restricted type's field with a value outside the valid range is undefined behavior +mir_transform_must_not_suspend = {$pre}`{$def_path}`{$post} held across a suspend point, but should not be + .label = the value is held across this suspend point + .note = {$reason} + .help = consider using a block (`{"{ ... }"}`) to shrink the value's scope, ending before the suspend point + +mir_transform_mutation_layout_constrained_borrow_label = borrow of layout constrained field with interior mutability +mir_transform_mutation_layout_constrained_borrow_note = references to fields of layout constrained fields lose the constraints. Coupled with interior mutability, the field can be changed to invalid values +mir_transform_mutation_layout_constrained_label = mutation of layout constrained field +mir_transform_mutation_layout_constrained_note = mutating layout constrained fields cannot statically be checked for valid values +mir_transform_operation_will_panic = this operation will panic at runtime + +mir_transform_requires_unsafe = {$details} is unsafe and requires unsafe {$op_in_unsafe_fn_allowed -> + [true] function or block + *[false] block + } + .not_inherited = items do not inherit unsafety from separate enclosing items + +mir_transform_simd_shuffle_last_const = last argument of `simd_shuffle` is required to be a `const` item + +mir_transform_target_feature_call_label = call to function with `#[target_feature]` +mir_transform_target_feature_call_note = can only be called if the required target features are available + +mir_transform_unaligned_packed_ref = reference to packed field is unaligned + .note = packed structs are only aligned by one byte, and many modern architectures penalize unaligned field accesses + .note_ub = creating a misaligned reference is undefined behavior (even if that reference is never dereferenced) + .help = copy the field contents to a local variable, or replace the reference with a raw pointer and use `read_unaligned`/`write_unaligned` (loads and stores via `*p` must be properly aligned even when using raw pointers) + +mir_transform_union_access_label = access to union field +mir_transform_union_access_note = the field may not be properly initialized: using uninitialized data will cause undefined behavior +mir_transform_unsafe_op_in_unsafe_fn = {$details} is unsafe and requires unsafe block (error E0133) + +mir_transform_unused_unsafe = unnecessary `unsafe` block + .label = because it's nested under this `unsafe` block + +mir_transform_use_of_asm_label = use of inline assembly +mir_transform_use_of_asm_note = inline assembly is entirely unchecked and can cause undefined behavior +mir_transform_use_of_extern_static_label = use of extern static +mir_transform_use_of_extern_static_note = extern statics are not controlled by the Rust type system: invalid data, aliasing violations or data races will cause undefined behavior +mir_transform_use_of_static_mut_label = use of mutable static +mir_transform_use_of_static_mut_note = mutable statics can be mutated by multiple threads: aliasing violations or data races will cause undefined behavior diff --git a/compiler/rustc_mir_transform/src/add_call_guards.rs b/compiler/rustc_mir_transform/src/add_call_guards.rs index e1e354efa..fb4705e07 100644 --- a/compiler/rustc_mir_transform/src/add_call_guards.rs +++ b/compiler/rustc_mir_transform/src/add_call_guards.rs @@ -1,5 +1,5 @@ use crate::MirPass; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs index 896fcd9cd..ef2a0c790 100644 --- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs +++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs @@ -10,7 +10,7 @@ use rustc_middle::mir::patch::MirPatch; /// they are dropped from an aligned address. /// /// For example, if we have something like -/// ```ignore (ilustrative) +/// ```ignore (illustrative) /// #[repr(packed)] /// struct Foo { /// dealign: u8, @@ -25,7 +25,7 @@ use rustc_middle::mir::patch::MirPatch; /// its address is not aligned. /// /// Instead, we move `foo.data` to a local and drop that: -/// ```ignore (ilustrative) +/// ```ignore (illustrative) /// storage.live(drop_temp) /// drop_temp = foo.data; /// drop(drop_temp) -> next @@ -80,7 +80,7 @@ fn add_move_for_packed_drop<'tcx>( is_cleanup: bool, ) { debug!("add_move_for_packed_drop({:?} @ {:?})", terminator, loc); - let TerminatorKind::Drop { ref place, target, unwind } = terminator.kind else { + let TerminatorKind::Drop { ref place, target, unwind, replace } = terminator.kind else { unreachable!(); }; @@ -98,6 +98,11 @@ fn add_move_for_packed_drop<'tcx>( patch.add_assign(loc, Place::from(temp), Rvalue::Use(Operand::Move(*place))); patch.patch_terminator( loc.block, - TerminatorKind::Drop { place: Place::from(temp), target: storage_dead_block, unwind }, + TerminatorKind::Drop { + place: Place::from(temp), + target: storage_dead_block, + unwind, + replace, + }, ); } diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs index 916f2904d..187d38b38 100644 --- a/compiler/rustc_mir_transform/src/add_retag.rs +++ b/compiler/rustc_mir_transform/src/add_retag.rs @@ -59,7 +59,7 @@ impl<'tcx> MirPass<'tcx> for AddRetag { let basic_blocks = body.basic_blocks.as_mut(); let local_decls = &body.local_decls; let needs_retag = |place: &Place<'tcx>| { - !place.has_deref() // we're not eally interested in stores to "outside" locations, they are hard to keep track of anyway + !place.has_deref() // we're not really interested in stores to "outside" locations, they are hard to keep track of anyway && may_contain_reference(place.ty(&*local_decls, tcx).ty, /*depth*/ 3, tcx) && !local_decls[place.local].is_deref_temp() }; diff --git a/compiler/rustc_mir_transform/src/check_alignment.rs b/compiler/rustc_mir_transform/src/check_alignment.rs index 9311666c9..ea0780688 100644 --- a/compiler/rustc_mir_transform/src/check_alignment.rs +++ b/compiler/rustc_mir_transform/src/check_alignment.rs @@ -1,7 +1,7 @@ use crate::MirPass; use rustc_hir::def_id::DefId; use rustc_hir::lang_items::LangItem; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir::*; use rustc_middle::mir::{ interpret::{ConstValue, Scalar}, @@ -14,6 +14,10 @@ pub struct CheckAlignment; impl<'tcx> MirPass<'tcx> for CheckAlignment { fn is_enabled(&self, sess: &Session) -> bool { + // FIXME(#112480) MSVC and rustc disagree on minimum stack alignment on x86 Windows + if sess.target.llvm_target == "i686-pc-windows-msvc" { + return false; + } sess.opts.debug_assertions } @@ -232,10 +236,10 @@ fn insert_alignment_check<'tcx>( cond: Operand::Copy(is_ok), expected: true, target: new_block, - msg: AssertKind::MisalignedPointerDereference { + msg: Box::new(AssertKind::MisalignedPointerDereference { required: Operand::Copy(alignment), found: Operand::Copy(addr), - }, + }), unwind: UnwindAction::Terminate, }, }); diff --git a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs index 3d32c5865..b79150737 100644 --- a/compiler/rustc_mir_transform/src/check_const_item_mutation.rs +++ b/compiler/rustc_mir_transform/src/check_const_item_mutation.rs @@ -1,11 +1,12 @@ -use rustc_errors::{DiagnosticBuilder, DiagnosticMessage}; +use rustc_hir::HirId; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use rustc_session::lint::builtin::CONST_ITEM_MUTATION; use rustc_span::def_id::DefId; +use rustc_span::Span; -use crate::MirLint; +use crate::{errors, MirLint}; pub struct CheckConstItemMutation; @@ -58,16 +59,14 @@ impl<'tcx> ConstMutationChecker<'_, 'tcx> { } } - fn lint_const_item_usage( + /// If we should lint on this usage, return the [`HirId`], source [`Span`] + /// and [`Span`] of the const item to use in the lint. + fn should_lint_const_item_usage( &self, place: &Place<'tcx>, const_item: DefId, location: Location, - msg: impl Into<DiagnosticMessage>, - decorate: impl for<'a, 'b> FnOnce( - &'a mut DiagnosticBuilder<'b, ()>, - ) -> &'a mut DiagnosticBuilder<'b, ()>, - ) { + ) -> Option<(HirId, Span, Span)> { // Don't lint on borrowing/assigning when a dereference is involved. // If we 'leave' the temporary via a dereference, we must // be modifying something else @@ -83,16 +82,9 @@ impl<'tcx> ConstMutationChecker<'_, 'tcx> { .assert_crate_local() .lint_root; - self.tcx.struct_span_lint_hir( - CONST_ITEM_MUTATION, - lint_root, - source_info.span, - msg, - |lint| { - decorate(lint) - .span_note(self.tcx.def_span(const_item), "`const` item defined here") - }, - ); + Some((lint_root, source_info.span, self.tcx.def_span(const_item))) + } else { + None } } } @@ -104,10 +96,14 @@ impl<'tcx> Visitor<'tcx> for ConstMutationChecker<'_, 'tcx> { // Assigning directly to a constant (e.g. `FOO = true;`) is a hard error, // so emitting a lint would be redundant. if !lhs.projection.is_empty() { - if let Some(def_id) = self.is_const_item_without_destructor(lhs.local) { - self.lint_const_item_usage(&lhs, def_id, loc, "attempting to modify a `const` item",|lint| { - lint.note("each usage of a `const` item creates a new temporary; the original `const` item will not be modified") - }) + if let Some(def_id) = self.is_const_item_without_destructor(lhs.local) + && let Some((lint_root, span, item)) = self.should_lint_const_item_usage(&lhs, def_id, loc) { + self.tcx.emit_spanned_lint( + CONST_ITEM_MUTATION, + lint_root, + span, + errors::ConstMutate::Modify { konst: item } + ); } } // We are looking for MIR of the form: @@ -134,21 +130,31 @@ impl<'tcx> Visitor<'tcx> for ConstMutationChecker<'_, 'tcx> { // the `self` parameter of a method call (as the terminator of our current // BasicBlock). If so, we emit a more specific lint. let method_did = self.target_local.and_then(|target_local| { - crate::util::find_self_call(self.tcx, &self.body, target_local, loc.block) + rustc_middle::util::find_self_call( + self.tcx, + &self.body, + target_local, + loc.block, + ) }); let lint_loc = if method_did.is_some() { self.body.terminator_loc(loc.block) } else { loc }; - self.lint_const_item_usage(place, def_id, lint_loc, "taking a mutable reference to a `const` item", |lint| { - lint - .note("each usage of a `const` item creates a new temporary") - .note("the mutable reference will refer to this temporary, not the original `const` item"); - - if let Some((method_did, _substs)) = method_did { - lint.span_note(self.tcx.def_span(method_did), "mutable reference created due to call to this method"); - } - lint - }); + let method_call = if let Some((method_did, _)) = method_did { + Some(self.tcx.def_span(method_did)) + } else { + None + }; + if let Some((lint_root, span, item)) = + self.should_lint_const_item_usage(place, def_id, lint_loc) + { + self.tcx.emit_spanned_lint( + CONST_ITEM_MUTATION, + lint_root, + span, + errors::ConstMutate::MutBorrow { method_call, konst: item }, + ); + } } } self.super_rvalue(rvalue, loc); diff --git a/compiler/rustc_mir_transform/src/check_packed_ref.rs b/compiler/rustc_mir_transform/src/check_packed_ref.rs index f5f1c1010..2e6cf603d 100644 --- a/compiler/rustc_mir_transform/src/check_packed_ref.rs +++ b/compiler/rustc_mir_transform/src/check_packed_ref.rs @@ -1,10 +1,9 @@ -use rustc_errors::struct_span_err; use rustc_middle::mir::visit::{PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt}; -use crate::util; use crate::MirLint; +use crate::{errors, util}; pub struct CheckPackedRef; @@ -49,22 +48,7 @@ impl<'tcx> Visitor<'tcx> for PackedRefChecker<'_, 'tcx> { // shouldn't do. span_bug!(self.source_info.span, "builtin derive created an unaligned reference"); } else { - struct_span_err!( - self.tcx.sess, - self.source_info.span, - E0793, - "reference to packed field is unaligned" - ) - .note( - "fields of packed structs are not properly aligned, and creating \ - a misaligned reference is undefined behavior (even if that \ - reference is never dereferenced)", - ).help( - "copy the field contents to a local variable, or replace the \ - reference with a raw pointer and use `read_unaligned`/`write_unaligned` \ - (loads and stores via `*p` must be properly aligned even when using raw pointers)" - ) - .emit(); + self.tcx.sess.emit_err(errors::UnalignedPackedRef { span: self.source_info.span }); } } } diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs index d908f6b3a..069514d8a 100644 --- a/compiler/rustc_mir_transform/src/check_unsafety.rs +++ b/compiler/rustc_mir_transform/src/check_unsafety.rs @@ -1,5 +1,4 @@ use rustc_data_structures::unord::{UnordItems, UnordSet}; -use rustc_errors::struct_span_err; use rustc_hir as hir; use rustc_hir::def::DefKind; use rustc_hir::def_id::{DefId, LocalDefId}; @@ -8,13 +7,15 @@ use rustc_hir::intravisit; use rustc_hir::{BlockCheckMode, ExprKind, Node}; use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; -use rustc_middle::ty::query::Providers; +use rustc_middle::query::Providers; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::lint::builtin::{UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE}; use rustc_session::lint::Level; use std::ops::Bound; +use crate::errors; + pub struct UnsafetyChecker<'a, 'tcx> { body: &'a Body<'tcx>, body_did: LocalDefId, @@ -148,7 +149,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> { if let Some(uv) = maybe_uneval { if uv.promoted.is_none() { - let def_id = uv.def.def_id_for_type_of(); + let def_id = uv.def; if self.tcx.def_kind(def_id) == DefKind::InlineConst { let local_def_id = def_id.expect_local(); let UnsafetyCheckResult { violations, used_unsafe_blocks, .. } = @@ -375,22 +376,7 @@ impl<'tcx> UnsafetyChecker<'_, 'tcx> { } pub(crate) fn provide(providers: &mut Providers) { - *providers = Providers { - unsafety_check_result: |tcx, def_id| { - if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { - tcx.unsafety_check_result_for_const_arg(def) - } else { - unsafety_check_result(tcx, ty::WithOptConstParam::unknown(def_id)) - } - }, - unsafety_check_result_for_const_arg: |tcx, (did, param_did)| { - unsafety_check_result( - tcx, - ty::WithOptConstParam { did, const_param_did: Some(param_did) }, - ) - }, - ..*providers - }; + *providers = Providers { unsafety_check_result, ..*providers }; } /// Context information for [`UnusedUnsafeVisitor`] traversal, @@ -492,10 +478,7 @@ fn check_unused_unsafe( unused_unsafes } -fn unsafety_check_result( - tcx: TyCtxt<'_>, - def: ty::WithOptConstParam<LocalDefId>, -) -> &UnsafetyCheckResult { +fn unsafety_check_result(tcx: TyCtxt<'_>, def: LocalDefId) -> &UnsafetyCheckResult { debug!("unsafety_violations({:?})", def); // N.B., this borrow is valid because all the consumers of @@ -510,13 +493,13 @@ fn unsafety_check_result( }); } - let param_env = tcx.param_env(def.did); + let param_env = tcx.param_env(def); - let mut checker = UnsafetyChecker::new(body, def.did, tcx, param_env); + let mut checker = UnsafetyChecker::new(body, def, tcx, param_env); checker.visit_body(&body); - let unused_unsafes = (!tcx.is_typeck_child(def.did.to_def_id())) - .then(|| check_unused_unsafe(tcx, def.did, &checker.used_unsafe_blocks)); + let unused_unsafes = (!tcx.is_typeck_child(def.to_def_id())) + .then(|| check_unused_unsafe(tcx, def, &checker.used_unsafe_blocks)); tcx.arena.alloc(UnsafetyCheckResult { violations: checker.violations, @@ -527,21 +510,12 @@ fn unsafety_check_result( fn report_unused_unsafe(tcx: TyCtxt<'_>, kind: UnusedUnsafe, id: HirId) { let span = tcx.sess.source_map().guess_head_span(tcx.hir().span(id)); - let msg = "unnecessary `unsafe` block"; - tcx.struct_span_lint_hir(UNUSED_UNSAFE, id, span, msg, |lint| { - lint.span_label(span, msg); - match kind { - UnusedUnsafe::Unused => {} - UnusedUnsafe::InUnsafeBlock(id) => { - lint.span_label( - tcx.sess.source_map().guess_head_span(tcx.hir().span(id)), - "because it's nested under this `unsafe` block", - ); - } - } - - lint - }); + let nested_parent = if let UnusedUnsafe::InUnsafeBlock(id) = kind { + Some(tcx.sess.source_map().guess_head_span(tcx.hir().span(id))) + } else { + None + }; + tcx.emit_spanned_lint(UNUSED_UNSAFE, id, span, errors::UnusedUnsafe { span, nested_parent }); } pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) { @@ -555,26 +529,11 @@ pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) { let UnsafetyCheckResult { violations, unused_unsafes, .. } = tcx.unsafety_check_result(def_id); for &UnsafetyViolation { source_info, lint_root, kind, details } in violations.iter() { - let (description, note) = details.description_and_note(); + let details = errors::RequiresUnsafeDetail { violation: details, span: source_info.span }; match kind { UnsafetyViolationKind::General => { - // once - let unsafe_fn_msg = if unsafe_op_in_unsafe_fn_allowed(tcx, lint_root) { - " function or" - } else { - "" - }; - - let mut err = struct_span_err!( - tcx.sess, - source_info.span, - E0133, - "{} is unsafe and requires unsafe{} block", - description, - unsafe_fn_msg, - ); - err.span_label(source_info.span, description).note(note); + let op_in_unsafe_fn_allowed = unsafe_op_in_unsafe_fn_allowed(tcx, lint_root); let note_non_inherited = tcx.hir().parent_iter(lint_root).find(|(id, node)| { if let Node::Expr(block) = node && let ExprKind::Block(block, _) = block.kind @@ -590,22 +549,23 @@ pub fn check_unsafety(tcx: TyCtxt<'_>, def_id: LocalDefId) { false } }); - if let Some((id, _)) = note_non_inherited { - let span = tcx.hir().span(id); - err.span_label( - tcx.sess.source_map().guess_head_span(span), - "items do not inherit unsafety from separate enclosing items", - ); - } - - err.emit(); + let enclosing = if let Some((id, _)) = note_non_inherited { + Some(tcx.sess.source_map().guess_head_span(tcx.hir().span(id))) + } else { + None + }; + tcx.sess.emit_err(errors::RequiresUnsafe { + span: source_info.span, + enclosing, + details, + op_in_unsafe_fn_allowed, + }); } - UnsafetyViolationKind::UnsafeFn => tcx.struct_span_lint_hir( + UnsafetyViolationKind::UnsafeFn => tcx.emit_spanned_lint( UNSAFE_OP_IN_UNSAFE_FN, lint_root, source_info.span, - format!("{} is unsafe and requires unsafe block (error E0133)", description,), - |lint| lint.span_label(source_info.span, description).note(note), + errors::UnsafeOpInUnsafeFn { details }, ), } } diff --git a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs index 0923824db..d435d3ee6 100644 --- a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs +++ b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs @@ -24,7 +24,6 @@ impl<'tcx> MirPass<'tcx> for CleanupPostBorrowck { for statement in basic_block.statements.iter_mut() { match statement.kind { StatementKind::AscribeUserType(..) - | StatementKind::PlaceMention(..) | StatementKind::Assign(box (_, Rvalue::Ref(_, BorrowKind::Shallow, _))) | StatementKind::FakeRead(..) => statement.make_nop(), _ => (), diff --git a/compiler/rustc_mir_transform/src/const_debuginfo.rs b/compiler/rustc_mir_transform/src/const_debuginfo.rs index 6f0ae4f07..f662ce645 100644 --- a/compiler/rustc_mir_transform/src/const_debuginfo.rs +++ b/compiler/rustc_mir_transform/src/const_debuginfo.rs @@ -10,19 +10,19 @@ use rustc_middle::{ }; use crate::MirPass; -use rustc_index::{bit_set::BitSet, vec::IndexVec}; +use rustc_index::{bit_set::BitSet, IndexVec}; pub struct ConstDebugInfo; impl<'tcx> MirPass<'tcx> for ConstDebugInfo { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.opts.unstable_opts.unsound_mir_opts && sess.mir_opt_level() > 0 + sess.mir_opt_level() > 0 } fn run_pass(&self, _tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { trace!("running ConstDebugInfo on {:?}", body.source); - for (local, constant) in find_optimization_oportunities(body) { + for (local, constant) in find_optimization_opportunities(body) { for debuginfo in &mut body.var_debug_info { if let VarDebugInfoContents::Place(p) = debuginfo.value { if p.local == local && p.projection.is_empty() { @@ -45,7 +45,7 @@ struct LocalUseVisitor { local_assignment_locations: IndexVec<Local, Option<Location>>, } -fn find_optimization_oportunities<'tcx>(body: &Body<'tcx>) -> Vec<(Local, Constant<'tcx>)> { +fn find_optimization_opportunities<'tcx>(body: &Body<'tcx>) -> Vec<(Local, Constant<'tcx>)> { let mut visitor = LocalUseVisitor { local_mutating_uses: IndexVec::from_elem(0, &body.local_decls), local_assignment_locations: IndexVec::from_elem(None, &body.local_decls), diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs index 1bb45341e..1ba1951af 100644 --- a/compiler/rustc_mir_transform/src/const_prop.rs +++ b/compiler/rustc_mir_transform/src/const_prop.rs @@ -7,7 +7,7 @@ use rustc_const_eval::const_eval::CheckAlignment; use rustc_data_structures::fx::FxHashSet; use rustc_hir::def::DefKind; use rustc_index::bit_set::BitSet; -use rustc_index::vec::{IndexSlice, IndexVec}; +use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir::visit::{ MutVisitor, MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor, }; @@ -18,13 +18,12 @@ use rustc_middle::ty::{self, ConstKind, Instance, ParamEnv, Ty, TyCtxt, TypeVisi use rustc_span::{def_id::DefId, Span, DUMMY_SP}; use rustc_target::abi::{self, Align, HasDataLayout, Size, TargetDataLayout}; use rustc_target::spec::abi::Abi as CallAbi; -use rustc_trait_selection::traits; use crate::MirPass; use rustc_const_eval::interpret::{ - self, compile_time_machine, AllocId, ConstAllocation, ConstValue, CtfeValidationMode, Frame, - ImmTy, Immediate, InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, PlaceTy, Pointer, - Scalar, StackPopCleanup, + self, compile_time_machine, AllocId, ConstAllocation, ConstValue, Frame, ImmTy, Immediate, + InterpCx, InterpResult, LocalValue, MemoryKind, OpTy, PlaceTy, Pointer, Scalar, + StackPopCleanup, }; /// The maximum number of bytes that we'll allocate space for a local or the return value. @@ -84,42 +83,6 @@ impl<'tcx> MirPass<'tcx> for ConstProp { return; } - // Check if it's even possible to satisfy the 'where' clauses - // for this item. - // This branch will never be taken for any normal function. - // However, it's possible to `#!feature(trivial_bounds)]` to write - // a function with impossible to satisfy clauses, e.g.: - // `fn foo() where String: Copy {}` - // - // We don't usually need to worry about this kind of case, - // since we would get a compilation error if the user tried - // to call it. However, since we can do const propagation - // even without any calls to the function, we need to make - // sure that it even makes sense to try to evaluate the body. - // If there are unsatisfiable where clauses, then all bets are - // off, and we just give up. - // - // We manually filter the predicates, skipping anything that's not - // "global". We are in a potentially generic context - // (e.g. we are evaluating a function without substituting generic - // parameters, so this filtering serves two purposes: - // - // 1. We skip evaluating any predicates that we would - // never be able prove are unsatisfiable (e.g. `<T as Foo>` - // 2. We avoid trying to normalize predicates involving generic - // parameters (e.g. `<T as Foo>::MyItem`). This can confuse - // the normalization code (leading to cycle errors), since - // it's usually never invoked in this way. - let predicates = tcx - .predicates_of(def_id.to_def_id()) - .predicates - .iter() - .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None }); - if traits::impossible_predicates(tcx, traits::elaborate(tcx, predicates).collect()) { - trace!("ConstProp skipped for {:?}: found unsatisfiable predicates", def_id); - return; - } - trace!("ConstProp starting for {:?}", def_id); let dummy_body = &Body::new( @@ -428,7 +391,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Returns the value, if any, of evaluating `c`. fn eval_constant(&mut self, c: &Constant<'tcx>) -> Option<OpTy<'tcx>> { // FIXME we need to revisit this for #67176 - if c.needs_subst() { + if c.has_param() { return None; } @@ -501,16 +464,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { return None; } - // Do not try creating references, nor any types with potentially-complex - // invariants. This avoids an issue where checking validity would do a - // bunch of work generating a nice message about the invariant violation, - // only to not show it to anyone (since this isn't the lint). - Rvalue::Cast(CastKind::Transmute, op, dst_ty) if !dst_ty.is_primitive() => { - trace!("skipping Transmute of {:?} to {:?}", op, dst_ty); - - return None; - } - // There's no other checking to do at this time. Rvalue::Aggregate(..) | Rvalue::Use(..) @@ -527,7 +480,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } // FIXME we need to revisit this for #67176 - if rvalue.needs_subst() { + if rvalue.has_param() { return None; } if !rvalue @@ -628,18 +581,6 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } trace!("attempting to replace {:?} with {:?}", rval, value); - if let Err(e) = self.ecx.const_validate_operand( - value, - vec![], - // FIXME: is ref tracking too expensive? - // FIXME: what is the point of ref tracking if we do not even check the tracked refs? - &mut interpret::RefTracking::empty(), - CtfeValidationMode::Regular, - ) { - trace!("validation error, attempt failed: {:?}", e); - return; - } - // FIXME> figure out what to do when read_immediate_raw fails let imm = self.ecx.read_immediate_raw(value).ok(); @@ -773,13 +714,22 @@ impl CanConstProp { } } -impl Visitor<'_> for CanConstProp { +impl<'tcx> Visitor<'tcx> for CanConstProp { + fn visit_place(&mut self, place: &Place<'tcx>, mut context: PlaceContext, loc: Location) { + use rustc_middle::mir::visit::PlaceContext::*; + + // Dereferencing just read the addess of `place.local`. + if place.projection.first() == Some(&PlaceElem::Deref) { + context = NonMutatingUse(NonMutatingUseContext::Copy); + } + + self.visit_local(place.local, context, loc); + self.visit_projection(place.as_ref(), context, loc); + } + fn visit_local(&mut self, local: Local, context: PlaceContext, _: Location) { use rustc_middle::mir::visit::PlaceContext::*; match context { - // Projections are fine, because `&mut foo.x` will be caught by - // `MutatingUseContext::Borrow` elsewhere. - MutatingUse(MutatingUseContext::Projection) // These are just stores, where the storing is not propagatable, but there may be later // mutations of the same local via `Store` | MutatingUse(MutatingUseContext::Call) @@ -810,7 +760,7 @@ impl Visitor<'_> for CanConstProp { NonMutatingUse(NonMutatingUseContext::Copy) | NonMutatingUse(NonMutatingUseContext::Move) | NonMutatingUse(NonMutatingUseContext::Inspect) - | NonMutatingUse(NonMutatingUseContext::Projection) + | NonMutatingUse(NonMutatingUseContext::PlaceMention) | NonUse(_) => {} // These could be propagated with a smarter analysis or just some careful thinking about @@ -825,9 +775,11 @@ impl Visitor<'_> for CanConstProp { | NonMutatingUse(NonMutatingUseContext::AddressOf) | MutatingUse(MutatingUseContext::Borrow) | MutatingUse(MutatingUseContext::AddressOf) => { - trace!("local {:?} can't be propagaged because it's used: {:?}", local, context); + trace!("local {:?} can't be propagated because it's used: {:?}", local, context); self.can_const_prop[local] = ConstPropMode::NoPropagation; } + MutatingUse(MutatingUseContext::Projection) + | NonMutatingUse(NonMutatingUseContext::Projection) => bug!("visit_place should not pass {context:?} for {local:?}"), } } } @@ -853,6 +805,24 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> { } } + fn process_projection_elem( + &mut self, + elem: PlaceElem<'tcx>, + _: Location, + ) -> Option<PlaceElem<'tcx>> { + if let PlaceElem::Index(local) = elem + && let Some(value) = self.get_const(local.into()) + && self.should_const_prop(&value) + && let interpret::Operand::Immediate(interpret::Immediate::Scalar(scalar)) = *value + && let Ok(offset) = scalar.to_target_usize(&self.tcx) + && let Some(min_length) = offset.checked_add(1) + { + Some(PlaceElem::ConstantIndex { offset, min_length, from_end: false }) + } else { + None + } + } + fn visit_assign( &mut self, place: &mut Place<'tcx>, diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs index 699fe4489..0fe49b8a1 100644 --- a/compiler/rustc_mir_transform/src/const_prop_lint.rs +++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs @@ -1,6 +1,8 @@ //! Propagates constants for early reporting of statically known //! assertion failures +use std::fmt::Debug; + use either::Left; use rustc_const_eval::interpret::Immediate; @@ -17,7 +19,6 @@ use rustc_middle::ty::InternalSubsts; use rustc_middle::ty::{ self, ConstInt, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitableExt, }; -use rustc_session::lint; use rustc_span::Span; use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout}; use rustc_trait_selection::traits; @@ -25,6 +26,7 @@ use rustc_trait_selection::traits; use crate::const_prop::CanConstProp; use crate::const_prop::ConstPropMachine; use crate::const_prop::ConstPropMode; +use crate::errors::AssertLint; use crate::MirLint; /// The maximum number of bytes that we'll allocate space for a local or the return value. @@ -281,7 +283,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { /// Returns the value, if any, of evaluating `c`. fn eval_constant(&mut self, c: &Constant<'tcx>, location: Location) -> Option<OpTy<'tcx>> { // FIXME we need to revisit this for #67176 - if c.needs_subst() { + if c.has_param() { return None; } @@ -311,18 +313,9 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } } - fn report_assert_as_lint( - &self, - lint: &'static lint::Lint, - location: Location, - message: &'static str, - panic: AssertKind<impl std::fmt::Debug>, - ) { - let source_info = self.body().source_info(location); + fn report_assert_as_lint(&self, source_info: &SourceInfo, lint: AssertLint<impl Debug>) { if let Some(lint_root) = self.lint_root(*source_info) { - self.tcx.struct_span_lint_hir(lint, lint_root, source_info.span, message, |lint| { - lint.span_label(source_info.span, format!("{:?}", panic)) - }); + self.tcx.emit_spanned_lint(lint.lint(), lint_root, source_info.span, lint); } } @@ -335,11 +328,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // `AssertKind` only has an `OverflowNeg` variant, so make sure that is // appropriate to use. assert_eq!(op, UnOp::Neg, "Neg is the only UnOp that can overflow"); + let source_info = self.body().source_info(location); self.report_assert_as_lint( - lint::builtin::ARITHMETIC_OVERFLOW, - location, - "this arithmetic operation will overflow", - AssertKind::OverflowNeg(val.to_const_int()), + source_info, + AssertLint::ArithmeticOverflow( + source_info.span, + AssertKind::OverflowNeg(val.to_const_int()), + ), ); return None; } @@ -368,25 +363,25 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let left_size = self.ecx.layout_of(left_ty).ok()?.size; let right_size = r.layout.size; let r_bits = r.to_scalar().to_bits(right_size).ok(); - if r_bits.map_or(false, |b| b >= left_size.bits() as u128) { + if r_bits.is_some_and(|b| b >= left_size.bits() as u128) { debug!("check_binary_op: reporting assert for {:?}", location); + let source_info = self.body().source_info(location); + let panic = AssertKind::Overflow( + op, + match l { + Some(l) => l.to_const_int(), + // Invent a dummy value, the diagnostic ignores it anyway + None => ConstInt::new( + ScalarInt::try_from_uint(1_u8, left_size).unwrap(), + left_ty.is_signed(), + left_ty.is_ptr_sized_integral(), + ), + }, + r.to_const_int(), + ); self.report_assert_as_lint( - lint::builtin::ARITHMETIC_OVERFLOW, - location, - "this arithmetic operation will overflow", - AssertKind::Overflow( - op, - match l { - Some(l) => l.to_const_int(), - // Invent a dummy value, the diagnostic ignores it anyway - None => ConstInt::new( - ScalarInt::try_from_uint(1_u8, left_size).unwrap(), - left_ty.is_signed(), - left_ty.is_ptr_sized_integral(), - ), - }, - r.to_const_int(), - ), + source_info, + AssertLint::ArithmeticOverflow(source_info.span, panic), ); return None; } @@ -398,11 +393,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { let (_res, overflow, _ty) = this.ecx.overflowing_binary_op(op, &l, &r)?; Ok(overflow) })? { + let source_info = self.body().source_info(location); self.report_assert_as_lint( - lint::builtin::ARITHMETIC_OVERFLOW, - location, - "this arithmetic operation will overflow", - AssertKind::Overflow(op, l.to_const_int(), r.to_const_int()), + source_info, + AssertLint::ArithmeticOverflow( + source_info.span, + AssertKind::Overflow(op, l.to_const_int(), r.to_const_int()), + ), ); return None; } @@ -474,7 +471,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { } // FIXME we need to revisit this for #67176 - if rvalue.needs_subst() { + if rvalue.has_param() { return None; } if !rvalue.ty(self.local_decls(), self.tcx).is_sized(self.tcx, self.param_env) { @@ -493,7 +490,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { cond: &Operand<'tcx>, location: Location, ) -> Option<!> { - let ref value = self.eval_operand(&cond, location)?; + let value = &self.eval_operand(&cond, location)?; trace!("assertion on {:?} should be {:?}", value, expected); let expected = Scalar::from_bool(expected); @@ -543,11 +540,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> { // Need proper const propagator for these. _ => return None, }; + let source_info = self.body().source_info(location); self.report_assert_as_lint( - lint::builtin::UNCONDITIONAL_PANIC, - location, - "this operation will panic at runtime", - msg, + source_info, + AssertLint::UnconditionalPanic(source_info.span, msg), ); } diff --git a/compiler/rustc_mir_transform/src/copy_prop.rs b/compiler/rustc_mir_transform/src/copy_prop.rs index b571215f2..3df459dfa 100644 --- a/compiler/rustc_mir_transform/src/copy_prop.rs +++ b/compiler/rustc_mir_transform/src/copy_prop.rs @@ -1,5 +1,5 @@ use rustc_index::bit_set::BitSet; -use rustc_index::vec::IndexSlice; +use rustc_index::IndexSlice; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; @@ -33,9 +33,8 @@ impl<'tcx> MirPass<'tcx> for CopyProp { } fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); let borrowed_locals = borrowed_locals(body); - let ssa = SsaLocals::new(tcx, param_env, body, &borrowed_locals); + let ssa = SsaLocals::new(body); let fully_moved = fully_moved_locals(&ssa, body); debug!(?fully_moved); @@ -76,7 +75,7 @@ fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { fn fully_moved_locals(ssa: &SsaLocals, body: &Body<'_>) -> BitSet<Local> { let mut fully_moved = BitSet::new_filled(body.local_decls.len()); - for (_, rvalue) in ssa.assignments(body) { + for (_, rvalue, _) in ssa.assignments(body) { let (Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) | Rvalue::CopyForDeref(place)) = rvalue else { continue }; @@ -162,20 +161,22 @@ impl<'tcx> MutVisitor<'tcx> for Replacer<'_, 'tcx> { } fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, loc: Location) { - match stmt.kind { - // When removing storage statements, we need to remove both (#107511). - StatementKind::StorageLive(l) | StatementKind::StorageDead(l) - if self.storage_to_remove.contains(l) => - { - stmt.make_nop() - } - StatementKind::Assign(box (ref place, ref mut rvalue)) - if place.as_local().is_some() => - { - // Do not replace assignments. - self.visit_rvalue(rvalue, loc) - } - _ => self.super_statement(stmt, loc), + // When removing storage statements, we need to remove both (#107511). + if let StatementKind::StorageLive(l) | StatementKind::StorageDead(l) = stmt.kind + && self.storage_to_remove.contains(l) + { + stmt.make_nop(); + return + } + + self.super_statement(stmt, loc); + + // Do not leave tautological assignments around. + if let StatementKind::Assign(box (lhs, ref rhs)) = stmt.kind + && let Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)) | Rvalue::CopyForDeref(rhs) = *rhs + && lhs == rhs + { + stmt.make_nop(); } } } diff --git a/compiler/rustc_mir_transform/src/coverage/debug.rs b/compiler/rustc_mir_transform/src/coverage/debug.rs index 725883b83..6a3d42511 100644 --- a/compiler/rustc_mir_transform/src/coverage/debug.rs +++ b/compiler/rustc_mir_transform/src/coverage/debug.rs @@ -118,7 +118,7 @@ use rustc_middle::mir::spanview::{self, SpanViewable}; use rustc_data_structures::fx::FxHashMap; use rustc_middle::mir::coverage::*; -use rustc_middle::mir::{self, BasicBlock, TerminatorKind}; +use rustc_middle::mir::{self, BasicBlock}; use rustc_middle::ty::TyCtxt; use rustc_span::Span; @@ -292,10 +292,8 @@ impl DebugCounters { } pub fn some_block_label(&self, operand: ExpressionOperandId) -> Option<&String> { - self.some_counters.as_ref().map_or(None, |counters| { - counters - .get(&operand) - .map_or(None, |debug_counter| debug_counter.some_block_label.as_ref()) + self.some_counters.as_ref().and_then(|counters| { + counters.get(&operand).and_then(|debug_counter| debug_counter.some_block_label.as_ref()) }) } @@ -641,7 +639,7 @@ pub(super) fn dump_coverage_spanview<'tcx>( let def_id = mir_source.def_id(); let span_viewables = span_viewables(tcx, mir_body, basic_coverage_blocks, &coverage_spans); - let mut file = create_dump_file(tcx, "html", false, pass_name, &0, mir_body) + let mut file = create_dump_file(tcx, "html", false, pass_name, &0i32, mir_body) .expect("Unexpected error creating MIR spanview HTML file"); let crate_name = tcx.crate_name(def_id.krate); let item_name = tcx.def_path(def_id).to_filename_friendly_no_crate(); @@ -742,7 +740,7 @@ pub(super) fn dump_coverage_graphviz<'tcx>( .join("\n ") )); } - let mut file = create_dump_file(tcx, "dot", false, pass_name, &0, mir_body) + let mut file = create_dump_file(tcx, "dot", false, pass_name, &0i32, mir_body) .expect("Unexpected error creating BasicCoverageBlock graphviz DOT file"); graphviz_writer .write_graphviz(tcx, &mut file) @@ -798,7 +796,7 @@ fn bcb_to_string_sections<'tcx>( } let non_term_blocks = bcb_data.basic_blocks[0..len - 1] .iter() - .map(|&bb| format!("{:?}: {}", bb, term_type(&mir_body[bb].terminator().kind))) + .map(|&bb| format!("{:?}: {}", bb, mir_body[bb].terminator().kind.name())) .collect::<Vec<_>>(); if non_term_blocks.len() > 0 { sections.push(non_term_blocks.join("\n")); @@ -806,28 +804,7 @@ fn bcb_to_string_sections<'tcx>( sections.push(format!( "{:?}: {}", bcb_data.basic_blocks.last().unwrap(), - term_type(&bcb_data.terminator(mir_body).kind) + bcb_data.terminator(mir_body).kind.name(), )); sections } - -/// Returns a simple string representation of a `TerminatorKind` variant, independent of any -/// values it might hold. -pub(super) fn term_type(kind: &TerminatorKind<'_>) -> &'static str { - match kind { - TerminatorKind::Goto { .. } => "Goto", - TerminatorKind::SwitchInt { .. } => "SwitchInt", - TerminatorKind::Resume => "Resume", - TerminatorKind::Terminate => "Terminate", - TerminatorKind::Return => "Return", - TerminatorKind::Unreachable => "Unreachable", - TerminatorKind::Drop { .. } => "Drop", - TerminatorKind::Call { .. } => "Call", - TerminatorKind::Assert { .. } => "Assert", - TerminatorKind::Yield { .. } => "Yield", - TerminatorKind::GeneratorDrop => "GeneratorDrop", - TerminatorKind::FalseEdge { .. } => "FalseEdge", - TerminatorKind::FalseUnwind { .. } => "FalseUnwind", - TerminatorKind::InlineAsm { .. } => "InlineAsm", - } -} diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs index 7391a77b0..ea1223fbc 100644 --- a/compiler/rustc_mir_transform/src/coverage/graph.rs +++ b/compiler/rustc_mir_transform/src/coverage/graph.rs @@ -5,10 +5,11 @@ use rustc_data_structures::fx::FxHashMap; use rustc_data_structures::graph::dominators::{self, Dominators}; use rustc_data_structures::graph::{self, GraphSuccessors, WithNumNodes, WithStartNode}; use rustc_index::bit_set::BitSet; -use rustc_index::vec::{IndexSlice, IndexVec}; +use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::mir::coverage::*; use rustc_middle::mir::{self, BasicBlock, BasicBlockData, Terminator, TerminatorKind}; +use std::cmp::Ordering; use std::ops::{Index, IndexMut}; const ID_SEPARATOR: &str = ","; @@ -111,7 +112,7 @@ impl CoverageGraph { if predecessors.len() > 1 { "predecessors.len() > 1".to_owned() } else { - format!("bb {} is not in precessors: {:?}", bb.index(), predecessors) + format!("bb {} is not in predecessors: {:?}", bb.index(), predecessors) } ); } @@ -212,8 +213,12 @@ impl CoverageGraph { } #[inline(always)] - pub fn dominators(&self) -> &Dominators<BasicCoverageBlock> { - self.dominators.as_ref().unwrap() + pub fn rank_partial_cmp( + &self, + a: BasicCoverageBlock, + b: BasicCoverageBlock, + ) -> Option<Ordering> { + self.dominators.as_ref().unwrap().rank_partial_cmp(a, b) } } @@ -650,26 +655,6 @@ pub(super) fn find_loop_backedges( let mut backedges = IndexVec::from_elem_n(Vec::<BasicCoverageBlock>::new(), num_bcbs); // Identify loops by their backedges. - // - // The computational complexity is bounded by: n(s) x d where `n` is the number of - // `BasicCoverageBlock` nodes (the simplified/reduced representation of the CFG derived from the - // MIR); `s` is the average number of successors per node (which is most likely less than 2, and - // independent of the size of the function, so it can be treated as a constant); - // and `d` is the average number of dominators per node. - // - // The average number of dominators depends on the size and complexity of the function, and - // nodes near the start of the function's control flow graph typically have less dominators - // than nodes near the end of the CFG. Without doing a detailed mathematical analysis, I - // think the resulting complexity has the characteristics of O(n log n). - // - // The overall complexity appears to be comparable to many other MIR transform algorithms, and I - // don't expect that this function is creating a performance hot spot, but if this becomes an - // issue, there may be ways to optimize the `dominates` algorithm (as indicated by an - // existing `FIXME` comment in that code), or possibly ways to optimize it's usage here, perhaps - // by keeping track of results for visited `BasicCoverageBlock`s if they can be used to short - // circuit downstream `dominates` checks. - // - // For now, that kind of optimization seems unnecessarily complicated. for (bcb, _) in basic_coverage_blocks.iter_enumerated() { for &successor in &basic_coverage_blocks.successors[bcb] { if basic_coverage_blocks.dominates(successor, bcb) { diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs index 5ecb2d6a6..076e714d7 100644 --- a/compiler/rustc_mir_transform/src/coverage/mod.rs +++ b/compiler/rustc_mir_transform/src/coverage/mod.rs @@ -16,7 +16,7 @@ use crate::MirPass; use rustc_data_structures::graph::WithNumNodes; use rustc_data_structures::sync::Lrc; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::hir; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; use rustc_middle::mir::coverage::*; @@ -514,7 +514,7 @@ fn make_code_region( // Extend an empty span by one character so the region will be counted. let CharPos(char_pos) = start_col; if span.hi() == body_span.hi() { - start_col = CharPos(char_pos - 1); + start_col = CharPos(char_pos.saturating_sub(1)); } else { end_col = CharPos(char_pos + 1); } @@ -577,5 +577,10 @@ fn get_body_span<'tcx>( fn hash_mir_source<'tcx>(tcx: TyCtxt<'tcx>, hir_body: &'tcx rustc_hir::Body<'tcx>) -> u64 { // FIXME(cjgillot) Stop hashing HIR manually here. let owner = hir_body.id().hir_id.owner; - tcx.hir_owner_nodes(owner).unwrap().opt_hash_including_bodies.unwrap().to_smaller_hash() + tcx.hir_owner_nodes(owner) + .unwrap() + .opt_hash_including_bodies + .unwrap() + .to_smaller_hash() + .as_u64() } diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs index 3bd7f31b4..74b4b4a07 100644 --- a/compiler/rustc_mir_transform/src/coverage/query.rs +++ b/compiler/rustc_mir_transform/src/coverage/query.rs @@ -2,7 +2,7 @@ use super::*; use rustc_middle::mir::coverage::*; use rustc_middle::mir::{self, Body, Coverage, CoverageInfo}; -use rustc_middle::ty::query::Providers; +use rustc_middle::query::Providers; use rustc_middle::ty::{self, TyCtxt}; use rustc_span::def_id::DefId; @@ -164,7 +164,6 @@ fn is_inlined(body: &Body<'_>, statement: &Statement<'_>) -> bool { /// whether that means const mir or runtime mir. For `const fn` this opts for runtime /// mir. fn mir_body(tcx: TyCtxt<'_>, def_id: DefId) -> &mir::Body<'_> { - let id = ty::WithOptConstParam::unknown(def_id); - let def = ty::InstanceDef::Item(id); + let def = ty::InstanceDef::Item(def_id); tcx.instance_mir(def) } diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs index 287ae2170..d27200419 100644 --- a/compiler/rustc_mir_transform/src/coverage/spans.rs +++ b/compiler/rustc_mir_transform/src/coverage/spans.rs @@ -1,4 +1,3 @@ -use super::debug::term_type; use super::graph::{BasicCoverageBlock, BasicCoverageBlockData, CoverageGraph, START_BCB}; use itertools::Itertools; @@ -40,7 +39,7 @@ impl CoverageStatement { "{}: @{}.{}: {:?}", source_range_no_file(tcx, span), bb.index(), - term_type(&term.kind), + term.kind.name(), term.kind ) } @@ -345,7 +344,7 @@ impl<'a, 'tcx> CoverageSpans<'a, 'tcx> { // before the dominated equal spans). When later comparing two spans in // order, the first will either dominate the second, or they will have no // dominator relationship. - self.basic_coverage_blocks.dominators().rank_partial_cmp(a.bcb, b.bcb) + self.basic_coverage_blocks.rank_partial_cmp(a.bcb, b.bcb) } } else { // Sort hi() in reverse order so shorter spans are attempted after longer spans. diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs index 0f6c06e37..90b58933d 100644 --- a/compiler/rustc_mir_transform/src/coverage/tests.rs +++ b/compiler/rustc_mir_transform/src/coverage/tests.rs @@ -25,7 +25,6 @@ //! to: `rustc_span::create_default_session_globals_then(|| { test_here(); })`. use super::counters; -use super::debug; use super::graph; use super::spans; @@ -34,7 +33,7 @@ use coverage_test_macros::let_bcb; use itertools::Itertools; use rustc_data_structures::graph::WithNumNodes; use rustc_data_structures::graph::WithSuccessors; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::coverage::CoverageKind; use rustc_middle::mir::*; use rustc_middle::ty; @@ -188,12 +187,12 @@ fn debug_basic_blocks(mir_body: &Body<'_>) -> String { | TerminatorKind::Goto { target } | TerminatorKind::InlineAsm { destination: Some(target), .. } | TerminatorKind::Yield { resume: target, .. } => { - format!("{}{:?}:{} -> {:?}", sp, bb, debug::term_type(kind), target) + format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), target) } TerminatorKind::SwitchInt { targets, .. } => { - format!("{}{:?}:{} -> {:?}", sp, bb, debug::term_type(kind), targets) + format!("{}{:?}:{} -> {:?}", sp, bb, kind.name(), targets) } - _ => format!("{}{:?}:{}", sp, bb, debug::term_type(kind)), + _ => format!("{}{:?}:{}", sp, bb, kind.name()), } }) .collect::<Vec<_>>() @@ -215,7 +214,7 @@ fn print_mir_graphviz(name: &str, mir_body: &Body<'_>) { " {:?} [label=\"{:?}: {}\"];\n{}", bb, bb, - debug::term_type(&data.terminator().kind), + data.terminator().kind.name(), mir_body .basic_blocks .successors(bb) @@ -244,7 +243,7 @@ fn print_coverage_graphviz( " {:?} [label=\"{:?}: {}\"];\n{}", bcb, bcb, - debug::term_type(&bcb_data.terminator(mir_body).kind), + bcb_data.terminator(mir_body).kind.name(), basic_coverage_blocks .successors(bcb) .map(|successor| { format!(" {:?} -> {:?};", bcb, successor) }) diff --git a/compiler/rustc_mir_transform/src/ctfe_limit.rs b/compiler/rustc_mir_transform/src/ctfe_limit.rs index 1b3ac78fb..bf5722b3d 100644 --- a/compiler/rustc_mir_transform/src/ctfe_limit.rs +++ b/compiler/rustc_mir_transform/src/ctfe_limit.rs @@ -47,7 +47,7 @@ fn has_back_edge( return false; } // Check if any of the dominators of the node are also the node's successor. - doms.dominators(node).any(|dom| node_data.terminator().successors().any(|succ| succ == dom)) + node_data.terminator().successors().any(|succ| doms.dominates(succ, node)) } fn insert_counter(basic_block_data: &mut BasicBlockData<'_>) { diff --git a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs index d4db7e2de..7adfc9dff 100644 --- a/compiler/rustc_mir_transform/src/dataflow_const_prop.rs +++ b/compiler/rustc_mir_transform/src/dataflow_const_prop.rs @@ -70,22 +70,6 @@ struct ConstAnalysis<'a, 'tcx> { param_env: ty::ParamEnv<'tcx>, } -impl<'tcx> ConstAnalysis<'_, 'tcx> { - fn eval_discriminant( - &self, - enum_ty: Ty<'tcx>, - variant_index: VariantIdx, - ) -> Option<ScalarTy<'tcx>> { - if !enum_ty.is_enum() { - return None; - } - let discr = enum_ty.discriminant_for_variant(self.tcx, variant_index)?; - let discr_layout = self.tcx.layout_of(self.param_env.and(discr.ty)).ok()?; - let discr_value = Scalar::try_from_uint(discr.val, discr_layout.size)?; - Some(ScalarTy(discr_value, discr.ty)) - } -} - impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { type Value = FlatSet<ScalarTy<'tcx>>; @@ -95,22 +79,22 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { &self.map } - fn handle_statement(&self, statement: &Statement<'tcx>, state: &mut State<Self::Value>) { - match statement.kind { - StatementKind::SetDiscriminant { box ref place, variant_index } => { - state.flood_discr(place.as_ref(), &self.map); - if self.map.find_discr(place.as_ref()).is_some() { - let enum_ty = place.ty(self.local_decls, self.tcx).ty; - if let Some(discr) = self.eval_discriminant(enum_ty, variant_index) { - state.assign_discr( - place.as_ref(), - ValueOrPlace::Value(FlatSet::Elem(discr)), - &self.map, - ); - } - } + fn handle_set_discriminant( + &self, + place: Place<'tcx>, + variant_index: VariantIdx, + state: &mut State<Self::Value>, + ) { + state.flood_discr(place.as_ref(), &self.map); + if self.map.find_discr(place.as_ref()).is_some() { + let enum_ty = place.ty(self.local_decls, self.tcx).ty; + if let Some(discr) = self.eval_discriminant(enum_ty, variant_index) { + state.assign_discr( + place.as_ref(), + ValueOrPlace::Value(FlatSet::Elem(discr)), + &self.map, + ); } - _ => self.super_statement(statement, state), } } @@ -126,59 +110,55 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { // we must make sure that all `target as Variant#i` are `Top`. state.flood(target.as_ref(), self.map()); - if let Some(target_idx) = self.map().find(target.as_ref()) { - let (variant_target, variant_index) = match **kind { - AggregateKind::Tuple | AggregateKind::Closure(..) => { - (Some(target_idx), None) - } - AggregateKind::Adt(def_id, variant_index, ..) => { - match self.tcx.def_kind(def_id) { - DefKind::Struct => (Some(target_idx), None), - DefKind::Enum => ( - self.map.apply(target_idx, TrackElem::Variant(variant_index)), - Some(variant_index), - ), - _ => (None, None), - } - } - _ => (None, None), - }; - if let Some(variant_target_idx) = variant_target { - for (field_index, operand) in operands.iter().enumerate() { - if let Some(field) = self.map().apply( - variant_target_idx, - TrackElem::Field(FieldIdx::from_usize(field_index)), - ) { - let result = self.handle_operand(operand, state); - state.insert_idx(field, result, self.map()); - } + let Some(target_idx) = self.map().find(target.as_ref()) else { return }; + + let (variant_target, variant_index) = match **kind { + AggregateKind::Tuple | AggregateKind::Closure(..) => (Some(target_idx), None), + AggregateKind::Adt(def_id, variant_index, ..) => { + match self.tcx.def_kind(def_id) { + DefKind::Struct => (Some(target_idx), None), + DefKind::Enum => ( + self.map.apply(target_idx, TrackElem::Variant(variant_index)), + Some(variant_index), + ), + _ => return, } } - if let Some(variant_index) = variant_index - && let Some(discr_idx) = self.map().apply(target_idx, TrackElem::Discriminant) - { - // We are assigning the discriminant as part of an aggregate. - // This discriminant can only alias a variant field's value if the operand - // had an invalid value for that type. - // Using invalid values is UB, so we are allowed to perform the assignment - // without extra flooding. - let enum_ty = target.ty(self.local_decls, self.tcx).ty; - if let Some(discr_val) = self.eval_discriminant(enum_ty, variant_index) { - state.insert_value_idx(discr_idx, FlatSet::Elem(discr_val), &self.map); + _ => return, + }; + if let Some(variant_target_idx) = variant_target { + for (field_index, operand) in operands.iter().enumerate() { + if let Some(field) = self.map().apply( + variant_target_idx, + TrackElem::Field(FieldIdx::from_usize(field_index)), + ) { + let result = self.handle_operand(operand, state); + state.insert_idx(field, result, self.map()); } } } + if let Some(variant_index) = variant_index + && let Some(discr_idx) = self.map().apply(target_idx, TrackElem::Discriminant) + { + // We are assigning the discriminant as part of an aggregate. + // This discriminant can only alias a variant field's value if the operand + // had an invalid value for that type. + // Using invalid values is UB, so we are allowed to perform the assignment + // without extra flooding. + let enum_ty = target.ty(self.local_decls, self.tcx).ty; + if let Some(discr_val) = self.eval_discriminant(enum_ty, variant_index) { + state.insert_value_idx(discr_idx, FlatSet::Elem(discr_val), &self.map); + } + } } Rvalue::CheckedBinaryOp(op, box (left, right)) => { // Flood everything now, so we can use `insert_value_idx` directly later. state.flood(target.as_ref(), self.map()); - let target = self.map().find(target.as_ref()); + let Some(target) = self.map().find(target.as_ref()) else { return }; - let value_target = target - .and_then(|target| self.map().apply(target, TrackElem::Field(0_u32.into()))); - let overflow_target = target - .and_then(|target| self.map().apply(target, TrackElem::Field(1_u32.into()))); + let value_target = self.map().apply(target, TrackElem::Field(0_u32.into())); + let overflow_target = self.map().apply(target, TrackElem::Field(1_u32.into())); if value_target.is_some() || overflow_target.is_some() { let (val, overflow) = self.binary_op(state, *op, left, right); @@ -228,8 +208,8 @@ impl<'tcx> ValueAnalysis<'tcx> for ConstAnalysis<'_, 'tcx> { _ => unreachable!(), } .map(|result| ValueOrPlace::Value(self.wrap_immediate(result, *ty))) - .unwrap_or(ValueOrPlace::top()), - _ => ValueOrPlace::top(), + .unwrap_or(ValueOrPlace::TOP), + _ => ValueOrPlace::TOP, }, Rvalue::BinaryOp(op, box (left, right)) => { // Overflows must be ignored here. @@ -323,7 +303,7 @@ impl<'tcx> std::fmt::Debug for ScalarTy<'tcx> { impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { pub fn new(tcx: TyCtxt<'tcx>, body: &'a Body<'tcx>, map: Map) -> Self { - let param_env = tcx.param_env(body.source.def_id()); + let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); Self { map, tcx, @@ -351,7 +331,7 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } (FlatSet::Bottom, _) | (_, FlatSet::Bottom) => (FlatSet::Bottom, FlatSet::Bottom), (_, _) => { - // Could attempt some algebraic simplifcations here. + // Could attempt some algebraic simplifications here. (FlatSet::Top, FlatSet::Top) } } @@ -377,6 +357,20 @@ impl<'a, 'tcx> ConstAnalysis<'a, 'tcx> { } } + fn eval_discriminant( + &self, + enum_ty: Ty<'tcx>, + variant_index: VariantIdx, + ) -> Option<ScalarTy<'tcx>> { + if !enum_ty.is_enum() { + return None; + } + let discr = enum_ty.discriminant_for_variant(self.tcx, variant_index)?; + let discr_layout = self.tcx.layout_of(self.param_env.and(discr.ty)).ok()?; + let discr_value = Scalar::try_from_uint(discr.val, discr_layout.size)?; + Some(ScalarTy(discr_value, discr.ty)) + } + fn wrap_scalar(&self, scalar: Scalar, ty: Ty<'tcx>) -> FlatSet<ScalarTy<'tcx>> { FlatSet::Elem(ScalarTy(scalar, ty)) } @@ -520,21 +514,6 @@ impl<'tcx, 'map, 'a> Visitor<'tcx> for OperandCollector<'tcx, 'map, 'a> { _ => (), } } - - fn visit_rvalue(&mut self, rvalue: &Rvalue<'tcx>, location: Location) { - match rvalue { - Rvalue::Discriminant(place) => { - match self.state.get_discr(place.as_ref(), self.visitor.map) { - FlatSet::Top => (), - FlatSet::Elem(value) => { - self.visitor.before_effect.insert((location, *place), value); - } - FlatSet::Bottom => (), - } - } - _ => self.super_rvalue(rvalue, location), - } - } } struct DummyMachine; diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs index 18c407b42..7bc5183a0 100644 --- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs +++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs @@ -54,11 +54,10 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, borrowed: &BitS | StatementKind::Coverage(_) | StatementKind::Intrinsic(_) | StatementKind::ConstEvalCounter + | StatementKind::PlaceMention(_) | StatementKind::Nop => (), - StatementKind::FakeRead(_) - | StatementKind::PlaceMention(_) - | StatementKind::AscribeUserType(_, _) => { + StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => { bug!("{:?} not found in this MIR phase!", &statement.kind) } } diff --git a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs index e5c3fa564..a133c9d47 100644 --- a/compiler/rustc_mir_transform/src/deduce_param_attrs.rs +++ b/compiler/rustc_mir_transform/src/deduce_param_attrs.rs @@ -8,8 +8,8 @@ use rustc_hir::def_id::LocalDefId; use rustc_index::bit_set::BitSet; use rustc_middle::mir::visit::{NonMutatingUseContext, PlaceContext, Visitor}; -use rustc_middle::mir::{Body, Local, Location, Operand, Terminator, TerminatorKind, RETURN_PLACE}; -use rustc_middle::ty::{self, DeducedParamAttrs, ParamEnv, Ty, TyCtxt}; +use rustc_middle::mir::{Body, Location, Operand, Place, Terminator, TerminatorKind, RETURN_PLACE}; +use rustc_middle::ty::{self, DeducedParamAttrs, Ty, TyCtxt}; use rustc_session::config::OptLevel; /// A visitor that determines which arguments have been mutated. We can't use the mutability field @@ -29,31 +29,31 @@ impl DeduceReadOnly { } impl<'tcx> Visitor<'tcx> for DeduceReadOnly { - fn visit_local(&mut self, local: Local, mut context: PlaceContext, _: Location) { + fn visit_place(&mut self, place: &Place<'tcx>, context: PlaceContext, _location: Location) { // We're only interested in arguments. - if local == RETURN_PLACE || local.index() > self.mutable_args.domain_size() { + if place.local == RETURN_PLACE || place.local.index() > self.mutable_args.domain_size() { return; } - // Replace place contexts that are moves with copies. This is safe in all cases except - // function argument position, which we already handled in `visit_terminator()` by using the - // ArgumentChecker. See the comment in that method for more details. - // - // In the future, we might want to move this out into a separate pass, but for now let's - // just do it on the fly because that's faster. - if matches!(context, PlaceContext::NonMutatingUse(NonMutatingUseContext::Move)) { - context = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy); - } - - match context { - PlaceContext::MutatingUse(..) - | PlaceContext::NonMutatingUse(NonMutatingUseContext::Move) => { + let mark_as_mutable = match context { + PlaceContext::MutatingUse(..) => { // This is a mutation, so mark it as such. - self.mutable_args.insert(local.index() - 1); + true + } + PlaceContext::NonMutatingUse(NonMutatingUseContext::AddressOf) => { + // Whether mutating though a `&raw const` is allowed is still undecided, so we + // disable any sketchy `readonly` optimizations for now. + // But we only need to do this if the pointer would point into the argument. + !place.is_indirect() } PlaceContext::NonMutatingUse(..) | PlaceContext::NonUse(..) => { // Not mutating, so it's fine. + false } + }; + + if mark_as_mutable { + self.mutable_args.insert(place.local.index() - 1); } } @@ -198,11 +198,12 @@ pub fn deduced_param_attrs<'tcx>( // see [1]. // // [1]: https://github.com/rust-lang/rust/pull/103172#discussion_r999139997 + let param_env = tcx.param_env_reveal_all_normalized(def_id); let mut deduced_param_attrs = tcx.arena.alloc_from_iter( body.local_decls.iter().skip(1).take(body.arg_count).enumerate().map( |(arg_index, local_decl)| DeducedParamAttrs { read_only: !deduce_read_only.mutable_args.contains(arg_index) - && local_decl.ty.is_freeze(tcx, ParamEnv::reveal_all()), + && local_decl.ty.is_freeze(tcx, param_env), }, ), ); diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs index b8a5b92be..a39026751 100644 --- a/compiler/rustc_mir_transform/src/deref_separator.rs +++ b/compiler/rustc_mir_transform/src/deref_separator.rs @@ -1,5 +1,5 @@ use crate::MirPass; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::visit::NonUseContext::VarDebugInfo; use rustc_middle::mir::visit::{MutVisitor, PlaceContext}; diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 391649177..78758e2db 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -69,7 +69,7 @@ //! of this is that such liveness analysis can report more accurate results about whole locals at //! a time. For example, consider: //! -//! ```ignore (syntax-highliting-only) +//! ```ignore (syntax-highlighting-only) //! _1 = u; //! // unrelated code //! _1.f1 = v; @@ -360,7 +360,7 @@ struct FilterInformation<'a, 'body, 'alloc, 'tcx> { } // We first implement some utility functions which we will expose removing candidates according to -// different needs. Throughout the livenss filtering, the `candidates` are only ever accessed +// different needs. Throughout the liveness filtering, the `candidates` are only ever accessed // through these methods, and not directly. impl<'alloc> Candidates<'alloc> { /// Just `Vec::retain`, but the condition is inverted and we add debugging output @@ -582,10 +582,9 @@ impl WriteInfo { | StatementKind::Nop | StatementKind::Coverage(_) | StatementKind::StorageLive(_) - | StatementKind::StorageDead(_) => (), - StatementKind::FakeRead(_) - | StatementKind::AscribeUserType(_, _) - | StatementKind::PlaceMention(_) => { + | StatementKind::StorageDead(_) + | StatementKind::PlaceMention(_) => (), + StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => { bug!("{:?} not found in this MIR phase", statement) } } diff --git a/compiler/rustc_mir_transform/src/dump_mir.rs b/compiler/rustc_mir_transform/src/dump_mir.rs index 594cbd897..746e3d965 100644 --- a/compiler/rustc_mir_transform/src/dump_mir.rs +++ b/compiler/rustc_mir_transform/src/dump_mir.rs @@ -12,7 +12,7 @@ use rustc_session::config::OutputType; pub struct Marker(pub &'static str); impl<'tcx> MirPass<'tcx> for Marker { - fn name(&self) -> &str { + fn name(&self) -> &'static str { self.0 } diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs index 856234994..f31653caa 100644 --- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs +++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs @@ -4,7 +4,7 @@ use crate::MirPass; use rustc_hir::def_id::DefId; -use rustc_index::vec::Idx; +use rustc_index::Idx; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::visit::MutVisitor; use rustc_middle::mir::*; diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs index a702113bd..fda0e1023 100644 --- a/compiler/rustc_mir_transform/src/elaborate_drops.rs +++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs @@ -1,7 +1,7 @@ use crate::deref_separator::deref_finder; use crate::MirPass; -use rustc_data_structures::fx::FxHashMap; use rustc_index::bit_set::BitSet; +use rustc_index::IndexVec; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt}; @@ -14,7 +14,7 @@ use rustc_mir_dataflow::un_derefer::UnDerefer; use rustc_mir_dataflow::MoveDataParamEnv; use rustc_mir_dataflow::{on_all_children_bits, on_all_drop_children_bits}; use rustc_mir_dataflow::{Analysis, ResultsCursor}; -use rustc_span::{DesugaringKind, Span}; +use rustc_span::Span; use rustc_target::abi::{FieldIdx, VariantIdx}; use std::fmt; @@ -24,7 +24,7 @@ use std::fmt; /// In general, the compiler cannot determine at compile time whether a destructor will run or not. /// /// At a high level, this pass refines Drop to only run the destructor if the -/// target is initialized. The way this is achievied is by inserting drop flags for every variable +/// target is initialized. The way this is achieved is by inserting drop flags for every variable /// that may be dropped, and then using those flags to determine whether a destructor should run. /// Once this is complete, Drop terminators in the MIR correspond to a call to the "drop glue" or /// "drop shim" for the type of the dropped place. @@ -84,12 +84,13 @@ impl<'tcx> MirPass<'tcx> for ElaborateDrops { let reachable = traversal::reachable_as_bitset(body); + let drop_flags = IndexVec::from_elem(None, &env.move_data.move_paths); ElaborateDropsCtxt { tcx, body, env: &env, init_data: InitializationData { inits, uninits }, - drop_flags: Default::default(), + drop_flags, patch: MirPatch::new(body), un_derefer: un_derefer, reachable, @@ -293,7 +294,7 @@ struct ElaborateDropsCtxt<'a, 'tcx> { body: &'a Body<'tcx>, env: &'a MoveDataParamEnv<'tcx>, init_data: InitializationData<'a, 'tcx>, - drop_flags: FxHashMap<MovePathIndex, Local>, + drop_flags: IndexVec<MovePathIndex, Option<Local>>, patch: MirPatch<'tcx>, un_derefer: UnDerefer<'tcx>, reachable: BitSet<BasicBlock>, @@ -312,11 +313,11 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { let tcx = self.tcx; let patch = &mut self.patch; debug!("create_drop_flag({:?})", self.body.span); - self.drop_flags.entry(index).or_insert_with(|| patch.new_internal(tcx.types.bool, span)); + self.drop_flags[index].get_or_insert_with(|| patch.new_internal(tcx.types.bool, span)); } fn drop_flag(&mut self, index: MovePathIndex) -> Option<Place<'tcx>> { - self.drop_flags.get(&index).map(|t| Place::from(*t)) + self.drop_flags[index].map(Place::from) } /// create a patch that elaborates all drops in the input @@ -365,7 +366,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { if maybe_dead { self.tcx.sess.delay_span_bug( terminator.source_info.span, - &format!( + format!( "drop of untracked, uninitialized value {:?}, place {:?} ({:?})", bb, place, path ), @@ -400,7 +401,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { let terminator = data.terminator(); match terminator.kind { - TerminatorKind::Drop { mut place, target, unwind } => { + TerminatorKind::Drop { mut place, target, unwind, replace } => { if let Some(new_place) = self.un_derefer.derefer(place.as_ref(), self.body) { place = new_place; } @@ -433,13 +434,10 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { ) } LookupResult::Parent(..) => { - if !matches!( - terminator.source_info.span.desugaring_kind(), - Some(DesugaringKind::Replace), - ) { + if !replace { self.tcx.sess.delay_span_bug( terminator.source_info.span, - &format!("drop of untracked value {:?}", bb), + format!("drop of untracked value {:?}", bb), ); } // A drop and replace behind a pointer/array/whatever. @@ -463,7 +461,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { } fn set_drop_flag(&mut self, loc: Location, path: MovePathIndex, val: DropFlagState) { - if let Some(&flag) = self.drop_flags.get(&path) { + if let Some(flag) = self.drop_flags[path] { let span = self.patch.source_info_for_location(self.body, loc).span; let val = self.constant_bool(span, val.value()); self.patch.add_assign(loc, Place::from(flag), val); @@ -474,7 +472,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> { let loc = Location::START; let span = self.patch.source_info_for_location(self.body, loc).span; let false_ = self.constant_bool(span, false); - for flag in self.drop_flags.values() { + for flag in self.drop_flags.iter().flatten() { self.patch.add_assign(loc, Place::from(*flag), false_.clone()); } } diff --git a/compiler/rustc_mir_transform/src/errors.rs b/compiler/rustc_mir_transform/src/errors.rs new file mode 100644 index 000000000..602e40d51 --- /dev/null +++ b/compiler/rustc_mir_transform/src/errors.rs @@ -0,0 +1,245 @@ +use rustc_errors::{ + DecorateLint, DiagnosticBuilder, DiagnosticMessage, EmissionGuarantee, Handler, IntoDiagnostic, +}; +use rustc_macros::{Diagnostic, LintDiagnostic, Subdiagnostic}; +use rustc_middle::mir::{AssertKind, UnsafetyViolationDetails}; +use rustc_session::lint::{self, Lint}; +use rustc_span::Span; + +#[derive(LintDiagnostic)] +pub(crate) enum ConstMutate { + #[diag(mir_transform_const_modify)] + #[note] + Modify { + #[note(mir_transform_const_defined_here)] + konst: Span, + }, + #[diag(mir_transform_const_mut_borrow)] + #[note] + #[note(mir_transform_note2)] + MutBorrow { + #[note(mir_transform_note3)] + method_call: Option<Span>, + #[note(mir_transform_const_defined_here)] + konst: Span, + }, +} + +#[derive(Diagnostic)] +#[diag(mir_transform_unaligned_packed_ref, code = "E0793")] +#[note] +#[note(mir_transform_note_ub)] +#[help] +pub(crate) struct UnalignedPackedRef { + #[primary_span] + pub span: Span, +} + +#[derive(LintDiagnostic)] +#[diag(mir_transform_unused_unsafe)] +pub(crate) struct UnusedUnsafe { + #[label(mir_transform_unused_unsafe)] + pub span: Span, + #[label] + pub nested_parent: Option<Span>, +} + +pub(crate) struct RequiresUnsafe { + pub span: Span, + pub details: RequiresUnsafeDetail, + pub enclosing: Option<Span>, + pub op_in_unsafe_fn_allowed: bool, +} + +// The primary message for this diagnostic should be '{$label} is unsafe and...', +// so we need to eagerly translate the label here, which isn't supported by the derive API +// We could also exhaustively list out the primary messages for all unsafe violations, +// but this would result in a lot of duplication. +impl<'sess, G: EmissionGuarantee> IntoDiagnostic<'sess, G> for RequiresUnsafe { + #[track_caller] + fn into_diagnostic(self, handler: &'sess Handler) -> DiagnosticBuilder<'sess, G> { + let mut diag = + handler.struct_diagnostic(crate::fluent_generated::mir_transform_requires_unsafe); + diag.code(rustc_errors::DiagnosticId::Error("E0133".to_string())); + diag.set_span(self.span); + diag.span_label(self.span, self.details.label()); + diag.note(self.details.note()); + let desc = handler.eagerly_translate_to_string(self.details.label(), [].into_iter()); + diag.set_arg("details", desc); + diag.set_arg("op_in_unsafe_fn_allowed", self.op_in_unsafe_fn_allowed); + if let Some(sp) = self.enclosing { + diag.span_label(sp, crate::fluent_generated::mir_transform_not_inherited); + } + diag + } +} + +#[derive(Copy, Clone)] +pub(crate) struct RequiresUnsafeDetail { + pub span: Span, + pub violation: UnsafetyViolationDetails, +} + +impl RequiresUnsafeDetail { + fn note(self) -> DiagnosticMessage { + use UnsafetyViolationDetails::*; + match self.violation { + CallToUnsafeFunction => crate::fluent_generated::mir_transform_call_to_unsafe_note, + UseOfInlineAssembly => crate::fluent_generated::mir_transform_use_of_asm_note, + InitializingTypeWith => { + crate::fluent_generated::mir_transform_initializing_valid_range_note + } + CastOfPointerToInt => crate::fluent_generated::mir_transform_const_ptr2int_note, + UseOfMutableStatic => crate::fluent_generated::mir_transform_use_of_static_mut_note, + UseOfExternStatic => crate::fluent_generated::mir_transform_use_of_extern_static_note, + DerefOfRawPointer => crate::fluent_generated::mir_transform_deref_ptr_note, + AccessToUnionField => crate::fluent_generated::mir_transform_union_access_note, + MutationOfLayoutConstrainedField => { + crate::fluent_generated::mir_transform_mutation_layout_constrained_note + } + BorrowOfLayoutConstrainedField => { + crate::fluent_generated::mir_transform_mutation_layout_constrained_borrow_note + } + CallToFunctionWith => crate::fluent_generated::mir_transform_target_feature_call_note, + } + } + + fn label(self) -> DiagnosticMessage { + use UnsafetyViolationDetails::*; + match self.violation { + CallToUnsafeFunction => crate::fluent_generated::mir_transform_call_to_unsafe_label, + UseOfInlineAssembly => crate::fluent_generated::mir_transform_use_of_asm_label, + InitializingTypeWith => { + crate::fluent_generated::mir_transform_initializing_valid_range_label + } + CastOfPointerToInt => crate::fluent_generated::mir_transform_const_ptr2int_label, + UseOfMutableStatic => crate::fluent_generated::mir_transform_use_of_static_mut_label, + UseOfExternStatic => crate::fluent_generated::mir_transform_use_of_extern_static_label, + DerefOfRawPointer => crate::fluent_generated::mir_transform_deref_ptr_label, + AccessToUnionField => crate::fluent_generated::mir_transform_union_access_label, + MutationOfLayoutConstrainedField => { + crate::fluent_generated::mir_transform_mutation_layout_constrained_label + } + BorrowOfLayoutConstrainedField => { + crate::fluent_generated::mir_transform_mutation_layout_constrained_borrow_label + } + CallToFunctionWith => crate::fluent_generated::mir_transform_target_feature_call_label, + } + } +} + +pub(crate) struct UnsafeOpInUnsafeFn { + pub details: RequiresUnsafeDetail, +} + +impl<'a> DecorateLint<'a, ()> for UnsafeOpInUnsafeFn { + #[track_caller] + fn decorate_lint<'b>( + self, + diag: &'b mut DiagnosticBuilder<'a, ()>, + ) -> &'b mut DiagnosticBuilder<'a, ()> { + let desc = diag + .handler() + .expect("lint should not yet be emitted") + .eagerly_translate_to_string(self.details.label(), [].into_iter()); + diag.set_arg("details", desc); + diag.span_label(self.details.span, self.details.label()); + diag.note(self.details.note()); + diag + } + + fn msg(&self) -> DiagnosticMessage { + crate::fluent_generated::mir_transform_unsafe_op_in_unsafe_fn + } +} + +pub(crate) enum AssertLint<P> { + ArithmeticOverflow(Span, AssertKind<P>), + UnconditionalPanic(Span, AssertKind<P>), +} + +impl<'a, P: std::fmt::Debug> DecorateLint<'a, ()> for AssertLint<P> { + fn decorate_lint<'b>( + self, + diag: &'b mut DiagnosticBuilder<'a, ()>, + ) -> &'b mut DiagnosticBuilder<'a, ()> { + diag.span_label(self.span(), format!("{:?}", self.panic())); + diag + } + + fn msg(&self) -> DiagnosticMessage { + match self { + AssertLint::ArithmeticOverflow(..) => { + crate::fluent_generated::mir_transform_arithmetic_overflow + } + AssertLint::UnconditionalPanic(..) => { + crate::fluent_generated::mir_transform_operation_will_panic + } + } + } +} + +impl<P> AssertLint<P> { + pub fn lint(&self) -> &'static Lint { + match self { + AssertLint::ArithmeticOverflow(..) => lint::builtin::ARITHMETIC_OVERFLOW, + AssertLint::UnconditionalPanic(..) => lint::builtin::UNCONDITIONAL_PANIC, + } + } + pub fn span(&self) -> Span { + match self { + AssertLint::ArithmeticOverflow(sp, _) | AssertLint::UnconditionalPanic(sp, _) => *sp, + } + } + pub fn panic(&self) -> &AssertKind<P> { + match self { + AssertLint::ArithmeticOverflow(_, p) | AssertLint::UnconditionalPanic(_, p) => p, + } + } +} + +#[derive(LintDiagnostic)] +#[diag(mir_transform_ffi_unwind_call)] +pub(crate) struct FfiUnwindCall { + #[label(mir_transform_ffi_unwind_call)] + pub span: Span, + pub foreign: bool, +} + +#[derive(LintDiagnostic)] +#[diag(mir_transform_fn_item_ref)] +pub(crate) struct FnItemRef { + #[suggestion(code = "{sugg}", applicability = "unspecified")] + pub span: Span, + pub sugg: String, + pub ident: String, +} + +#[derive(LintDiagnostic)] +#[diag(mir_transform_must_not_suspend)] +pub(crate) struct MustNotSupend<'a> { + #[label] + pub yield_sp: Span, + #[subdiagnostic] + pub reason: Option<MustNotSuspendReason>, + #[help] + pub src_sp: Span, + pub pre: &'a str, + pub def_path: String, + pub post: &'a str, +} + +#[derive(Subdiagnostic)] +#[note(mir_transform_note)] +pub(crate) struct MustNotSuspendReason { + #[primary_span] + pub span: Span, + pub reason: String, +} + +#[derive(Diagnostic)] +#[diag(mir_transform_simd_shuffle_last_const)] +pub(crate) struct SimdShuffleLastConst { + #[primary_span] + pub span: Span, +} diff --git a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs index c9b24adba..58cc161dd 100644 --- a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs +++ b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs @@ -1,13 +1,15 @@ use rustc_hir::def_id::{LocalDefId, LOCAL_CRATE}; use rustc_middle::mir::*; use rustc_middle::query::LocalCrate; +use rustc_middle::query::Providers; use rustc_middle::ty::layout; -use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt}; use rustc_session::lint::builtin::FFI_UNWIND_CALLS; use rustc_target::spec::abi::Abi; use rustc_target::spec::PanicStrategy; +use crate::errors; + fn abi_can_unwind(abi: Abi) -> bool { use Abi::*; match abi { @@ -48,7 +50,7 @@ fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool { return false; } - let body = &*tcx.mir_built(ty::WithOptConstParam::unknown(local_def_id)).borrow(); + let body = &*tcx.mir_built(local_def_id).borrow(); let body_ty = tcx.type_of(def_id).skip_binder(); let body_abi = match body_ty.kind() { @@ -107,13 +109,13 @@ fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool { .lint_root; let span = terminator.source_info.span; - let msg = match fn_def_id { - Some(_) => "call to foreign function with FFI-unwind ABI", - None => "call to function pointer with FFI-unwind ABI", - }; - tcx.struct_span_lint_hir(FFI_UNWIND_CALLS, lint_root, span, msg, |lint| { - lint.span_label(span, msg) - }); + let foreign = fn_def_id.is_some(); + tcx.emit_spanned_lint( + FFI_UNWIND_CALLS, + lint_root, + span, + errors::FfiUnwindCall { span, foreign }, + ); tainted = true; } diff --git a/compiler/rustc_mir_transform/src/function_item_references.rs b/compiler/rustc_mir_transform/src/function_item_references.rs index 8601c1b2d..5989dbebf 100644 --- a/compiler/rustc_mir_transform/src/function_item_references.rs +++ b/compiler/rustc_mir_transform/src/function_item_references.rs @@ -1,14 +1,13 @@ use itertools::Itertools; -use rustc_errors::Applicability; use rustc_hir::def_id::DefId; use rustc_middle::mir::visit::Visitor; use rustc_middle::mir::*; -use rustc_middle::ty::{self, EarlyBinder, GenericArgKind, PredicateKind, SubstsRef, Ty, TyCtxt}; +use rustc_middle::ty::{self, EarlyBinder, PredicateKind, SubstsRef, Ty, TyCtxt}; use rustc_session::lint::builtin::FUNCTION_ITEM_REFERENCES; use rustc_span::{symbol::sym, Span}; use rustc_target::spec::abi::Abi; -use crate::MirLint; +use crate::{errors, MirLint}; pub struct FunctionItemReferences; @@ -45,14 +44,12 @@ impl<'tcx> Visitor<'tcx> for FunctionItemRefChecker<'_, 'tcx> { // Handle calls to `transmute` if self.tcx.is_diagnostic_item(sym::transmute, def_id) { let arg_ty = args[0].ty(self.body, self.tcx); - for generic_inner_ty in arg_ty.walk() { - if let GenericArgKind::Type(inner_ty) = generic_inner_ty.unpack() { - if let Some((fn_id, fn_substs)) = - FunctionItemRefChecker::is_fn_ref(inner_ty) - { - let span = self.nth_arg_span(&args, 0); - self.emit_lint(fn_id, fn_substs, source_info, span); - } + for inner_ty in arg_ty.walk().filter_map(|arg| arg.as_type()) { + if let Some((fn_id, fn_substs)) = + FunctionItemRefChecker::is_fn_ref(inner_ty) + { + let span = self.nth_arg_span(&args, 0); + self.emit_lint(fn_id, fn_substs, source_info, span); } } } else { @@ -82,24 +79,22 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { let arg_defs = self.tcx.fn_sig(def_id).subst_identity().skip_binder().inputs(); for (arg_num, arg_def) in arg_defs.iter().enumerate() { // For all types reachable from the argument type in the fn sig - for generic_inner_ty in arg_def.walk() { - if let GenericArgKind::Type(inner_ty) = generic_inner_ty.unpack() { - // If the inner type matches the type bound by `Pointer` - if inner_ty == bound_ty { - // Do a substitution using the parameters from the callsite - let subst_ty = EarlyBinder(inner_ty).subst(self.tcx, substs_ref); - if let Some((fn_id, fn_substs)) = - FunctionItemRefChecker::is_fn_ref(subst_ty) - { - let mut span = self.nth_arg_span(args, arg_num); - if span.from_expansion() { - // The operand's ctxt wouldn't display the lint since it's inside a macro so - // we have to use the callsite's ctxt. - let callsite_ctxt = span.source_callsite().ctxt(); - span = span.with_ctxt(callsite_ctxt); - } - self.emit_lint(fn_id, fn_substs, source_info, span); + for inner_ty in arg_def.walk().filter_map(|arg| arg.as_type()) { + // If the inner type matches the type bound by `Pointer` + if inner_ty == bound_ty { + // Do a substitution using the parameters from the callsite + let subst_ty = EarlyBinder(inner_ty).subst(self.tcx, substs_ref); + if let Some((fn_id, fn_substs)) = + FunctionItemRefChecker::is_fn_ref(subst_ty) + { + let mut span = self.nth_arg_span(args, arg_num); + if span.from_expansion() { + // The operand's ctxt wouldn't display the lint since it's inside a macro so + // we have to use the callsite's ctxt. + let callsite_ctxt = span.source_callsite().ctxt(); + span = span.with_ctxt(callsite_ctxt); } + self.emit_lint(fn_id, fn_substs, source_info, span); } } } @@ -178,27 +173,21 @@ impl<'tcx> FunctionItemRefChecker<'_, 'tcx> { let num_args = fn_sig.inputs().map_bound(|inputs| inputs.len()).skip_binder(); let variadic = if fn_sig.c_variadic() { ", ..." } else { "" }; let ret = if fn_sig.output().skip_binder().is_unit() { "" } else { " -> _" }; - self.tcx.struct_span_lint_hir( + let sugg = format!( + "{} as {}{}fn({}{}){}", + if params.is_empty() { ident.clone() } else { format!("{}::<{}>", ident, params) }, + unsafety, + abi, + vec!["_"; num_args].join(", "), + variadic, + ret, + ); + + self.tcx.emit_spanned_lint( FUNCTION_ITEM_REFERENCES, lint_root, span, - "taking a reference to a function item does not give a function pointer", - |lint| { - lint.span_suggestion( - span, - format!("cast `{}` to obtain a function pointer", ident), - format!( - "{} as {}{}fn({}{}){}", - if params.is_empty() { ident } else { format!("{}::<{}>", ident, params) }, - unsafety, - abi, - vec!["_"; num_args].join(", "), - variadic, - ret, - ), - Applicability::Unspecified, - ) - }, + errors::FnItemRef { span, sugg, ident }, ); } } diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs index 4c4423721..89567ed0a 100644 --- a/compiler/rustc_mir_transform/src/generator.rs +++ b/compiler/rustc_mir_transform/src/generator.rs @@ -51,6 +51,7 @@ //! Otherwise it drops all the values in scope at the last suspension point. use crate::deref_separator::deref_finder; +use crate::errors; use crate::simplify; use crate::MirPass; use rustc_data_structures::fx::{FxHashMap, FxHashSet}; @@ -59,7 +60,7 @@ use rustc_hir as hir; use rustc_hir::lang_items::LangItem; use rustc_hir::GeneratorKind; use rustc_index::bit_set::{BitMatrix, BitSet, GrowableBitSet}; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::{Idx, IndexVec}; use rustc_middle::mir::dump_mir; use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::*; @@ -865,7 +866,7 @@ fn sanitize_witness<'tcx>( _ => { tcx.sess.delay_span_bug( body.span, - &format!("unexpected generator witness type {:?}", witness.kind()), + format!("unexpected generator witness type {:?}", witness.kind()), ); return; } @@ -1044,7 +1045,10 @@ fn elaborate_generator_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { for (block, block_data) in body.basic_blocks.iter_enumerated() { let (target, unwind, source_info) = match block_data.terminator() { - Terminator { source_info, kind: TerminatorKind::Drop { place, target, unwind } } => { + Terminator { + source_info, + kind: TerminatorKind::Drop { place, target, unwind, replace: _ }, + } => { if let Some(local) = place.as_local() { if local == SELF_ARG { (target, unwind, source_info) @@ -1150,7 +1154,7 @@ fn insert_panic_block<'tcx>( literal: ConstantKind::from_bool(tcx, false), })), expected: true, - msg: message, + msg: Box::new(message), target: assert_block, unwind: UnwindAction::Continue, }; @@ -1303,6 +1307,7 @@ fn insert_clean_drop(body: &mut Body<'_>) -> BasicBlock { place: Place::from(SELF_ARG), target: return_block, unwind: UnwindAction::Continue, + replace: false, }; let source_info = SourceInfo::outermost(body.span); @@ -1396,10 +1401,10 @@ fn create_cases<'tcx>( pub(crate) fn mir_generator_witnesses<'tcx>( tcx: TyCtxt<'tcx>, def_id: LocalDefId, -) -> GeneratorLayout<'tcx> { +) -> Option<GeneratorLayout<'tcx>> { assert!(tcx.sess.opts.unstable_opts.drop_tracking_mir); - let (body, _) = tcx.mir_promoted(ty::WithOptConstParam::unknown(def_id)); + let (body, _) = tcx.mir_promoted(def_id); let body = body.borrow(); let body = &*body; @@ -1409,6 +1414,7 @@ pub(crate) fn mir_generator_witnesses<'tcx>( // Get the interior types and substs which typeck computed let movable = match *gen_ty.kind() { ty::Generator(_, _, movability) => movability == hir::Movability::Movable, + ty::Error(_) => return None, _ => span_bug!(body.span, "unexpected generator type {}", gen_ty), }; @@ -1424,7 +1430,7 @@ pub(crate) fn mir_generator_witnesses<'tcx>( check_suspend_tys(tcx, &generator_layout, &body); - generator_layout + Some(generator_layout) } impl<'tcx> MirPass<'tcx> for StateTransform { @@ -1451,8 +1457,7 @@ impl<'tcx> MirPass<'tcx> for StateTransform { ) } _ => { - tcx.sess - .delay_span_bug(body.span, &format!("unexpected generator type {}", gen_ty)); + tcx.sess.delay_span_bug(body.span, format!("unexpected generator type {}", gen_ty)); return; } }; @@ -1800,7 +1805,7 @@ fn check_must_not_suspend_ty<'tcx>( // FIXME: support adding the attribute to TAITs ty::Alias(ty::Opaque, ty::AliasTy { def_id: def, .. }) => { let mut has_emitted = false; - for &(predicate, _) in tcx.explicit_item_bounds(def) { + for &(predicate, _) in tcx.explicit_item_bounds(def).skip_binder() { // We only look at the `DefId`, so it is safe to skip the binder here. if let ty::PredicateKind::Clause(ty::Clause::Trait(ref poly_trait_predicate)) = predicate.kind().skip_binder() @@ -1869,7 +1874,7 @@ fn check_must_not_suspend_ty<'tcx>( }, ) } - // If drop tracking is enabled, we want to look through references, since the referrent + // If drop tracking is enabled, we want to look through references, since the referent // may not be considered live across the await point. ty::Ref(_region, ty, _mutability) => { let descr_pre = &format!("{}reference{} to ", data.descr_pre, plural_suffix); @@ -1892,36 +1897,21 @@ fn check_must_not_suspend_def( data: SuspendCheckData<'_>, ) -> bool { if let Some(attr) = tcx.get_attr(def_id, sym::must_not_suspend) { - let msg = rustc_errors::DelayDm(|| { - format!( - "{}`{}`{} held across a suspend point, but should not be", - data.descr_pre, - tcx.def_path_str(def_id), - data.descr_post, - ) + let reason = attr.value_str().map(|s| errors::MustNotSuspendReason { + span: data.source_span, + reason: s.as_str().to_string(), }); - tcx.struct_span_lint_hir( + tcx.emit_spanned_lint( rustc_session::lint::builtin::MUST_NOT_SUSPEND, hir_id, data.source_span, - msg, - |lint| { - // add span pointing to the offending yield/await - lint.span_label(data.yield_span, "the value is held across this suspend point"); - - // Add optional reason note - if let Some(note) = attr.value_str() { - // FIXME(guswynn): consider formatting this better - lint.span_note(data.source_span, note.as_str()); - } - - // Add some quick suggestions on what to do - // FIXME: can `drop` work as a suggestion here as well? - lint.span_help( - data.source_span, - "consider using a block (`{ ... }`) \ - to shrink the value's scope, ending before the suspend point", - ) + errors::MustNotSupend { + yield_sp: data.yield_span, + reason, + src_sp: data.source_span, + pre: data.descr_pre, + def_path: tcx.def_path_str(def_id), + post: data.descr_post, }, ); diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index f0cb317f4..233de6dc6 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -3,13 +3,13 @@ use crate::deref_separator::deref_finder; use rustc_attr::InlineAttr; use rustc_hir::def_id::DefId; use rustc_index::bit_set::BitSet; -use rustc_index::vec::Idx; +use rustc_index::Idx; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; +use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; use rustc_session::config::OptLevel; -use rustc_span::{hygiene::ExpnKind, ExpnData, LocalExpnId, Span}; use rustc_target::abi::{FieldIdx, FIRST_VARIANT}; use rustc_target::spec::abi::Abi; @@ -26,8 +26,6 @@ const CALL_PENALTY: usize = 25; const LANDINGPAD_PENALTY: usize = 50; const RESUME_PENALTY: usize = 45; -const UNKNOWN_SIZE_COST: usize = 10; - const TOP_DOWN_DEPTH_LIMIT: usize = 5; pub struct Inline; @@ -169,8 +167,20 @@ impl<'tcx> Inliner<'tcx> { ) -> Result<std::ops::Range<BasicBlock>, &'static str> { let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id()); self.check_codegen_attributes(callsite, callee_attrs)?; + + let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); + let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; + let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; + for arg in args { + if !arg.ty(&caller_body.local_decls, self.tcx).is_sized(self.tcx, self.param_env) { + // We do not allow inlining functions with unsized params. Inlining these functions + // could create unsized locals, which are unsound and being phased out. + return Err("Call has unsized argument"); + } + } + self.check_mir_is_available(caller_body, &callsite.callee)?; - let callee_body = self.tcx.instance_mir(callsite.callee.def); + let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?; self.check_mir_body(callsite, callee_body, callee_attrs)?; if !self.tcx.consider_optimizing(|| { @@ -182,7 +192,7 @@ impl<'tcx> Inliner<'tcx> { let Ok(callee_body) = callsite.callee.try_subst_mir_and_normalize_erasing_regions( self.tcx, self.param_env, - callee_body.clone(), + ty::EarlyBinder(callee_body.clone()), ) else { return Err("failed to normalize callee body"); }; @@ -190,9 +200,6 @@ impl<'tcx> Inliner<'tcx> { // Check call signature compatibility. // Normally, this shouldn't be required, but trait normalization failure can create a // validation ICE. - let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); - let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; - let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; let output_type = callee_body.return_ty(); if !util::is_subtype(self.tcx, self.param_env, output_type, destination_ty) { trace!(?output_type, ?destination_ty); @@ -350,14 +357,8 @@ impl<'tcx> Inliner<'tcx> { callsite: &CallSite<'tcx>, callee_attrs: &CodegenFnAttrs, ) -> Result<(), &'static str> { - match callee_attrs.inline { - InlineAttr::Never => return Err("never inline hint"), - InlineAttr::Always | InlineAttr::Hint => {} - InlineAttr::None => { - if self.tcx.sess.mir_opt_level() <= 2 { - return Err("at mir-opt-level=2, only #[inline] is inlined"); - } - } + if let InlineAttr::Never = callee_attrs.inline { + return Err("never inline hint"); } // Only inline local functions if they would be eligible for cross-crate @@ -436,6 +437,10 @@ impl<'tcx> Inliner<'tcx> { validation: Ok(()), }; + for var_debug_info in callee_body.var_debug_info.iter() { + checker.visit_var_debug_info(var_debug_info); + } + // Traverse the MIR manually so we can account for the effects of inlining on the CFG. let mut work_list = vec![START_BLOCK]; let mut visited = BitSet::new_empty(callee_body.basic_blocks.len()); @@ -448,14 +453,16 @@ impl<'tcx> Inliner<'tcx> { checker.visit_basic_block_data(bb, blk); let term = blk.terminator(); - if let TerminatorKind::Drop { ref place, target, unwind } = term.kind { + if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind { work_list.push(target); // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty); + let ty = callsite + .callee + .subst_mir(self.tcx, ty::EarlyBinder(&place.ty(callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) && let UnwindAction::Cleanup(unwind) = unwind { - work_list.push(unwind); - } + work_list.push(unwind); + } } else if callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set && matches!(term.kind, TerminatorKind::InlineAsm { .. }) { @@ -470,12 +477,6 @@ impl<'tcx> Inliner<'tcx> { } } - // Count up the cost of local variables and temps, if we know the size - // use that, otherwise we use a moderately-large dummy cost. - for v in callee_body.vars_and_temps_iter() { - checker.visit_local_decl(v, &callee_body.local_decls[v]); - } - // Abort if type validation found anything fishy. checker.validation?; @@ -553,16 +554,6 @@ impl<'tcx> Inliner<'tcx> { // Copy the arguments if needed. let args: Vec<_> = self.make_call_args(args, &callsite, caller_body, &callee_body); - let mut expn_data = ExpnData::default( - ExpnKind::Inlined, - callsite.source_info.span, - self.tcx.sess.edition(), - None, - None, - ); - expn_data.def_site = callee_body.span; - let expn_data = - self.tcx.with_stable_hashing_context(|hcx| LocalExpnId::fresh(expn_data, hcx)); let mut integrator = Integrator { args: &args, new_locals: Local::new(caller_body.local_decls.len()).., @@ -574,7 +565,6 @@ impl<'tcx> Inliner<'tcx> { cleanup_block: unwind, in_cleanup_block: false, tcx: self.tcx, - expn_data, always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), }; @@ -770,14 +760,6 @@ impl<'tcx> Inliner<'tcx> { } } -fn type_size_of<'tcx>( - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - ty: Ty<'tcx>, -) -> Option<u64> { - tcx.layout_of(param_env.and(ty)).ok().map(|layout| layout.size.bytes()) -} - /// Verify that the callee body is compatible with the caller. /// /// This visitor mostly computes the inlining cost, @@ -810,7 +792,9 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { match terminator.kind { TerminatorKind::Drop { ref place, unwind, .. } => { // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = self.instance.subst_mir(tcx, &place.ty(self.callee_body, tcx).ty); + let ty = self + .instance + .subst_mir(tcx, ty::EarlyBinder(&place.ty(self.callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) { self.cost += CALL_PENALTY; if let UnwindAction::Cleanup(_) = unwind { @@ -821,7 +805,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { - let fn_ty = self.instance.subst_mir(tcx, &f.literal.ty()); + let fn_ty = self.instance.subst_mir(tcx, ty::EarlyBinder(&f.literal.ty())); self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { // Don't give intrinsics the extra penalty for calls INSTR_COST @@ -851,24 +835,6 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { self.super_terminator(terminator, location); } - /// Count up the cost of local variables and temps, if we know the size - /// use that, otherwise we use a moderately-large dummy cost. - fn visit_local_decl(&mut self, local: Local, local_decl: &LocalDecl<'tcx>) { - let tcx = self.tcx; - let ptr_size = tcx.data_layout.pointer_size.bytes(); - - let ty = self.instance.subst_mir(tcx, &local_decl.ty); - // Cost of the var is the size in machine-words, if we know - // it. - if let Some(size) = type_size_of(tcx, self.param_env, ty) { - self.cost += ((size + ptr_size - 1) / ptr_size) as usize; - } else { - self.cost += UNKNOWN_SIZE_COST; - } - - self.super_local_decl(local, local_decl) - } - /// This method duplicates code from MIR validation in an attempt to detect type mismatches due /// to normalization failure. fn visit_projection_elem( @@ -883,7 +849,16 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { let parent = Place { local, projection: self.tcx.mk_place_elems(proj_base) }; let parent_ty = parent.ty(&self.callee_body.local_decls, self.tcx); let check_equal = |this: &mut Self, f_ty| { - if !util::is_equal_up_to_subtyping(this.tcx, this.param_env, ty, f_ty) { + // Fast path if there is nothing to substitute. + if ty == f_ty { + return; + } + let ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder(&ty)); + let f_ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder(&f_ty)); + if ty == f_ty { + return; + } + if !util::is_subtype(this.tcx, this.param_env, ty, f_ty) { trace!(?ty, ?f_ty); this.validation = Err("failed to normalize projection type"); return; @@ -982,7 +957,6 @@ struct Integrator<'a, 'tcx> { cleanup_block: UnwindAction, in_cleanup_block: bool, tcx: TyCtxt<'tcx>, - expn_data: LocalExpnId, always_live_locals: BitSet<Local>, } @@ -1068,11 +1042,6 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { *scope = self.map_scope(*scope); } - fn visit_span(&mut self, span: &mut Span) { - // Make sure that all spans track the fact that they were inlined. - *span = span.fresh_expansion(self.expn_data); - } - fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) { self.in_cleanup_block = data.is_cleanup; self.super_basic_block_data(block, data); @@ -1164,3 +1133,27 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { } } } + +#[instrument(skip(tcx), level = "debug")] +fn try_instance_mir<'tcx>( + tcx: TyCtxt<'tcx>, + instance: InstanceDef<'tcx>, +) -> Result<&'tcx Body<'tcx>, &'static str> { + match instance { + ty::InstanceDef::DropGlue(_, Some(ty)) => match ty.kind() { + ty::Adt(def, substs) => { + let fields = def.all_fields(); + for field in fields { + let field_ty = field.ty(tcx, substs); + if field_ty.has_param() && field_ty.has_projections() { + return Err("cannot build drop shim for polymorphic type"); + } + } + + Ok(tcx.instance_mir(instance)) + } + _ => Ok(tcx.instance_mir(instance)), + }, + _ => Ok(tcx.instance_mir(instance)), + } +} diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs index 8aa3c23d0..1ccf06f61 100644 --- a/compiler/rustc_mir_transform/src/inline/cycle.rs +++ b/compiler/rustc_mir_transform/src/inline/cycle.rs @@ -13,7 +13,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( tcx: TyCtxt<'tcx>, (root, target): (ty::Instance<'tcx>, LocalDefId), ) -> bool { - trace!(%root, target = %tcx.def_path_str(target.to_def_id())); + trace!(%root, target = %tcx.def_path_str(target)); let param_env = tcx.param_env_reveal_all_normalized(target); assert_ne!( root.def_id().expect_local(), @@ -44,7 +44,11 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( ) -> bool { trace!(%caller); for &(callee, substs) in tcx.mir_inliner_callees(caller.def) { - let Ok(substs) = caller.try_subst_mir_and_normalize_erasing_regions(tcx, param_env, substs) else { + let Ok(substs) = caller.try_subst_mir_and_normalize_erasing_regions( + tcx, + param_env, + ty::EarlyBinder(substs), + ) else { trace!(?caller, ?param_env, ?substs, "cannot normalize, skipping"); continue; }; @@ -92,7 +96,7 @@ pub(crate) fn mir_callgraph_reachable<'tcx>( // FIXME: A not fully substituted drop shim can cause ICEs if one attempts to // have its MIR built. Likely oli-obk just screwed up the `ParamEnv`s, so this // needs some more analysis. - if callee.needs_subst() { + if callee.has_param() { continue; } } @@ -148,8 +152,7 @@ pub(crate) fn mir_inliner_callees<'tcx>( let guard; let body = match (instance, instance.def_id().as_local()) { (InstanceDef::Item(_), Some(def_id)) => { - let def = ty::WithOptConstParam::unknown(def_id); - steal = tcx.mir_promoted(def).0; + steal = tcx.mir_promoted(def_id).0; guard = steal.borrow(); &*guard } diff --git a/compiler/rustc_mir_transform/src/instcombine.rs b/compiler/rustc_mir_transform/src/instsimplify.rs index 3d06a0a49..e4dc61762 100644 --- a/compiler/rustc_mir_transform/src/instcombine.rs +++ b/compiler/rustc_mir_transform/src/instsimplify.rs @@ -1,26 +1,23 @@ //! Performs various peephole optimizations. +use crate::simplify::simplify_duplicate_switch_targets; use crate::MirPass; use rustc_hir::Mutability; -use rustc_middle::mir::{ - BinOp, Body, CastKind, Constant, ConstantKind, LocalDecls, Operand, Place, ProjectionElem, - Rvalue, SourceInfo, Statement, StatementKind, SwitchTargets, Terminator, TerminatorKind, UnOp, -}; +use rustc_middle::mir::*; use rustc_middle::ty::layout::ValidityRequirement; -use rustc_middle::ty::util::IntTypeExt; use rustc_middle::ty::{self, ParamEnv, SubstsRef, Ty, TyCtxt}; use rustc_span::symbol::Symbol; use rustc_target::abi::FieldIdx; -pub struct InstCombine; +pub struct InstSimplify; -impl<'tcx> MirPass<'tcx> for InstCombine { +impl<'tcx> MirPass<'tcx> for InstSimplify { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { sess.mir_opt_level() > 0 } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let ctx = InstCombineContext { + let ctx = InstSimplifyContext { tcx, local_decls: &body.local_decls, param_env: tcx.param_env_reveal_all_normalized(body.source.def_id()), @@ -29,43 +26,43 @@ impl<'tcx> MirPass<'tcx> for InstCombine { for statement in block.statements.iter_mut() { match statement.kind { StatementKind::Assign(box (_place, ref mut rvalue)) => { - ctx.combine_bool_cmp(&statement.source_info, rvalue); - ctx.combine_ref_deref(&statement.source_info, rvalue); - ctx.combine_len(&statement.source_info, rvalue); - ctx.combine_cast(&statement.source_info, rvalue); + ctx.simplify_bool_cmp(&statement.source_info, rvalue); + ctx.simplify_ref_deref(&statement.source_info, rvalue); + ctx.simplify_len(&statement.source_info, rvalue); + ctx.simplify_cast(&statement.source_info, rvalue); } _ => {} } } - ctx.combine_primitive_clone( + ctx.simplify_primitive_clone( &mut block.terminator.as_mut().unwrap(), &mut block.statements, ); - ctx.combine_intrinsic_assert( + ctx.simplify_intrinsic_assert( &mut block.terminator.as_mut().unwrap(), &mut block.statements, ); - ctx.combine_duplicate_switch_targets(&mut block.terminator.as_mut().unwrap()); + simplify_duplicate_switch_targets(block.terminator.as_mut().unwrap()); } } } -struct InstCombineContext<'tcx, 'a> { +struct InstSimplifyContext<'tcx, 'a> { tcx: TyCtxt<'tcx>, local_decls: &'a LocalDecls<'tcx>, param_env: ParamEnv<'tcx>, } -impl<'tcx> InstCombineContext<'tcx, '_> { - fn should_combine(&self, source_info: &SourceInfo, rvalue: &Rvalue<'tcx>) -> bool { +impl<'tcx> InstSimplifyContext<'tcx, '_> { + fn should_simplify(&self, source_info: &SourceInfo, rvalue: &Rvalue<'tcx>) -> bool { self.tcx.consider_optimizing(|| { - format!("InstCombine - Rvalue: {:?} SourceInfo: {:?}", rvalue, source_info) + format!("InstSimplify - Rvalue: {:?} SourceInfo: {:?}", rvalue, source_info) }) } /// Transform boolean comparisons into logical operations. - fn combine_bool_cmp(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { + fn simplify_bool_cmp(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { match rvalue { Rvalue::BinaryOp(op @ (BinOp::Eq | BinOp::Ne), box (a, b)) => { let new = match (op, self.try_eval_bool(a), self.try_eval_bool(b)) { @@ -96,7 +93,7 @@ impl<'tcx> InstCombineContext<'tcx, '_> { _ => None, }; - if let Some(new) = new && self.should_combine(source_info, rvalue) { + if let Some(new) = new && self.should_simplify(source_info, rvalue) { *rvalue = new; } } @@ -111,14 +108,14 @@ impl<'tcx> InstCombineContext<'tcx, '_> { } /// Transform "&(*a)" ==> "a". - fn combine_ref_deref(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { + fn simplify_ref_deref(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { if let Rvalue::Ref(_, _, place) = rvalue { if let Some((base, ProjectionElem::Deref)) = place.as_ref().last_projection() { if rvalue.ty(self.local_decls, self.tcx) != base.ty(self.local_decls, self.tcx).ty { return; } - if !self.should_combine(source_info, rvalue) { + if !self.should_simplify(source_info, rvalue) { return; } @@ -131,11 +128,11 @@ impl<'tcx> InstCombineContext<'tcx, '_> { } /// Transform "Len([_; N])" ==> "N". - fn combine_len(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { + fn simplify_len(&self, source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { if let Rvalue::Len(ref place) = *rvalue { let place_ty = place.ty(self.local_decls, self.tcx).ty; if let ty::Array(_, len) = *place_ty.kind() { - if !self.should_combine(source_info, rvalue) { + if !self.should_simplify(source_info, rvalue) { return; } @@ -146,7 +143,7 @@ impl<'tcx> InstCombineContext<'tcx, '_> { } } - fn combine_cast(&self, _source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { + fn simplify_cast(&self, _source_info: &SourceInfo, rvalue: &mut Rvalue<'tcx>) { if let Rvalue::Cast(kind, operand, cast_ty) = rvalue { let operand_ty = operand.ty(self.local_decls, self.tcx); if operand_ty == *cast_ty { @@ -165,18 +162,6 @@ impl<'tcx> InstCombineContext<'tcx, '_> { return; } - // Transmuting a fieldless enum to its repr is a discriminant read - if let ty::Adt(adt_def, ..) = operand_ty.kind() - && adt_def.is_enum() - && adt_def.is_payloadfree() - && let Some(place) = operand.place() - && let Some(repr_int) = adt_def.repr().int - && repr_int.to_ty(self.tcx) == *cast_ty - { - *rvalue = Rvalue::Discriminant(place); - return; - } - // Transmuting a transparent struct/union to a field's type is a projection if let ty::Adt(adt_def, substs) = operand_ty.kind() && adt_def.repr().transparent() @@ -198,7 +183,7 @@ impl<'tcx> InstCombineContext<'tcx, '_> { } } - fn combine_primitive_clone( + fn simplify_primitive_clone( &self, terminator: &mut Terminator<'tcx>, statements: &mut Vec<Statement<'tcx>>, @@ -241,7 +226,7 @@ impl<'tcx> InstCombineContext<'tcx, '_> { if !self.tcx.consider_optimizing(|| { format!( - "InstCombine - Call: {:?} SourceInfo: {:?}", + "InstSimplify - Call: {:?} SourceInfo: {:?}", (fn_def_id, fn_substs), terminator.source_info ) @@ -264,20 +249,7 @@ impl<'tcx> InstCombineContext<'tcx, '_> { terminator.kind = TerminatorKind::Goto { target: destination_block }; } - fn combine_duplicate_switch_targets(&self, terminator: &mut Terminator<'tcx>) { - let TerminatorKind::SwitchInt { targets, .. } = &mut terminator.kind - else { return }; - - let otherwise = targets.otherwise(); - if targets.iter().any(|t| t.1 == otherwise) { - *targets = SwitchTargets::new( - targets.iter().filter(|t| t.1 != otherwise), - targets.otherwise(), - ); - } - } - - fn combine_intrinsic_assert( + fn simplify_intrinsic_assert( &self, terminator: &mut Terminator<'tcx>, _statements: &mut Vec<Statement<'tcx>>, diff --git a/compiler/rustc_mir_transform/src/large_enums.rs b/compiler/rustc_mir_transform/src/large_enums.rs index 9447a2ff0..430a6f6ce 100644 --- a/compiler/rustc_mir_transform/src/large_enums.rs +++ b/compiler/rustc_mir_transform/src/large_enums.rs @@ -120,7 +120,7 @@ impl EnumSizeOpt { fn optim<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let mut alloc_cache = FxHashMap::default(); let body_did = body.source.def_id(); - let param_env = tcx.param_env(body_did); + let param_env = tcx.param_env_reveal_all_normalized(body_did); let blocks = body.basic_blocks.as_mut(); let local_decls = &mut body.local_decls; diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index fc12d423c..65864dc01 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -1,4 +1,6 @@ #![allow(rustc::potential_query_instability)] +#![deny(rustc::untranslatable_diagnostic)] +#![deny(rustc::diagnostic_outside_of_impl)] #![feature(box_patterns)] #![feature(drain_filter)] #![feature(let_chains)] @@ -23,18 +25,19 @@ use rustc_data_structures::fx::FxIndexSet; use rustc_data_structures::steal::Steal; use rustc_hir as hir; use rustc_hir::def::DefKind; -use rustc_hir::def_id::{DefId, LocalDefId}; +use rustc_hir::def_id::LocalDefId; use rustc_hir::intravisit::{self, Visitor}; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir::visit::Visitor as _; use rustc_middle::mir::{ traversal, AnalysisPhase, Body, ClearCrossCrate, ConstQualifs, Constant, LocalDecl, MirPass, MirPhase, Operand, Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, - Statement, StatementKind, TerminatorKind, + Statement, StatementKind, TerminatorKind, START_BLOCK, }; -use rustc_middle::ty::query::Providers; +use rustc_middle::query::Providers; use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt}; use rustc_span::sym; +use rustc_trait_selection::traits; #[macro_use] mod pass_manager; @@ -48,6 +51,7 @@ mod add_retag; mod check_const_item_mutation; mod check_packed_ref; pub mod check_unsafety; +mod remove_place_mention; // This pass is public to allow external drivers to perform MIR cleanup pub mod cleanup_post_borrowck; mod const_debuginfo; @@ -67,11 +71,12 @@ pub mod dump_mir; mod early_otherwise_branch; mod elaborate_box_derefs; mod elaborate_drops; +mod errors; mod ffi_unwind_calls; mod function_item_references; mod generator; mod inline; -mod instcombine; +mod instsimplify; mod large_enums; mod lower_intrinsics; mod lower_slice_len; @@ -79,6 +84,7 @@ mod match_branches; mod multiple_return_terminators; mod normalize_array_len; mod nrvo; +mod ref_prop; mod remove_noop_landing_pads; mod remove_storage_markers; mod remove_uninit_drops; @@ -103,6 +109,11 @@ use rustc_const_eval::transform::promote_consts; use rustc_const_eval::transform::validate; use rustc_mir_dataflow::rustc_peek; +use rustc_errors::{DiagnosticMessage, SubdiagnosticMessage}; +use rustc_fluent_macro::fluent_messages; + +fluent_messages! { "../messages.ftl" } + pub fn provide(providers: &mut Providers) { check_unsafety::provide(providers); coverage::query::provide(providers); @@ -111,36 +122,17 @@ pub fn provide(providers: &mut Providers) { *providers = Providers { mir_keys, mir_const, - mir_const_qualif: |tcx, def_id| { - if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { - tcx.mir_const_qualif_const_arg(def) - } else { - mir_const_qualif(tcx, ty::WithOptConstParam::unknown(def_id)) - } - }, - mir_const_qualif_const_arg: |tcx, (did, param_did)| { - mir_const_qualif(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) }) - }, + mir_const_qualif, mir_promoted, mir_drops_elaborated_and_const_checked, mir_for_ctfe, - mir_for_ctfe_of_const_arg, mir_generator_witnesses: generator::mir_generator_witnesses, optimized_mir, is_mir_available, is_ctfe_mir_available: |tcx, did| is_mir_available(tcx, did), mir_callgraph_reachable: inline::cycle::mir_callgraph_reachable, mir_inliner_callees: inline::cycle::mir_inliner_callees, - promoted_mir: |tcx, def_id| { - if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { - tcx.promoted_mir_of_const_arg(def) - } else { - promoted_mir(tcx, ty::WithOptConstParam::unknown(def_id)) - } - }, - promoted_mir_of_const_arg: |tcx, (did, param_did)| { - promoted_mir(tcx, ty::WithOptConstParam { did, const_param_did: Some(param_did) }) - }, + promoted_mir, deduced_param_attrs: deduce_param_attrs::deduced_param_attrs, ..*providers }; @@ -234,8 +226,8 @@ fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> { set } -fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> ConstQualifs { - let const_kind = tcx.hir().body_const_context(def.did); +fn mir_const_qualif(tcx: TyCtxt<'_>, def: LocalDefId) -> ConstQualifs { + let const_kind = tcx.hir().body_const_context(def); // No need to const-check a non-const `fn`. if const_kind.is_none() { @@ -253,7 +245,7 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> return Default::default(); } - let ccx = check_consts::ConstCx { body, tcx, const_kind, param_env: tcx.param_env(def.did) }; + let ccx = check_consts::ConstCx { body, tcx, const_kind, param_env: tcx.param_env(def) }; let mut validator = check_consts::check::Checker::new(&ccx); validator.check_body(); @@ -266,22 +258,14 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> /// Make MIR ready for const evaluation. This is run on all MIR, not just on consts! /// FIXME(oli-obk): it's unclear whether we still need this phase (and its corresponding query). /// We used to have this for pre-miri MIR based const eval. -fn mir_const(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> &Steal<Body<'_>> { - if let Some(def) = def.try_upgrade(tcx) { - return tcx.mir_const(def); - } - +fn mir_const(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> { // Unsafety check uses the raw mir, so make sure it is run. if !tcx.sess.opts.unstable_opts.thir_unsafeck { - if let Some(param_did) = def.const_param_did { - tcx.ensure_with_value().unsafety_check_result_for_const_arg((def.did, param_did)); - } else { - tcx.ensure_with_value().unsafety_check_result(def.did); - } + tcx.ensure_with_value().unsafety_check_result(def); } // has_ffi_unwind_calls query uses the raw mir, so make sure it is run. - tcx.ensure_with_value().has_ffi_unwind_calls(def.did); + tcx.ensure_with_value().has_ffi_unwind_calls(def); let mut body = tcx.mir_built(def).steal(); @@ -296,7 +280,7 @@ fn mir_const(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> &Steal< &Lint(check_const_item_mutation::CheckConstItemMutation), &Lint(function_item_references::FunctionItemReferences), // What we need to do constant evaluation. - &simplify::SimplifyCfg::new("initial"), + &simplify::SimplifyCfg::Initial, &rustc_peek::SanityCheck, // Just a lint ], None, @@ -307,16 +291,12 @@ fn mir_const(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> &Steal< /// Compute the main MIR body and the list of MIR bodies of the promoteds. fn mir_promoted( tcx: TyCtxt<'_>, - def: ty::WithOptConstParam<LocalDefId>, + def: LocalDefId, ) -> (&Steal<Body<'_>>, &Steal<IndexVec<Promoted, Body<'_>>>) { - if let Some(def) = def.try_upgrade(tcx) { - return tcx.mir_promoted(def); - } - // Ensure that we compute the `mir_const_qualif` for constants at // this point, before we steal the mir-const result. // Also this means promotion can rely on all const checks having been done. - let const_qualifs = tcx.mir_const_qualif_opt_const_arg(def); + let const_qualifs = tcx.mir_const_qualif(def); let mut body = tcx.mir_const(def).steal(); if let Some(error_reported) = const_qualifs.tainted_by_errors { body.tainted_by_errors = Some(error_reported); @@ -334,11 +314,7 @@ fn mir_promoted( pm::run_passes( tcx, &mut body, - &[ - &promote_pass, - &simplify::SimplifyCfg::new("promote-consts"), - &coverage::InstrumentCoverage, - ], + &[&promote_pass, &simplify::SimplifyCfg::PromoteConsts, &coverage::InstrumentCoverage], Some(MirPhase::Analysis(AnalysisPhase::Initial)), ); @@ -348,38 +324,22 @@ fn mir_promoted( /// Compute the MIR that is used during CTFE (and thus has no optimizations run on it) fn mir_for_ctfe(tcx: TyCtxt<'_>, def_id: LocalDefId) -> &Body<'_> { - if let Some(def) = ty::WithOptConstParam::try_lookup(def_id, tcx) { - tcx.mir_for_ctfe_of_const_arg(def) - } else { - tcx.arena.alloc(inner_mir_for_ctfe(tcx, ty::WithOptConstParam::unknown(def_id))) - } + tcx.arena.alloc(inner_mir_for_ctfe(tcx, def_id)) } -/// Same as `mir_for_ctfe`, but used to get the MIR of a const generic parameter. -/// The docs on `WithOptConstParam` explain this a bit more, but the TLDR is that -/// we'd get cycle errors with `mir_for_ctfe`, because typeck would need to typeck -/// the const parameter while type checking the main body, which in turn would try -/// to type check the main body again. -fn mir_for_ctfe_of_const_arg(tcx: TyCtxt<'_>, (did, param_did): (LocalDefId, DefId)) -> &Body<'_> { - tcx.arena.alloc(inner_mir_for_ctfe( - tcx, - ty::WithOptConstParam { did, const_param_did: Some(param_did) }, - )) -} - -fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> Body<'_> { +fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: LocalDefId) -> Body<'_> { // FIXME: don't duplicate this between the optimized_mir/mir_for_ctfe queries - if tcx.is_constructor(def.did.to_def_id()) { + if tcx.is_constructor(def.to_def_id()) { // There's no reason to run all of the MIR passes on constructors when // we can just output the MIR we want directly. This also saves const // qualification and borrow checking the trouble of special casing // constructors. - return shim::build_adt_ctor(tcx, def.did.to_def_id()); + return shim::build_adt_ctor(tcx, def.to_def_id()); } let context = tcx .hir() - .body_const_context(def.did) + .body_const_context(def) .expect("mir_for_ctfe should not be used for runtime functions"); let body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone(); @@ -417,29 +377,19 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) - /// Obtain just the main MIR (no promoteds) and run some cleanups on it. This also runs /// mir borrowck *before* doing so in order to ensure that borrowck can be run and doesn't /// end up missing the source MIR due to stealing happening. -fn mir_drops_elaborated_and_const_checked( - tcx: TyCtxt<'_>, - def: ty::WithOptConstParam<LocalDefId>, -) -> &Steal<Body<'_>> { - if let Some(def) = def.try_upgrade(tcx) { - return tcx.mir_drops_elaborated_and_const_checked(def); - } - +fn mir_drops_elaborated_and_const_checked(tcx: TyCtxt<'_>, def: LocalDefId) -> &Steal<Body<'_>> { if tcx.sess.opts.unstable_opts.drop_tracking_mir - && let DefKind::Generator = tcx.def_kind(def.did) + && let DefKind::Generator = tcx.def_kind(def) { - tcx.ensure_with_value().mir_generator_witnesses(def.did); + tcx.ensure_with_value().mir_generator_witnesses(def); } - let mir_borrowck = tcx.mir_borrowck_opt_const_arg(def); + let mir_borrowck = tcx.mir_borrowck(def); - let is_fn_like = tcx.def_kind(def.did).is_fn_like(); + let is_fn_like = tcx.def_kind(def).is_fn_like(); if is_fn_like { - let did = def.did.to_def_id(); - let def = ty::WithOptConstParam::unknown(did); - // Do not compute the mir call graph without said call graph actually being used. if inline::Inline.is_enabled(&tcx.sess) { - tcx.ensure_with_value().mir_inliner_callees(ty::InstanceDef::Item(def)); + tcx.ensure_with_value().mir_inliner_callees(ty::InstanceDef::Item(def.to_def_id())); } } @@ -449,6 +399,50 @@ fn mir_drops_elaborated_and_const_checked( body.tainted_by_errors = Some(error_reported); } + // Check if it's even possible to satisfy the 'where' clauses + // for this item. + // + // This branch will never be taken for any normal function. + // However, it's possible to `#!feature(trivial_bounds)]` to write + // a function with impossible to satisfy clauses, e.g.: + // `fn foo() where String: Copy {}` + // + // We don't usually need to worry about this kind of case, + // since we would get a compilation error if the user tried + // to call it. However, since we optimize even without any + // calls to the function, we need to make sure that it even + // makes sense to try to evaluate the body. + // + // If there are unsatisfiable where clauses, then all bets are + // off, and we just give up. + // + // We manually filter the predicates, skipping anything that's not + // "global". We are in a potentially generic context + // (e.g. we are evaluating a function without substituting generic + // parameters, so this filtering serves two purposes: + // + // 1. We skip evaluating any predicates that we would + // never be able prove are unsatisfiable (e.g. `<T as Foo>` + // 2. We avoid trying to normalize predicates involving generic + // parameters (e.g. `<T as Foo>::MyItem`). This can confuse + // the normalization code (leading to cycle errors), since + // it's usually never invoked in this way. + let predicates = tcx + .predicates_of(body.source.def_id()) + .predicates + .iter() + .filter_map(|(p, _)| if p.is_global() { Some(*p) } else { None }); + if traits::impossible_predicates(tcx, traits::elaborate(tcx, predicates).collect()) { + trace!("found unsatisfiable predicates for {:?}", body.source); + // Clear the body to only contain a single `unreachable` statement. + let bbs = body.basic_blocks.as_mut(); + bbs.raw.truncate(1); + bbs[START_BLOCK].statements.clear(); + bbs[START_BLOCK].terminator_mut().kind = TerminatorKind::Unreachable; + body.var_debug_info.clear(); + body.local_decls.raw.truncate(body.arg_count + 1); + } + run_analysis_to_runtime_passes(tcx, &mut body); tcx.alloc_steal_mir(body) @@ -467,10 +461,7 @@ fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx> pm::run_passes( tcx, body, - &[ - &remove_uninit_drops::RemoveUninitDrops, - &simplify::SimplifyCfg::new("remove-false-edges"), - ], + &[&remove_uninit_drops::RemoveUninitDrops, &simplify::SimplifyCfg::RemoveFalseEdges], None, ); check_consts::post_drop_elaboration::check_live_drops(tcx, &body); // FIXME: make this a MIR lint @@ -492,7 +483,7 @@ fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let passes: &[&dyn MirPass<'tcx>] = &[ &cleanup_post_borrowck::CleanupPostBorrowck, &remove_noop_landing_pads::RemoveNoopLandingPads, - &simplify::SimplifyCfg::new("early-opt"), + &simplify::SimplifyCfg::EarlyOpt, &deref_separator::Derefer, ]; @@ -524,8 +515,11 @@ fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { /// Returns the sequence of passes that do the initial cleanup of runtime MIR. fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let passes: &[&dyn MirPass<'tcx>] = - &[&lower_intrinsics::LowerIntrinsics, &simplify::SimplifyCfg::new("elaborate-drops")]; + let passes: &[&dyn MirPass<'tcx>] = &[ + &lower_intrinsics::LowerIntrinsics, + &remove_place_mention::RemovePlaceMention, + &simplify::SimplifyCfg::ElaborateDrops, + ]; pm::run_passes(tcx, body, passes, Some(MirPhase::Runtime(RuntimePhase::PostCleanup))); @@ -551,7 +545,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &lower_slice_len::LowerSliceLenCalls, // has to be done before inlining, otherwise actual call will be almost always inlined. Also simple, so can just do first &unreachable_prop::UnreachablePropagation, &uninhabited_enum_branching::UninhabitedEnumBranching, - &o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")), + &o1(simplify::SimplifyCfg::AfterUninhabitedEnumBranching), &inline::Inline, &remove_storage_markers::RemoveStorageMarkers, &remove_zsts::RemoveZsts, @@ -562,25 +556,26 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &match_branches::MatchBranchSimplification, // inst combine is after MatchBranchSimplification to clean up Ne(_1, false) &multiple_return_terminators::MultipleReturnTerminators, - &instcombine::InstCombine, + &instsimplify::InstSimplify, &separate_const_switch::SeparateConstSwitch, - &simplify::SimplifyLocals::new("before-const-prop"), + &simplify::SimplifyLocals::BeforeConstProp, ©_prop::CopyProp, + &ref_prop::ReferencePropagation, &const_prop::ConstProp, &dataflow_const_prop::DataflowConstProp, // // Const-prop runs unconditionally, but doesn't mutate the MIR at mir-opt-level=0. &const_debuginfo::ConstDebugInfo, - &o1(simplify_branches::SimplifyConstCondition::new("after-const-prop")), + &o1(simplify_branches::SimplifyConstCondition::AfterConstProp), &early_otherwise_branch::EarlyOtherwiseBranch, &simplify_comparison_integral::SimplifyComparisonIntegral, &dead_store_elimination::DeadStoreElimination, &dest_prop::DestinationPropagation, - &o1(simplify_branches::SimplifyConstCondition::new("final")), + &o1(simplify_branches::SimplifyConstCondition::Final), &o1(remove_noop_landing_pads::RemoveNoopLandingPads), - &o1(simplify::SimplifyCfg::new("final")), + &o1(simplify::SimplifyCfg::Final), &nrvo::RenameReturnPlace, - &simplify::SimplifyLocals::new("final"), + &simplify::SimplifyLocals::Final, &multiple_return_terminators::MultipleReturnTerminators, &deduplicate_blocks::DeduplicateBlocks, &large_enums::EnumSizeOpt { discrepancy: 128 }, @@ -595,7 +590,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { /// Optimize the MIR and prepare it for codegen. fn optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> &Body<'_> { - assert_eq!(ty::WithOptConstParam::try_lookup(did, tcx), None); tcx.arena.alloc(inner_optimized_mir(tcx, did)) } @@ -617,8 +611,7 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { Some(other) => panic!("do not use `optimized_mir` for constants: {:?}", other), } debug!("about to call mir_drops_elaborated..."); - let body = - tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).steal(); + let body = tcx.mir_drops_elaborated_and_const_checked(did).steal(); let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::NotConst); debug!("body: {:#?}", body); run_optimization_passes(tcx, &mut body); @@ -628,21 +621,15 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { /// Fetch all the promoteds of an item and prepare their MIR bodies to be ready for /// constant evaluation once all substitutions become known. -fn promoted_mir( - tcx: TyCtxt<'_>, - def: ty::WithOptConstParam<LocalDefId>, -) -> &IndexVec<Promoted, Body<'_>> { - if tcx.is_constructor(def.did.to_def_id()) { +fn promoted_mir(tcx: TyCtxt<'_>, def: LocalDefId) -> &IndexVec<Promoted, Body<'_>> { + if tcx.is_constructor(def.to_def_id()) { return tcx.arena.alloc(IndexVec::new()); } - let tainted_by_errors = tcx.mir_borrowck_opt_const_arg(def).tainted_by_errors; + tcx.ensure_with_value().mir_borrowck(def); let mut promoted = tcx.mir_promoted(def).1.steal(); for body in &mut promoted { - if let Some(error_reported) = tainted_by_errors { - body.tainted_by_errors = Some(error_reported); - } run_analysis_to_runtime_passes(tcx, body); } diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs index c136642df..dae01e41e 100644 --- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs +++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs @@ -1,6 +1,6 @@ //! Lowers intrinsic calls -use crate::MirPass; +use crate::{errors, MirPass}; use rustc_middle::mir::*; use rustc_middle::ty::subst::SubstsRef; use rustc_middle::ty::{self, Ty, TyCtxt}; @@ -179,6 +179,29 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { } } } + sym::write_via_move => { + let target = target.unwrap(); + let Ok([ptr, val]) = <[_; 2]>::try_from(std::mem::take(args)) else { + span_bug!( + terminator.source_info.span, + "Wrong number of arguments for write_via_move intrinsic", + ); + }; + let derefed_place = + if let Some(place) = ptr.place() && let Some(local) = place.as_local() { + tcx.mk_place_deref(local.into()) + } else { + span_bug!(terminator.source_info.span, "Only passing a local is supported"); + }; + block.statements.push(Statement { + source_info: terminator.source_info, + kind: StatementKind::Assign(Box::new(( + derefed_place, + Rvalue::Use(val), + ))), + }); + terminator.kind = TerminatorKind::Goto { target }; + } sym::discriminant_value => { if let (Some(target), Some(arg)) = (*target, args[0].place()) { let arg = tcx.mk_place_deref(arg); @@ -192,6 +215,23 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { terminator.kind = TerminatorKind::Goto { target }; } } + sym::offset => { + let target = target.unwrap(); + let Ok([ptr, delta]) = <[_; 2]>::try_from(std::mem::take(args)) else { + span_bug!( + terminator.source_info.span, + "Wrong number of arguments for offset intrinsic", + ); + }; + block.statements.push(Statement { + source_info: terminator.source_info, + kind: StatementKind::Assign(Box::new(( + *destination, + Rvalue::BinaryOp(BinOp::Offset, Box::new((ptr, delta))), + ))), + }); + terminator.kind = TerminatorKind::Goto { target }; + } sym::option_payload_ptr => { if let (Some(target), Some(arg)) = (*target, args[0].place()) { let ty::RawPtr(ty::TypeAndMut { ty: dest_ty, .. }) = @@ -221,7 +261,7 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics { terminator.kind = TerminatorKind::Goto { target }; } } - sym::transmute => { + sym::transmute | sym::transmute_unchecked => { let dst_ty = destination.ty(local_decls, tcx).ty; let Ok([arg]) = <[_; 1]>::try_from(std::mem::take(args)) else { span_bug!( @@ -270,11 +310,7 @@ fn resolve_rust_intrinsic<'tcx>( } fn validate_simd_shuffle<'tcx>(tcx: TyCtxt<'tcx>, args: &[Operand<'tcx>], span: Span) { - match &args[2] { - Operand::Constant(_) => {} // all good - _ => { - let msg = "last argument of `simd_shuffle` is required to be a `const` item"; - tcx.sess.span_err(span, msg); - } + if !matches!(args[2], Operand::Constant(_)) { + tcx.sess.emit_err(errors::SimdShuffleLastConst { span }); } } diff --git a/compiler/rustc_mir_transform/src/lower_slice_len.rs b/compiler/rustc_mir_transform/src/lower_slice_len.rs index 7dc5878e0..6e40dfa0d 100644 --- a/compiler/rustc_mir_transform/src/lower_slice_len.rs +++ b/compiler/rustc_mir_transform/src/lower_slice_len.rs @@ -3,7 +3,7 @@ use crate::MirPass; use rustc_hir::def_id::DefId; -use rustc_index::vec::IndexSlice; +use rustc_index::IndexSlice; use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt}; diff --git a/compiler/rustc_mir_transform/src/match_branches.rs b/compiler/rustc_mir_transform/src/match_branches.rs index ce05db5b7..59942dc76 100644 --- a/compiler/rustc_mir_transform/src/match_branches.rs +++ b/compiler/rustc_mir_transform/src/match_branches.rs @@ -46,7 +46,7 @@ impl<'tcx> MirPass<'tcx> for MatchBranchSimplification { fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { let def_id = body.source.def_id(); - let param_env = tcx.param_env(def_id); + let param_env = tcx.param_env_reveal_all_normalized(def_id); let bbs = body.basic_blocks.as_mut(); let mut should_cleanup = false; diff --git a/compiler/rustc_mir_transform/src/normalize_array_len.rs b/compiler/rustc_mir_transform/src/normalize_array_len.rs index b36c8a0bd..3d61d33ce 100644 --- a/compiler/rustc_mir_transform/src/normalize_array_len.rs +++ b/compiler/rustc_mir_transform/src/normalize_array_len.rs @@ -3,11 +3,10 @@ use crate::ssa::SsaLocals; use crate::MirPass; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; use rustc_middle::ty::{self, TyCtxt}; -use rustc_mir_dataflow::impls::borrowed_locals; pub struct NormalizeArrayLen; @@ -24,9 +23,7 @@ impl<'tcx> MirPass<'tcx> for NormalizeArrayLen { } fn normalize_array_len_calls<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); - let borrowed_locals = borrowed_locals(body); - let ssa = SsaLocals::new(tcx, param_env, body, &borrowed_locals); + let ssa = SsaLocals::new(body); let slice_lengths = compute_slice_length(tcx, &ssa, body); debug!(?slice_lengths); @@ -41,7 +38,7 @@ fn compute_slice_length<'tcx>( ) -> IndexVec<Local, Option<ty::Const<'tcx>>> { let mut slice_lengths = IndexVec::from_elem(None, &body.local_decls); - for (local, rvalue) in ssa.assignments(body) { + for (local, rvalue, _) in ssa.assignments(body) { match rvalue { Rvalue::Cast( CastKind::Pointer(ty::adjustment::PointerCast::Unsize), diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs index b6e73eaad..5ce96012b 100644 --- a/compiler/rustc_mir_transform/src/nrvo.rs +++ b/compiler/rustc_mir_transform/src/nrvo.rs @@ -34,7 +34,8 @@ pub struct RenameReturnPlace; impl<'tcx> MirPass<'tcx> for RenameReturnPlace { fn is_enabled(&self, sess: &rustc_session::Session) -> bool { - sess.mir_opt_level() > 0 + // #111005 + sess.mir_opt_level() > 0 && sess.opts.unstable_opts.unsound_mir_opts } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut mir::Body<'tcx>) { @@ -107,7 +108,7 @@ fn local_eligible_for_nrvo(body: &mut mir::Body<'_>) -> Option<Local> { // If multiple different locals are copied to the return place. We can't pick a // single one to rename. - if copied_to_return_place.map_or(false, |old| old != returned_local) { + if copied_to_return_place.is_some_and(|old| old != returned_local) { return None; } diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs index e1b65823a..710eed3ed 100644 --- a/compiler/rustc_mir_transform/src/pass_manager.rs +++ b/compiler/rustc_mir_transform/src/pass_manager.rs @@ -6,7 +6,7 @@ use crate::{validate, MirPass}; /// Just like `MirPass`, except it cannot mutate `Body`. pub trait MirLint<'tcx> { - fn name(&self) -> &str { + fn name(&self) -> &'static str { let name = std::any::type_name::<Self>(); if let Some((_, tail)) = name.rsplit_once(':') { tail } else { name } } @@ -26,7 +26,7 @@ impl<'tcx, T> MirPass<'tcx> for Lint<T> where T: MirLint<'tcx>, { - fn name(&self) -> &str { + fn name(&self) -> &'static str { self.0.name() } @@ -49,7 +49,7 @@ impl<'tcx, T> MirPass<'tcx> for WithMinOptLevel<T> where T: MirPass<'tcx>, { - fn name(&self) -> &str { + fn name(&self) -> &'static str { self.1.name() } @@ -121,7 +121,7 @@ fn run_passes_inner<'tcx>( validate_body(tcx, body, format!("before pass {}", name)); } - pass.run_pass(tcx, body); + tcx.sess.time(name, || pass.run_pass(tcx, body)); if dump_enabled { dump_mir_for_pass(tcx, body, &name, true); diff --git a/compiler/rustc_mir_transform/src/ref_prop.rs b/compiler/rustc_mir_transform/src/ref_prop.rs new file mode 100644 index 000000000..bbd9f76ba --- /dev/null +++ b/compiler/rustc_mir_transform/src/ref_prop.rs @@ -0,0 +1,408 @@ +use rustc_data_structures::fx::FxHashSet; +use rustc_index::bit_set::BitSet; +use rustc_index::IndexVec; +use rustc_middle::mir::visit::*; +use rustc_middle::mir::*; +use rustc_middle::ty::TyCtxt; +use rustc_mir_dataflow::impls::MaybeStorageDead; +use rustc_mir_dataflow::storage::always_storage_live_locals; +use rustc_mir_dataflow::Analysis; + +use crate::ssa::{SsaLocals, StorageLiveLocals}; +use crate::MirPass; + +/// Propagate references using SSA analysis. +/// +/// MIR building may produce a lot of borrow-dereference patterns. +/// +/// This pass aims to transform the following pattern: +/// _1 = &raw? mut? PLACE; +/// _3 = *_1; +/// _4 = &raw? mut? *_1; +/// +/// Into +/// _1 = &raw? mut? PLACE; +/// _3 = PLACE; +/// _4 = &raw? mut? PLACE; +/// +/// where `PLACE` is a direct or an indirect place expression. +/// +/// There are 3 properties that need to be upheld for this transformation to be legal: +/// - place stability: `PLACE` must refer to the same memory wherever it appears; +/// - pointer liveness: we must not introduce dereferences of dangling pointers; +/// - `&mut` borrow uniqueness. +/// +/// # Stability +/// +/// If `PLACE` is an indirect projection, if its of the form `(*LOCAL).PROJECTIONS` where: +/// - `LOCAL` is SSA; +/// - all projections in `PROJECTIONS` have a stable offset (no dereference and no indexing). +/// +/// If `PLACE` is a direct projection of a local, we consider it as constant if: +/// - the local is always live, or it has a single `StorageLive`; +/// - all projections have a stable offset. +/// +/// # Liveness +/// +/// When performing a substitution, we must take care not to introduce uses of dangling locals. +/// To ensure this, we walk the body with the `MaybeStorageDead` dataflow analysis: +/// - if we want to replace `*x` by reborrow `*y` and `y` may be dead, we allow replacement and +/// mark storage statements on `y` for removal; +/// - if we want to replace `*x` by non-reborrow `y` and `y` must be live, we allow replacement; +/// - if we want to replace `*x` by non-reborrow `y` and `y` may be dead, we do not replace. +/// +/// # Uniqueness +/// +/// For `&mut` borrows, we also need to preserve the uniqueness property: +/// we must avoid creating a state where we interleave uses of `*_1` and `_2`. +/// To do it, we only perform full substitution of mutable borrows: +/// we replace either all or none of the occurrences of `*_1`. +/// +/// Some care has to be taken when `_1` is copied in other locals. +/// _1 = &raw? mut? _2; +/// _3 = *_1; +/// _4 = _1 +/// _5 = *_4 +/// In such cases, fully substituting `_1` means fully substituting all of the copies. +/// +/// For immutable borrows, we do not need to preserve such uniqueness property, +/// so we perform all the possible substitutions without removing the `_1 = &_2` statement. +pub struct ReferencePropagation; + +impl<'tcx> MirPass<'tcx> for ReferencePropagation { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + sess.mir_opt_level() >= 4 + } + + #[instrument(level = "trace", skip(self, tcx, body))] + fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + debug!(def_id = ?body.source.def_id()); + while propagate_ssa(tcx, body) {} + } +} + +fn propagate_ssa<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool { + let ssa = SsaLocals::new(body); + + let mut replacer = compute_replacement(tcx, body, &ssa); + debug!(?replacer.targets); + debug!(?replacer.allowed_replacements); + debug!(?replacer.storage_to_remove); + + replacer.visit_body_preserves_cfg(body); + + if replacer.any_replacement { + crate::simplify::remove_unused_definitions(body); + } + + replacer.any_replacement +} + +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +enum Value<'tcx> { + /// Not a pointer, or we can't know. + Unknown, + /// We know the value to be a pointer to this place. + /// The boolean indicates whether the reference is mutable, subject the uniqueness rule. + Pointer(Place<'tcx>, bool), +} + +/// For each local, save the place corresponding to `*local`. +#[instrument(level = "trace", skip(tcx, body))] +fn compute_replacement<'tcx>( + tcx: TyCtxt<'tcx>, + body: &Body<'tcx>, + ssa: &SsaLocals, +) -> Replacer<'tcx> { + let always_live_locals = always_storage_live_locals(body); + + // Compute which locals have a single `StorageLive` statement ever. + let storage_live = StorageLiveLocals::new(body, &always_live_locals); + + // Compute `MaybeStorageDead` dataflow to check that we only replace when the pointee is + // definitely live. + let mut maybe_dead = MaybeStorageDead::new(always_live_locals) + .into_engine(tcx, body) + .iterate_to_fixpoint() + .into_results_cursor(body); + + // Map for each local to the pointee. + let mut targets = IndexVec::from_elem(Value::Unknown, &body.local_decls); + // Set of locals for which we will remove their storage statement. This is useful for + // reborrowed references. + let mut storage_to_remove = BitSet::new_empty(body.local_decls.len()); + + let fully_replacable_locals = fully_replacable_locals(ssa); + + // Returns true iff we can use `place` as a pointee. + // + // Note that we only need to verify that there is a single `StorageLive` statement, and we do + // not need to verify that it dominates all uses of that local. + // + // Consider the three statements: + // SL : StorageLive(a) + // DEF: b = &raw? mut? a + // USE: stuff that uses *b + // + // First, we recall that DEF is checked to dominate USE. Now imagine for the sake of + // contradiction there is a DEF -> SL -> USE path. Consider two cases: + // + // - DEF dominates SL. We always have UB the first time control flow reaches DEF, + // because the storage of `a` is dead. Since DEF dominates USE, that means we cannot + // reach USE and so our optimization is ok. + // + // - DEF does not dominate SL. Then there is a `START_BLOCK -> SL` path not including DEF. + // But we can extend this path to USE, meaning there is also a `START_BLOCK -> USE` path not + // including DEF. This violates the DEF dominates USE condition, and so is impossible. + let is_constant_place = |place: Place<'_>| { + // We only allow `Deref` as the first projection, to avoid surprises. + if place.projection.first() == Some(&PlaceElem::Deref) { + // `place == (*some_local).xxx`, it is constant only if `some_local` is constant. + // We approximate constness using SSAness. + ssa.is_ssa(place.local) && place.projection[1..].iter().all(PlaceElem::is_stable_offset) + } else { + storage_live.has_single_storage(place.local) + && place.projection[..].iter().all(PlaceElem::is_stable_offset) + } + }; + + let mut can_perform_opt = |target: Place<'tcx>, loc: Location| { + if target.projection.first() == Some(&PlaceElem::Deref) { + // We are creating a reborrow. As `place.local` is a reference, removing the storage + // statements should not make it much harder for LLVM to optimize. + storage_to_remove.insert(target.local); + true + } else { + // This is a proper dereference. We can only allow it if `target` is live. + maybe_dead.seek_after_primary_effect(loc); + let maybe_dead = maybe_dead.contains(target.local); + !maybe_dead + } + }; + + for (local, rvalue, location) in ssa.assignments(body) { + debug!(?local); + + // Only visit if we have something to do. + let Value::Unknown = targets[local] else { bug!() }; + + let ty = body.local_decls[local].ty; + + // If this is not a reference or pointer, do nothing. + if !ty.is_any_ptr() { + debug!("not a reference or pointer"); + continue; + } + + // Whether the current local is subject to the uniqueness rule. + let needs_unique = ty.is_mutable_ptr(); + + // If this a mutable reference that we cannot fully replace, mark it as unknown. + if needs_unique && !fully_replacable_locals.contains(local) { + debug!("not fully replaceable"); + continue; + } + + debug!(?rvalue); + match rvalue { + // This is a copy, just use the value we have in store for the previous one. + // As we are visiting in `assignment_order`, ie. reverse postorder, `rhs` should + // have been visited before. + Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) + | Rvalue::CopyForDeref(place) => { + if let Some(rhs) = place.as_local() && ssa.is_ssa(rhs) { + let target = targets[rhs]; + // Only see through immutable reference and pointers, as we do not know yet if + // mutable references are fully replaced. + if !needs_unique && matches!(target, Value::Pointer(..)) { + targets[local] = target; + } else { + targets[local] = Value::Pointer(tcx.mk_place_deref(rhs.into()), needs_unique); + } + } + } + Rvalue::Ref(_, _, place) | Rvalue::AddressOf(_, place) => { + let mut place = *place; + // Try to see through `place` in order to collapse reborrow chains. + if place.projection.first() == Some(&PlaceElem::Deref) + && let Value::Pointer(target, inner_needs_unique) = targets[place.local] + // Only see through immutable reference and pointers, as we do not know yet if + // mutable references are fully replaced. + && !inner_needs_unique + // Only collapse chain if the pointee is definitely live. + && can_perform_opt(target, location) + { + place = target.project_deeper(&place.projection[1..], tcx); + } + assert_ne!(place.local, local); + if is_constant_place(place) { + targets[local] = Value::Pointer(place, needs_unique); + } + } + // We do not know what to do, so keep as not-a-pointer. + _ => {} + } + } + + debug!(?targets); + + let mut finder = ReplacementFinder { + targets: &mut targets, + can_perform_opt, + allowed_replacements: FxHashSet::default(), + }; + let reachable_blocks = traversal::reachable_as_bitset(body); + for (bb, bbdata) in body.basic_blocks.iter_enumerated() { + // Only visit reachable blocks as we rely on dataflow. + if reachable_blocks.contains(bb) { + finder.visit_basic_block_data(bb, bbdata); + } + } + + let allowed_replacements = finder.allowed_replacements; + return Replacer { + tcx, + targets, + storage_to_remove, + allowed_replacements, + fully_replacable_locals, + any_replacement: false, + }; + + struct ReplacementFinder<'a, 'tcx, F> { + targets: &'a mut IndexVec<Local, Value<'tcx>>, + can_perform_opt: F, + allowed_replacements: FxHashSet<(Local, Location)>, + } + + impl<'tcx, F> Visitor<'tcx> for ReplacementFinder<'_, 'tcx, F> + where + F: FnMut(Place<'tcx>, Location) -> bool, + { + fn visit_place(&mut self, place: &Place<'tcx>, ctxt: PlaceContext, loc: Location) { + if matches!(ctxt, PlaceContext::NonUse(_)) { + // There is no need to check liveness for non-uses. + return; + } + + if place.projection.first() != Some(&PlaceElem::Deref) { + // This is not a dereference, nothing to do. + return; + } + + let mut place = place.as_ref(); + loop { + if let Value::Pointer(target, needs_unique) = self.targets[place.local] { + let perform_opt = (self.can_perform_opt)(target, loc); + debug!(?place, ?target, ?needs_unique, ?perform_opt); + + // This a reborrow chain, recursively allow the replacement. + // + // This also allows to detect cases where `target.local` is not replacable, + // and mark it as such. + if let &[PlaceElem::Deref] = &target.projection[..] { + assert!(perform_opt); + self.allowed_replacements.insert((target.local, loc)); + place.local = target.local; + continue; + } else if perform_opt { + self.allowed_replacements.insert((target.local, loc)); + } else if needs_unique { + // This mutable reference is not fully replacable, so drop it. + self.targets[place.local] = Value::Unknown; + } + } + + break; + } + } + } +} + +/// Compute the set of locals that can be fully replaced. +/// +/// We consider a local to be replacable iff it's only used in a `Deref` projection `*_local` or +/// non-use position (like storage statements and debuginfo). +fn fully_replacable_locals(ssa: &SsaLocals) -> BitSet<Local> { + let mut replacable = BitSet::new_empty(ssa.num_locals()); + + // First pass: for each local, whether its uses can be fully replaced. + for local in ssa.locals() { + if ssa.num_direct_uses(local) == 0 { + replacable.insert(local); + } + } + + // Second pass: a local can only be fully replaced if all its copies can. + ssa.meet_copy_equivalence(&mut replacable); + + replacable +} + +/// Utility to help performing subtitution of `*pattern` by `target`. +struct Replacer<'tcx> { + tcx: TyCtxt<'tcx>, + targets: IndexVec<Local, Value<'tcx>>, + storage_to_remove: BitSet<Local>, + allowed_replacements: FxHashSet<(Local, Location)>, + any_replacement: bool, + fully_replacable_locals: BitSet<Local>, +} + +impl<'tcx> MutVisitor<'tcx> for Replacer<'tcx> { + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } + + fn visit_var_debug_info(&mut self, debuginfo: &mut VarDebugInfo<'tcx>) { + if let VarDebugInfoContents::Place(ref mut place) = debuginfo.value + && place.projection.is_empty() + && let Value::Pointer(target, _) = self.targets[place.local] + && target.projection.iter().all(|p| p.can_use_in_debuginfo()) + { + if let Some((&PlaceElem::Deref, rest)) = target.projection.split_last() { + *place = Place::from(target.local).project_deeper(rest, self.tcx); + self.any_replacement = true; + } else if self.fully_replacable_locals.contains(place.local) + && let Some(references) = debuginfo.references.checked_add(1) + { + debuginfo.references = references; + *place = target; + self.any_replacement = true; + } + } + } + + fn visit_place(&mut self, place: &mut Place<'tcx>, ctxt: PlaceContext, loc: Location) { + if place.projection.first() != Some(&PlaceElem::Deref) { + return; + } + + loop { + if let Value::Pointer(target, _) = self.targets[place.local] { + let perform_opt = matches!(ctxt, PlaceContext::NonUse(_)) + || self.allowed_replacements.contains(&(target.local, loc)); + + if perform_opt { + *place = target.project_deeper(&place.projection[1..], self.tcx); + self.any_replacement = true; + continue; + } + } + + break; + } + } + + fn visit_statement(&mut self, stmt: &mut Statement<'tcx>, loc: Location) { + match stmt.kind { + StatementKind::StorageLive(l) | StatementKind::StorageDead(l) + if self.storage_to_remove.contains(l) => + { + stmt.make_nop(); + } + // Do not remove assignments as they may still be useful for debuginfo. + _ => self.super_statement(stmt, loc), + } + } +} diff --git a/compiler/rustc_mir_transform/src/remove_place_mention.rs b/compiler/rustc_mir_transform/src/remove_place_mention.rs new file mode 100644 index 000000000..8be1c3757 --- /dev/null +++ b/compiler/rustc_mir_transform/src/remove_place_mention.rs @@ -0,0 +1,23 @@ +//! This pass removes `PlaceMention` statement, which has no effect at codegen. + +use crate::MirPass; +use rustc_middle::mir::*; +use rustc_middle::ty::TyCtxt; + +pub struct RemovePlaceMention; + +impl<'tcx> MirPass<'tcx> for RemovePlaceMention { + fn is_enabled(&self, sess: &rustc_session::Session) -> bool { + !sess.opts.unstable_opts.mir_keep_place_mention + } + + fn run_pass(&self, _: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + trace!("Running RemovePlaceMention on {:?}", body.source); + for data in body.basic_blocks.as_mut_preserves_cfg() { + data.statements.retain(|statement| match statement.kind { + StatementKind::PlaceMention(..) | StatementKind::Nop => false, + _ => true, + }) + } + } +} diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs index abe6cb285..23442f8b9 100644 --- a/compiler/rustc_mir_transform/src/reveal_all.rs +++ b/compiler/rustc_mir_transform/src/reveal_all.rs @@ -35,10 +35,22 @@ impl<'tcx> MutVisitor<'tcx> for RevealAllVisitor<'tcx> { } #[inline] + fn visit_constant(&mut self, constant: &mut Constant<'tcx>, _: Location) { + // We have to use `try_normalize_erasing_regions` here, since it's + // possible that we visit impossible-to-satisfy where clauses here, + // see #91745 + if let Ok(c) = self.tcx.try_normalize_erasing_regions(self.param_env, constant.literal) { + constant.literal = c; + } + } + + #[inline] fn visit_ty(&mut self, ty: &mut Ty<'tcx>, _: TyContext) { // We have to use `try_normalize_erasing_regions` here, since it's // possible that we visit impossible-to-satisfy where clauses here, // see #91745 - *ty = self.tcx.try_normalize_erasing_regions(self.param_env, *ty).unwrap_or(*ty); + if let Ok(t) = self.tcx.try_normalize_erasing_regions(self.param_env, *ty) { + *ty = t; + } } } diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs index ef367faf6..2479856b7 100644 --- a/compiler/rustc_mir_transform/src/separate_const_switch.rs +++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs @@ -303,8 +303,7 @@ fn find_determining_place<'tcx>( | Rvalue::NullaryOp(_, _) | Rvalue::ShallowInitBox(_, _) | Rvalue::UnaryOp(_, Operand::Constant(_)) - | Rvalue::Cast(_, Operand::Constant(_), _) - => return None, + | Rvalue::Cast(_, Operand::Constant(_), _) => return None, } } diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs index 2787fe2ce..0eb27c231 100644 --- a/compiler/rustc_mir_transform/src/shim.rs +++ b/compiler/rustc_mir_transform/src/shim.rs @@ -2,12 +2,12 @@ use rustc_hir as hir; use rustc_hir::def_id::DefId; use rustc_hir::lang_items::LangItem; use rustc_middle::mir::*; -use rustc_middle::ty::query::Providers; +use rustc_middle::query::Providers; use rustc_middle::ty::InternalSubsts; use rustc_middle::ty::{self, EarlyBinder, GeneratorSubsts, Ty, TyCtxt}; use rustc_target::abi::{FieldIdx, VariantIdx, FIRST_VARIANT}; -use rustc_index::vec::{Idx, IndexVec}; +use rustc_index::{Idx, IndexVec}; use rustc_span::Span; use rustc_target::spec::abi::Abi; @@ -95,7 +95,7 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<' &add_moves_for_packed_drops::AddMovesForPackedDrops, &deref_separator::Derefer, &remove_noop_landing_pads::RemoveNoopLandingPads, - &simplify::SimplifyCfg::new("make_shim"), + &simplify::SimplifyCfg::MakeShim, &add_call_guards::CriticalCallEdges, &abort_unwinding_calls::AbortUnwindingCalls, ], @@ -355,7 +355,7 @@ fn build_thread_local_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'t fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -> Body<'tcx> { debug!("build_clone_shim(def_id={:?})", def_id); - let param_env = tcx.param_env(def_id); + let param_env = tcx.param_env_reveal_all_normalized(def_id); let mut builder = CloneShimBuilder::new(tcx, def_id, self_ty); let is_copy = self_ty.is_copy_modulo_regions(tcx, param_env); @@ -544,6 +544,7 @@ impl<'tcx> CloneShimBuilder<'tcx> { place: dest_field, target: unwind, unwind: UnwindAction::Terminate, + replace: false, }, true, ); @@ -800,6 +801,7 @@ fn build_call_shim<'tcx>( place: rcvr_place(), target: BasicBlock::new(2), unwind: UnwindAction::Continue, + replace: false, }, false, ); @@ -815,6 +817,7 @@ fn build_call_shim<'tcx>( place: rcvr_place(), target: BasicBlock::new(4), unwind: UnwindAction::Terminate, + replace: false, }, true, ); @@ -836,7 +839,7 @@ fn build_call_shim<'tcx>( pub fn build_adt_ctor(tcx: TyCtxt<'_>, ctor_id: DefId) -> Body<'_> { debug_assert!(tcx.is_constructor(ctor_id)); - let param_env = tcx.param_env(ctor_id); + let param_env = tcx.param_env_reveal_all_normalized(ctor_id); // Normalize the sig. let sig = tcx diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs index c79e1cf08..e59219321 100644 --- a/compiler/rustc_mir_transform/src/simplify.rs +++ b/compiler/rustc_mir_transform/src/simplify.rs @@ -29,20 +29,38 @@ use crate::MirPass; use rustc_data_structures::fx::{FxHashSet, FxIndexSet}; -use rustc_index::vec::{Idx, IndexSlice, IndexVec}; +use rustc_index::{Idx, IndexSlice, IndexVec}; use rustc_middle::mir::coverage::*; use rustc_middle::mir::visit::{MutVisitor, MutatingUseContext, PlaceContext, Visitor}; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; use smallvec::SmallVec; -pub struct SimplifyCfg { - label: String, +pub enum SimplifyCfg { + Initial, + PromoteConsts, + RemoveFalseEdges, + EarlyOpt, + ElaborateDrops, + Final, + MakeShim, + AfterUninhabitedEnumBranching, } impl SimplifyCfg { - pub fn new(label: &str) -> Self { - SimplifyCfg { label: format!("SimplifyCfg-{}", label) } + pub fn name(&self) -> &'static str { + match self { + SimplifyCfg::Initial => "SimplifyCfg-initial", + SimplifyCfg::PromoteConsts => "SimplifyCfg-promote-consts", + SimplifyCfg::RemoveFalseEdges => "SimplifyCfg-remove-false-edges", + SimplifyCfg::EarlyOpt => "SimplifyCfg-early-opt", + SimplifyCfg::ElaborateDrops => "SimplifyCfg-elaborate-drops", + SimplifyCfg::Final => "SimplifyCfg-final", + SimplifyCfg::MakeShim => "SimplifyCfg-make_shim", + SimplifyCfg::AfterUninhabitedEnumBranching => { + "SimplifyCfg-after-uninhabited-enum-branching" + } + } } } @@ -56,12 +74,12 @@ pub fn simplify_cfg<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { } impl<'tcx> MirPass<'tcx> for SimplifyCfg { - fn name(&self) -> &str { - &self.label + fn name(&self) -> &'static str { + &self.name() } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - debug!("SimplifyCfg({:?}) - simplifying {:?}", self.label, body.source); + debug!("SimplifyCfg({:?}) - simplifying {:?}", self.name(), body.source); simplify_cfg(tcx, body); } } @@ -260,6 +278,18 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> { } } +pub fn simplify_duplicate_switch_targets(terminator: &mut Terminator<'_>) { + if let TerminatorKind::SwitchInt { targets, .. } = &mut terminator.kind { + let otherwise = targets.otherwise(); + if targets.iter().any(|t| t.1 == otherwise) { + *targets = SwitchTargets::new( + targets.iter().filter(|t| t.1 != otherwise), + targets.otherwise(), + ); + } + } +} + pub fn remove_duplicate_unreachable_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { struct OptApplier<'tcx> { tcx: TyCtxt<'tcx>, @@ -280,6 +310,8 @@ pub fn remove_duplicate_unreachable_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut B } } + simplify_duplicate_switch_targets(terminator); + self.super_terminator(terminator, location); } } @@ -423,19 +455,17 @@ fn save_unreachable_coverage( )); } -pub struct SimplifyLocals { - label: String, -} - -impl SimplifyLocals { - pub fn new(label: &str) -> SimplifyLocals { - SimplifyLocals { label: format!("SimplifyLocals-{}", label) } - } +pub enum SimplifyLocals { + BeforeConstProp, + Final, } impl<'tcx> MirPass<'tcx> for SimplifyLocals { - fn name(&self) -> &str { - &self.label + fn name(&self) -> &'static str { + match &self { + SimplifyLocals::BeforeConstProp => "SimplifyLocals-before-const-prop", + SimplifyLocals::Final => "SimplifyLocals-final", + } } fn is_enabled(&self, sess: &rustc_session::Session) -> bool { diff --git a/compiler/rustc_mir_transform/src/simplify_branches.rs b/compiler/rustc_mir_transform/src/simplify_branches.rs index 8164b3052..1ff488169 100644 --- a/compiler/rustc_mir_transform/src/simplify_branches.rs +++ b/compiler/rustc_mir_transform/src/simplify_branches.rs @@ -2,24 +2,21 @@ use crate::MirPass; use rustc_middle::mir::*; use rustc_middle::ty::TyCtxt; -/// A pass that replaces a branch with a goto when its condition is known. -pub struct SimplifyConstCondition { - label: String, -} - -impl SimplifyConstCondition { - pub fn new(label: &str) -> Self { - SimplifyConstCondition { label: format!("SimplifyConstCondition-{}", label) } - } +pub enum SimplifyConstCondition { + AfterConstProp, + Final, } - +/// A pass that replaces a branch with a goto when its condition is known. impl<'tcx> MirPass<'tcx> for SimplifyConstCondition { - fn name(&self) -> &str { - &self.label + fn name(&self) -> &'static str { + match self { + SimplifyConstCondition::AfterConstProp => "SimplifyConstCondition-after-const-prop", + SimplifyConstCondition::Final => "SimplifyConstCondition-final", + } } fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - let param_env = tcx.param_env(body.source.def_id()); + let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); for block in body.basic_blocks_mut() { let terminator = block.terminator_mut(); terminator.kind = match terminator.kind { diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs index dcad1518e..113ca2fc5 100644 --- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs +++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs @@ -37,7 +37,7 @@ impl<'tcx> MirPass<'tcx> for SimplifyComparisonIntegral { let opts = helper.find_optimizations(); let mut storage_deads_to_insert = vec![]; let mut storage_deads_to_remove: Vec<(usize, BasicBlock)> = vec![]; - let param_env = tcx.param_env(body.source.def_id()); + let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id()); for opt in opts { trace!("SUCCESS: Applying {:?}", opt); // replace terminator with a switchInt that switches on the integer directly diff --git a/compiler/rustc_mir_transform/src/sroa.rs b/compiler/rustc_mir_transform/src/sroa.rs index c798bd053..2d7729129 100644 --- a/compiler/rustc_mir_transform/src/sroa.rs +++ b/compiler/rustc_mir_transform/src/sroa.rs @@ -1,6 +1,6 @@ use crate::MirPass; use rustc_index::bit_set::{BitSet, GrowableBitSet}; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir::patch::MirPatch; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; diff --git a/compiler/rustc_mir_transform/src/ssa.rs b/compiler/rustc_mir_transform/src/ssa.rs index be026402d..7a0d3a025 100644 --- a/compiler/rustc_mir_transform/src/ssa.rs +++ b/compiler/rustc_mir_transform/src/ssa.rs @@ -1,11 +1,17 @@ +//! We denote as "SSA" the set of locals that verify the following properties: +//! 1/ They are only assigned-to once, either as a function parameter, or in an assign statement; +//! 2/ This single assignment dominates all uses; +//! +//! As a consequence of rule 2, we consider that borrowed locals are not SSA, even if they are +//! `Freeze`, as we do not track that the assignment dominates all uses of the borrow. + use either::Either; use rustc_data_structures::graph::dominators::Dominators; use rustc_index::bit_set::BitSet; -use rustc_index::vec::{IndexSlice, IndexVec}; +use rustc_index::{IndexSlice, IndexVec}; use rustc_middle::middle::resolve_bound_vars::Set1; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; -use rustc_middle::ty::{ParamEnv, TyCtxt}; #[derive(Debug)] pub struct SsaLocals { @@ -17,57 +23,60 @@ pub struct SsaLocals { assignment_order: Vec<Local>, /// Copy equivalence classes between locals. See `copy_classes` for documentation. copy_classes: IndexVec<Local, Local>, + /// Number of "direct" uses of each local, ie. uses that are not dereferences. + /// We ignore non-uses (Storage statements, debuginfo). + direct_uses: IndexVec<Local, u32>, } /// We often encounter MIR bodies with 1 or 2 basic blocks. In those cases, it's unnecessary to /// actually compute dominators, we can just compare block indices because bb0 is always the first -/// block, and in any body all other blocks are always always dominated by bb0. -struct SmallDominators { - inner: Option<Dominators<BasicBlock>>, -} - -trait DomExt { - fn dominates(self, _other: Self, dominators: &SmallDominators) -> bool; +/// block, and in any body all other blocks are always dominated by bb0. +struct SmallDominators<'a> { + inner: Option<&'a Dominators<BasicBlock>>, } -impl DomExt for Location { - fn dominates(self, other: Location, dominators: &SmallDominators) -> bool { - if self.block == other.block { - self.statement_index <= other.statement_index +impl SmallDominators<'_> { + fn dominates(&self, first: Location, second: Location) -> bool { + if first.block == second.block { + first.statement_index <= second.statement_index + } else if let Some(inner) = &self.inner { + inner.dominates(first.block, second.block) } else { - dominators.dominates(self.block, other.block) + first.block < second.block } } -} -impl SmallDominators { - fn dominates(&self, dom: BasicBlock, node: BasicBlock) -> bool { - if let Some(inner) = &self.inner { inner.dominates(dom, node) } else { dom < node } + fn check_dominates(&mut self, set: &mut Set1<LocationExtended>, loc: Location) { + let assign_dominates = match *set { + Set1::Empty | Set1::Many => false, + Set1::One(LocationExtended::Arg) => true, + Set1::One(LocationExtended::Plain(assign)) => { + self.dominates(assign.successor_within_block(), loc) + } + }; + // We are visiting a use that is not dominated by an assignment. + // Either there is a cycle involved, or we are reading for uninitialized local. + // Bail out. + if !assign_dominates { + *set = Set1::Many; + } } } impl SsaLocals { - pub fn new<'tcx>( - tcx: TyCtxt<'tcx>, - param_env: ParamEnv<'tcx>, - body: &Body<'tcx>, - borrowed_locals: &BitSet<Local>, - ) -> SsaLocals { + pub fn new<'tcx>(body: &Body<'tcx>) -> SsaLocals { let assignment_order = Vec::with_capacity(body.local_decls.len()); let assignments = IndexVec::from_elem(Set1::Empty, &body.local_decls); let dominators = if body.basic_blocks.len() > 2 { Some(body.basic_blocks.dominators()) } else { None }; let dominators = SmallDominators { inner: dominators }; - let mut visitor = SsaVisitor { assignments, assignment_order, dominators }; - for (local, decl) in body.local_decls.iter_enumerated() { - if matches!(body.local_kind(local), LocalKind::Arg) { - visitor.assignments[local] = Set1::One(LocationExtended::Arg); - } - if borrowed_locals.contains(local) && !decl.ty.is_freeze(tcx, param_env) { - visitor.assignments[local] = Set1::Many; - } + let direct_uses = IndexVec::from_elem(0, &body.local_decls); + let mut visitor = SsaVisitor { assignments, assignment_order, dominators, direct_uses }; + + for local in body.args_iter() { + visitor.assignments[local] = Set1::One(LocationExtended::Arg); } if body.basic_blocks.len() > 2 { @@ -85,36 +94,52 @@ impl SsaLocals { } debug!(?visitor.assignments); + debug!(?visitor.direct_uses); visitor .assignment_order .retain(|&local| matches!(visitor.assignments[local], Set1::One(_))); debug!(?visitor.assignment_order); - let copy_classes = compute_copy_classes(&visitor, body); - - SsaLocals { + let mut ssa = SsaLocals { assignments: visitor.assignments, assignment_order: visitor.assignment_order, - copy_classes, - } + direct_uses: visitor.direct_uses, + // This is filled by `compute_copy_classes`. + copy_classes: IndexVec::default(), + }; + compute_copy_classes(&mut ssa, body); + ssa + } + + pub fn num_locals(&self) -> usize { + self.assignments.len() + } + + pub fn locals(&self) -> impl Iterator<Item = Local> { + self.assignments.indices() } pub fn is_ssa(&self, local: Local) -> bool { matches!(self.assignments[local], Set1::One(_)) } + /// Return the number of uses if a local that are not "Deref". + pub fn num_direct_uses(&self, local: Local) -> u32 { + self.direct_uses[local] + } + pub fn assignments<'a, 'tcx>( &'a self, body: &'a Body<'tcx>, - ) -> impl Iterator<Item = (Local, &'a Rvalue<'tcx>)> + 'a { + ) -> impl Iterator<Item = (Local, &'a Rvalue<'tcx>, Location)> + 'a { self.assignment_order.iter().filter_map(|&local| { if let Set1::One(LocationExtended::Plain(loc)) = self.assignments[local] { // `loc` must point to a direct assignment to `local`. let Either::Left(stmt) = body.stmt_at(loc) else { bug!() }; let Some((target, rvalue)) = stmt.kind.as_assign() else { bug!() }; assert_eq!(target.as_local(), Some(local)); - Some((local, rvalue)) + Some((local, rvalue, loc)) } else { None } @@ -173,48 +198,32 @@ enum LocationExtended { Arg, } -struct SsaVisitor { - dominators: SmallDominators, +struct SsaVisitor<'a> { + dominators: SmallDominators<'a>, assignments: IndexVec<Local, Set1<LocationExtended>>, assignment_order: Vec<Local>, + direct_uses: IndexVec<Local, u32>, } -impl SsaVisitor { - fn check_assignment_dominates(&mut self, local: Local, loc: Location) { - let set = &mut self.assignments[local]; - let assign_dominates = match *set { - Set1::Empty | Set1::Many => false, - Set1::One(LocationExtended::Arg) => true, - Set1::One(LocationExtended::Plain(assign)) => { - assign.successor_within_block().dominates(loc, &self.dominators) - } - }; - // We are visiting a use that is not dominated by an assignment. - // Either there is a cycle involved, or we are reading for uninitialized local. - // Bail out. - if !assign_dominates { - *set = Set1::Many; - } - } -} - -impl<'tcx> Visitor<'tcx> for SsaVisitor { +impl<'tcx> Visitor<'tcx> for SsaVisitor<'_> { fn visit_local(&mut self, local: Local, ctxt: PlaceContext, loc: Location) { match ctxt { - PlaceContext::MutatingUse(MutatingUseContext::Store) => { - self.assignments[local].insert(LocationExtended::Plain(loc)); - if let Set1::One(_) = self.assignments[local] { - // Only record if SSA-like, to avoid growing the vector needlessly. - self.assignment_order.push(local); - } - } + PlaceContext::MutatingUse(MutatingUseContext::Projection) + | PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection) => bug!(), // Anything can happen with raw pointers, so remove them. - PlaceContext::NonMutatingUse(NonMutatingUseContext::AddressOf) - | PlaceContext::MutatingUse(_) => self.assignments[local] = Set1::Many, - // Immutable borrows are taken into account in `SsaLocals::new` by - // removing non-freeze locals. + // We do not verify that all uses of the borrow dominate the assignment to `local`, + // so we have to remove them too. + PlaceContext::NonMutatingUse( + NonMutatingUseContext::SharedBorrow + | NonMutatingUseContext::ShallowBorrow + | NonMutatingUseContext::AddressOf, + ) + | PlaceContext::MutatingUse(_) => { + self.assignments[local] = Set1::Many; + } PlaceContext::NonMutatingUse(_) => { - self.check_assignment_dominates(local, loc); + self.dominators.check_dominates(&mut self.assignments[local], loc); + self.direct_uses[local] += 1; } PlaceContext::NonUse(_) => {} } @@ -224,58 +233,113 @@ impl<'tcx> Visitor<'tcx> for SsaVisitor { if place.projection.first() == Some(&PlaceElem::Deref) { // Do not do anything for storage statements and debuginfo. if ctxt.is_use() { - // A use through a `deref` only reads from the local, and cannot write to it. - let new_ctxt = PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection); + // Only change the context if it is a real use, not a "use" in debuginfo. + let new_ctxt = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy); self.visit_projection(place.as_ref(), new_ctxt, loc); - self.check_assignment_dominates(place.local, loc); + self.dominators.check_dominates(&mut self.assignments[place.local], loc); } return; + } else { + self.visit_projection(place.as_ref(), ctxt, loc); + self.visit_local(place.local, ctxt, loc); + } + } + + fn visit_assign(&mut self, place: &Place<'tcx>, rvalue: &Rvalue<'tcx>, loc: Location) { + if let Some(local) = place.as_local() { + self.assignments[local].insert(LocationExtended::Plain(loc)); + if let Set1::One(_) = self.assignments[local] { + // Only record if SSA-like, to avoid growing the vector needlessly. + self.assignment_order.push(local); + } + } else { + self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), loc); } - self.super_place(place, ctxt, loc); + self.visit_rvalue(rvalue, loc); } } #[instrument(level = "trace", skip(ssa, body))] -fn compute_copy_classes(ssa: &SsaVisitor, body: &Body<'_>) -> IndexVec<Local, Local> { +fn compute_copy_classes(ssa: &mut SsaLocals, body: &Body<'_>) { + let mut direct_uses = std::mem::take(&mut ssa.direct_uses); let mut copies = IndexVec::from_fn_n(|l| l, body.local_decls.len()); - for &local in &ssa.assignment_order { - debug!(?local); - - if local == RETURN_PLACE { - // `_0` is special, we cannot rename it. - continue; - } - - // This is not SSA: mark that we don't know the value. - debug!(assignments = ?ssa.assignments[local]); - let Set1::One(LocationExtended::Plain(loc)) = ssa.assignments[local] else { continue }; - - // `loc` must point to a direct assignment to `local`. - let Either::Left(stmt) = body.stmt_at(loc) else { bug!() }; - let Some((_target, rvalue)) = stmt.kind.as_assign() else { bug!() }; - assert_eq!(_target.as_local(), Some(local)); - + for (local, rvalue, _) in ssa.assignments(body) { let (Rvalue::Use(Operand::Copy(place) | Operand::Move(place)) | Rvalue::CopyForDeref(place)) = rvalue else { continue }; let Some(rhs) = place.as_local() else { continue }; - let Set1::One(_) = ssa.assignments[rhs] else { continue }; + if !ssa.is_ssa(rhs) { + continue; + } // We visit in `assignment_order`, ie. reverse post-order, so `rhs` has been // visited before `local`, and we just have to copy the representing local. - copies[local] = copies[rhs]; + let head = copies[rhs]; + + if local == RETURN_PLACE { + // `_0` is special, we cannot rename it. Instead, rename the class of `rhs` to + // `RETURN_PLACE`. This is only possible if the class head is a temporary, not an + // argument. + if body.local_kind(head) != LocalKind::Temp { + continue; + } + for h in copies.iter_mut() { + if *h == head { + *h = RETURN_PLACE; + } + } + } else { + copies[local] = head; + } + direct_uses[rhs] -= 1; } debug!(?copies); + debug!(?direct_uses); // Invariant: `copies` must point to the head of an equivalence class. #[cfg(debug_assertions)] for &head in copies.iter() { assert_eq!(copies[head], head); } + debug_assert_eq!(copies[RETURN_PLACE], RETURN_PLACE); + + ssa.direct_uses = direct_uses; + ssa.copy_classes = copies; +} + +#[derive(Debug)] +pub(crate) struct StorageLiveLocals { + /// Set of "StorageLive" statements for each local. + storage_live: IndexVec<Local, Set1<LocationExtended>>, +} + +impl StorageLiveLocals { + pub(crate) fn new( + body: &Body<'_>, + always_storage_live_locals: &BitSet<Local>, + ) -> StorageLiveLocals { + let mut storage_live = IndexVec::from_elem(Set1::Empty, &body.local_decls); + for local in always_storage_live_locals.iter() { + storage_live[local] = Set1::One(LocationExtended::Arg); + } + for (block, bbdata) in body.basic_blocks.iter_enumerated() { + for (statement_index, statement) in bbdata.statements.iter().enumerate() { + if let StatementKind::StorageLive(local) = statement.kind { + storage_live[local] + .insert(LocationExtended::Plain(Location { block, statement_index })); + } + } + } + debug!(?storage_live); + StorageLiveLocals { storage_live } + } - copies + #[inline] + pub(crate) fn has_single_storage(&self, local: Local) -> bool { + matches!(self.storage_live[local], Set1::One(_)) + } } diff --git a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs index be0aa0fc4..5389b9f52 100644 --- a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs +++ b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs @@ -109,7 +109,9 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching { continue; }; - let layout = tcx.layout_of(tcx.param_env(body.source.def_id()).and(discriminant_ty)); + let layout = tcx.layout_of( + tcx.param_env_reveal_all_normalized(body.source.def_id()).and(discriminant_ty), + ); let allowed_variants = if let Ok(layout) = layout { variant_discriminants(&layout, discriminant_ty, tcx) |