summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_mir_transform
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_mir_transform')
-rw-r--r--compiler/rustc_mir_transform/src/abort_unwinding_calls.rs2
-rw-r--r--compiler/rustc_mir_transform/src/add_call_guards.rs2
-rw-r--r--compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs2
-rw-r--r--compiler/rustc_mir_transform/src/add_retag.rs19
-rw-r--r--compiler/rustc_mir_transform/src/check_unsafety.rs91
-rw-r--r--compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs2
-rw-r--r--compiler/rustc_mir_transform/src/const_goto.rs4
-rw-r--r--compiler/rustc_mir_transform/src/const_prop.rs127
-rw-r--r--compiler/rustc_mir_transform/src/const_prop_lint.rs66
-rw-r--r--compiler/rustc_mir_transform/src/coverage/graph.rs4
-rw-r--r--compiler/rustc_mir_transform/src/coverage/mod.rs4
-rw-r--r--compiler/rustc_mir_transform/src/coverage/query.rs4
-rw-r--r--compiler/rustc_mir_transform/src/coverage/spans.rs2
-rw-r--r--compiler/rustc_mir_transform/src/coverage/tests.rs6
-rw-r--r--compiler/rustc_mir_transform/src/dead_store_elimination.rs2
-rw-r--r--compiler/rustc_mir_transform/src/deaggregator.rs4
-rw-r--r--compiler/rustc_mir_transform/src/deduplicate_blocks.rs5
-rw-r--r--compiler/rustc_mir_transform/src/deref_separator.rs23
-rw-r--r--compiler/rustc_mir_transform/src/dest_prop.rs8
-rw-r--r--compiler/rustc_mir_transform/src/early_otherwise_branch.rs6
-rw-r--r--compiler/rustc_mir_transform/src/elaborate_box_derefs.rs40
-rw-r--r--compiler/rustc_mir_transform/src/elaborate_drops.rs16
-rw-r--r--compiler/rustc_mir_transform/src/ffi_unwind_calls.rs2
-rw-r--r--compiler/rustc_mir_transform/src/generator.rs108
-rw-r--r--compiler/rustc_mir_transform/src/inline.rs371
-rw-r--r--compiler/rustc_mir_transform/src/inline/cycle.rs2
-rw-r--r--compiler/rustc_mir_transform/src/lib.rs180
-rw-r--r--compiler/rustc_mir_transform/src/lower_intrinsics.rs31
-rw-r--r--compiler/rustc_mir_transform/src/multiple_return_terminators.rs2
-rw-r--r--compiler/rustc_mir_transform/src/normalize_array_len.rs4
-rw-r--r--compiler/rustc_mir_transform/src/nrvo.rs8
-rw-r--r--compiler/rustc_mir_transform/src/pass_manager.rs73
-rw-r--r--compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs4
-rw-r--r--compiler/rustc_mir_transform/src/remove_uninit_drops.rs2
-rw-r--r--compiler/rustc_mir_transform/src/required_consts.rs11
-rw-r--r--compiler/rustc_mir_transform/src/reveal_all.rs2
-rw-r--r--compiler/rustc_mir_transform/src/separate_const_switch.rs8
-rw-r--r--compiler/rustc_mir_transform/src/shim.rs131
-rw-r--r--compiler/rustc_mir_transform/src/simplify.rs10
-rw-r--r--compiler/rustc_mir_transform/src/simplify_comparison_integral.rs2
-rw-r--r--compiler/rustc_mir_transform/src/simplify_try.rs4
-rw-r--r--compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs6
-rw-r--r--compiler/rustc_mir_transform/src/unreachable_prop.rs76
43 files changed, 823 insertions, 653 deletions
diff --git a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
index 2502e8b60..d8f85d2e3 100644
--- a/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
+++ b/compiler/rustc_mir_transform/src/abort_unwinding_calls.rs
@@ -56,7 +56,7 @@ impl<'tcx> MirPass<'tcx> for AbortUnwindingCalls {
// example.
let mut calls_to_terminate = Vec::new();
let mut cleanups_to_remove = Vec::new();
- for (id, block) in body.basic_blocks().iter_enumerated() {
+ for (id, block) in body.basic_blocks.iter_enumerated() {
if block.is_cleanup {
continue;
}
diff --git a/compiler/rustc_mir_transform/src/add_call_guards.rs b/compiler/rustc_mir_transform/src/add_call_guards.rs
index f12c8560c..30966d22e 100644
--- a/compiler/rustc_mir_transform/src/add_call_guards.rs
+++ b/compiler/rustc_mir_transform/src/add_call_guards.rs
@@ -45,7 +45,7 @@ impl AddCallGuards {
// We need a place to store the new blocks generated
let mut new_blocks = Vec::new();
- let cur_len = body.basic_blocks().len();
+ let cur_len = body.basic_blocks.len();
for block in body.basic_blocks_mut() {
match block.terminator {
diff --git a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
index 8de0aad04..ffb5d8c6d 100644
--- a/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
+++ b/compiler/rustc_mir_transform/src/add_moves_for_packed_drops.rs
@@ -55,7 +55,7 @@ fn add_moves_for_packed_drops_patch<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>)
let mut patch = MirPatch::new(body);
let param_env = tcx.param_env(def_id);
- for (bb, data) in body.basic_blocks().iter_enumerated() {
+ for (bb, data) in body.basic_blocks.iter_enumerated() {
let loc = Location { block: bb, statement_index: data.statements.len() };
let terminator = data.terminator();
diff --git a/compiler/rustc_mir_transform/src/add_retag.rs b/compiler/rustc_mir_transform/src/add_retag.rs
index 9c5896c4e..036b55898 100644
--- a/compiler/rustc_mir_transform/src/add_retag.rs
+++ b/compiler/rustc_mir_transform/src/add_retag.rs
@@ -66,7 +66,6 @@ impl<'tcx> MirPass<'tcx> for AddRetag {
// We need an `AllCallEdges` pass before we can do any work.
super::add_call_guards::AllCallEdges.run_pass(tcx, body);
- let (span, arg_count) = (body.span, body.arg_count);
let basic_blocks = body.basic_blocks.as_mut();
let local_decls = &body.local_decls;
let needs_retag = |place: &Place<'tcx>| {
@@ -90,20 +89,18 @@ impl<'tcx> MirPass<'tcx> for AddRetag {
// PART 1
// Retag arguments at the beginning of the start block.
{
- // FIXME: Consider using just the span covering the function
- // argument declaration.
- let source_info = SourceInfo::outermost(span);
// Gather all arguments, skip return value.
- let places = local_decls
- .iter_enumerated()
- .skip(1)
- .take(arg_count)
- .map(|(local, _)| Place::from(local))
- .filter(needs_retag);
+ let places = local_decls.iter_enumerated().skip(1).take(body.arg_count).filter_map(
+ |(local, decl)| {
+ let place = Place::from(local);
+ needs_retag(&place).then_some((place, decl.source_info))
+ },
+ );
+
// Emit their retags.
basic_blocks[START_BLOCK].statements.splice(
0..0,
- places.map(|place| Statement {
+ places.map(|(place, source_info)| Statement {
source_info,
kind: StatementKind::Retag(RetagKind::FnEntry, Box::new(place)),
}),
diff --git a/compiler/rustc_mir_transform/src/check_unsafety.rs b/compiler/rustc_mir_transform/src/check_unsafety.rs
index d564f4801..beff19a3a 100644
--- a/compiler/rustc_mir_transform/src/check_unsafety.rs
+++ b/compiler/rustc_mir_transform/src/check_unsafety.rs
@@ -1,17 +1,16 @@
-use rustc_data_structures::fx::FxHashMap;
+use rustc_data_structures::fx::FxHashSet;
use rustc_errors::struct_span_err;
use rustc_hir as hir;
use rustc_hir::def_id::{DefId, LocalDefId};
use rustc_hir::hir_id::HirId;
use rustc_hir::intravisit;
use rustc_middle::mir::visit::{MutatingUseContext, PlaceContext, Visitor};
+use rustc_middle::mir::*;
use rustc_middle::ty::query::Providers;
use rustc_middle::ty::{self, TyCtxt};
-use rustc_middle::{lint, mir::*};
use rustc_session::lint::builtin::{UNSAFE_OP_IN_UNSAFE_FN, UNUSED_UNSAFE};
use rustc_session::lint::Level;
-use std::collections::hash_map;
use std::ops::Bound;
pub struct UnsafetyChecker<'a, 'tcx> {
@@ -23,10 +22,7 @@ pub struct UnsafetyChecker<'a, 'tcx> {
param_env: ty::ParamEnv<'tcx>,
/// Used `unsafe` blocks in this function. This is used for the "unused_unsafe" lint.
- ///
- /// The keys are the used `unsafe` blocks, the UnusedUnsafeKind indicates whether
- /// or not any of the usages happen at a place that doesn't allow `unsafe_op_in_unsafe_fn`.
- used_unsafe_blocks: FxHashMap<HirId, UsedUnsafeBlockData>,
+ used_unsafe_blocks: FxHashSet<HirId>,
}
impl<'a, 'tcx> UnsafetyChecker<'a, 'tcx> {
@@ -109,7 +105,8 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
// safe (at least as emitted during MIR construction)
}
- StatementKind::CopyNonOverlapping(..) => unreachable!(),
+ // Move to above list once mir construction uses it.
+ StatementKind::Intrinsic(..) => unreachable!(),
}
self.super_statement(statement, location);
}
@@ -130,10 +127,7 @@ impl<'tcx> Visitor<'tcx> for UnsafetyChecker<'_, 'tcx> {
&AggregateKind::Closure(def_id, _) | &AggregateKind::Generator(def_id, _, _) => {
let UnsafetyCheckResult { violations, used_unsafe_blocks, .. } =
self.tcx.unsafety_check_result(def_id);
- self.register_violations(
- violations,
- used_unsafe_blocks.iter().map(|(&h, &d)| (h, d)),
- );
+ self.register_violations(violations, used_unsafe_blocks.iter().copied());
}
},
_ => {}
@@ -257,22 +251,8 @@ impl<'tcx> UnsafetyChecker<'_, 'tcx> {
fn register_violations<'a>(
&mut self,
violations: impl IntoIterator<Item = &'a UnsafetyViolation>,
- new_used_unsafe_blocks: impl IntoIterator<Item = (HirId, UsedUnsafeBlockData)>,
+ new_used_unsafe_blocks: impl IntoIterator<Item = HirId>,
) {
- use UsedUnsafeBlockData::{AllAllowedInUnsafeFn, SomeDisallowedInUnsafeFn};
-
- let update_entry = |this: &mut Self, hir_id, new_usage| {
- match this.used_unsafe_blocks.entry(hir_id) {
- hash_map::Entry::Occupied(mut entry) => {
- if new_usage == SomeDisallowedInUnsafeFn {
- *entry.get_mut() = SomeDisallowedInUnsafeFn;
- }
- }
- hash_map::Entry::Vacant(entry) => {
- entry.insert(new_usage);
- }
- };
- };
let safety = self.body.source_scopes[self.source_info.scope]
.local_data
.as_ref()
@@ -299,22 +279,14 @@ impl<'tcx> UnsafetyChecker<'_, 'tcx> {
}
}),
Safety::BuiltinUnsafe => {}
- Safety::ExplicitUnsafe(hir_id) => violations.into_iter().for_each(|violation| {
- update_entry(
- self,
- hir_id,
- match self.tcx.lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, violation.lint_root).0
- {
- Level::Allow => AllAllowedInUnsafeFn(violation.lint_root),
- _ => SomeDisallowedInUnsafeFn,
- },
- )
+ Safety::ExplicitUnsafe(hir_id) => violations.into_iter().for_each(|_violation| {
+ self.used_unsafe_blocks.insert(hir_id);
}),
};
- new_used_unsafe_blocks
- .into_iter()
- .for_each(|(hir_id, usage_data)| update_entry(self, hir_id, usage_data));
+ new_used_unsafe_blocks.into_iter().for_each(|hir_id| {
+ self.used_unsafe_blocks.insert(hir_id);
+ });
}
fn check_mut_borrowing_layout_constrained_field(
&mut self,
@@ -411,34 +383,28 @@ enum Context {
struct UnusedUnsafeVisitor<'a, 'tcx> {
tcx: TyCtxt<'tcx>,
- used_unsafe_blocks: &'a FxHashMap<HirId, UsedUnsafeBlockData>,
+ used_unsafe_blocks: &'a FxHashSet<HirId>,
context: Context,
unused_unsafes: &'a mut Vec<(HirId, UnusedUnsafe)>,
}
impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> {
fn visit_block(&mut self, block: &'tcx hir::Block<'tcx>) {
- use UsedUnsafeBlockData::{AllAllowedInUnsafeFn, SomeDisallowedInUnsafeFn};
-
if let hir::BlockCheckMode::UnsafeBlock(hir::UnsafeSource::UserProvided) = block.rules {
let used = match self.tcx.lint_level_at_node(UNUSED_UNSAFE, block.hir_id) {
- (Level::Allow, _) => Some(SomeDisallowedInUnsafeFn),
- _ => self.used_unsafe_blocks.get(&block.hir_id).copied(),
+ (Level::Allow, _) => true,
+ _ => self.used_unsafe_blocks.contains(&block.hir_id),
};
let unused_unsafe = match (self.context, used) {
- (_, None) => UnusedUnsafe::Unused,
- (Context::Safe, Some(_))
- | (Context::UnsafeFn(_), Some(SomeDisallowedInUnsafeFn)) => {
+ (_, false) => UnusedUnsafe::Unused,
+ (Context::Safe, true) | (Context::UnsafeFn(_), true) => {
let previous_context = self.context;
self.context = Context::UnsafeBlock(block.hir_id);
intravisit::walk_block(self, block);
self.context = previous_context;
return;
}
- (Context::UnsafeFn(hir_id), Some(AllAllowedInUnsafeFn(lint_root))) => {
- UnusedUnsafe::InUnsafeFn(hir_id, lint_root)
- }
- (Context::UnsafeBlock(hir_id), Some(_)) => UnusedUnsafe::InUnsafeBlock(hir_id),
+ (Context::UnsafeBlock(hir_id), true) => UnusedUnsafe::InUnsafeBlock(hir_id),
};
self.unused_unsafes.push((block.hir_id, unused_unsafe));
}
@@ -462,7 +428,7 @@ impl<'tcx> intravisit::Visitor<'tcx> for UnusedUnsafeVisitor<'_, 'tcx> {
fn check_unused_unsafe(
tcx: TyCtxt<'_>,
def_id: LocalDefId,
- used_unsafe_blocks: &FxHashMap<HirId, UsedUnsafeBlockData>,
+ used_unsafe_blocks: &FxHashSet<HirId>,
) -> Vec<(HirId, UnusedUnsafe)> {
let body_id = tcx.hir().maybe_body_owned_by(def_id);
@@ -535,25 +501,6 @@ fn report_unused_unsafe(tcx: TyCtxt<'_>, kind: UnusedUnsafe, id: HirId) {
"because it's nested under this `unsafe` block",
);
}
- UnusedUnsafe::InUnsafeFn(id, usage_lint_root) => {
- db.span_label(
- tcx.sess.source_map().guess_head_span(tcx.hir().span(id)),
- "because it's nested under this `unsafe` fn",
- )
- .note(
- "this `unsafe` block does contain unsafe operations, \
- but those are already allowed in an `unsafe fn`",
- );
- let (level, source) =
- tcx.lint_level_at_node(UNSAFE_OP_IN_UNSAFE_FN, usage_lint_root);
- assert_eq!(level, Level::Allow);
- lint::explain_lint_level_source(
- UNSAFE_OP_IN_UNSAFE_FN,
- Level::Allow,
- source,
- &mut db,
- );
- }
}
db.emit();
diff --git a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
index 611d29a4e..3378923c2 100644
--- a/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
+++ b/compiler/rustc_mir_transform/src/cleanup_post_borrowck.rs
@@ -33,7 +33,7 @@ pub struct DeleteNonCodegenStatements<'tcx> {
impl<'tcx> MirPass<'tcx> for CleanupNonCodegenStatements {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
let mut delete = DeleteNonCodegenStatements { tcx };
- delete.visit_body(body);
+ delete.visit_body_preserves_cfg(body);
body.user_type_annotations.raw.clear();
for decl in &mut body.local_decls {
diff --git a/compiler/rustc_mir_transform/src/const_goto.rs b/compiler/rustc_mir_transform/src/const_goto.rs
index 5acf939f0..0a305a402 100644
--- a/compiler/rustc_mir_transform/src/const_goto.rs
+++ b/compiler/rustc_mir_transform/src/const_goto.rs
@@ -61,14 +61,14 @@ impl<'tcx> Visitor<'tcx> for ConstGotoOptimizationFinder<'_, 'tcx> {
let _: Option<_> = try {
let target = terminator.kind.as_goto()?;
// We only apply this optimization if the last statement is a const assignment
- let last_statement = self.body.basic_blocks()[location.block].statements.last()?;
+ let last_statement = self.body.basic_blocks[location.block].statements.last()?;
if let (place, Rvalue::Use(Operand::Constant(_const))) =
last_statement.kind.as_assign()?
{
// We found a constant being assigned to `place`.
// Now check that the target of this Goto switches on this place.
- let target_bb = &self.body.basic_blocks()[target];
+ let target_bb = &self.body.basic_blocks[target];
// The `StorageDead(..)` statement does not affect the functionality of mir.
// We can move this part of the statement up to the predecessor.
diff --git a/compiler/rustc_mir_transform/src/const_prop.rs b/compiler/rustc_mir_transform/src/const_prop.rs
index fbc0a767f..7a2a15a84 100644
--- a/compiler/rustc_mir_transform/src/const_prop.rs
+++ b/compiler/rustc_mir_transform/src/const_prop.rs
@@ -28,7 +28,7 @@ use crate::MirPass;
use rustc_const_eval::interpret::{
self, compile_time_machine, AllocId, ConstAllocation, ConstValue, CtfeValidationMode, Frame,
ImmTy, Immediate, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, PlaceTy,
- Pointer, Scalar, ScalarMaybeUninit, StackPopCleanup, StackPopUnwind,
+ Pointer, Scalar, StackPopCleanup, StackPopUnwind,
};
/// The maximum number of bytes that we'll allocate space for a local or the return value.
@@ -131,7 +131,7 @@ impl<'tcx> MirPass<'tcx> for ConstProp {
let dummy_body = &Body::new(
body.source,
- body.basic_blocks().clone(),
+ (*body.basic_blocks).clone(),
body.source_scopes.clone(),
body.local_decls.clone(),
Default::default(),
@@ -183,6 +183,18 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
type MemoryKind = !;
+ #[inline(always)]
+ fn enforce_alignment(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool {
+ // We do not check for alignment to avoid having to carry an `Align`
+ // in `ConstValue::ByRef`.
+ false
+ }
+
+ #[inline(always)]
+ fn enforce_validity(_ecx: &InterpCx<'mir, 'tcx, Self>) -> bool {
+ false // for now, we don't enforce validity
+ }
+
fn load_mir(
_ecx: &InterpCx<'mir, 'tcx, Self>,
_instance: ty::InstanceDef<'tcx>,
@@ -231,24 +243,6 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for ConstPropMachine<'mir, 'tcx>
throw_machine_stop_str!("pointer arithmetic or comparisons aren't supported in ConstProp")
}
- fn access_local<'a>(
- frame: &'a Frame<'mir, 'tcx, Self::Provenance, Self::FrameExtra>,
- local: Local,
- ) -> InterpResult<'tcx, &'a interpret::Operand<Self::Provenance>> {
- let l = &frame.locals[local];
-
- if matches!(
- l.value,
- LocalValue::Live(interpret::Operand::Immediate(interpret::Immediate::Uninit))
- ) {
- // For us "uninit" means "we don't know its value, might be initiailized or not".
- // So stop here.
- throw_machine_stop_str!("tried to access alocal with unknown value ")
- }
-
- l.access()
- }
-
fn access_local_mut<'a>(
ecx: &'a mut InterpCx<'mir, 'tcx, Self>,
frame: usize,
@@ -419,7 +413,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
let op = match self.ecx.eval_place_to_op(place, None) {
- Ok(op) => op,
+ Ok(op) => {
+ if matches!(*op, interpret::Operand::Immediate(Immediate::Uninit)) {
+ // Make sure nobody accidentally uses this value.
+ return None;
+ }
+ op
+ }
Err(e) => {
trace!("get_const failed: {}", e);
return None;
@@ -428,7 +428,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
// Try to read the local as an immediate so that if it is representable as a scalar, we can
// handle it as such, but otherwise, just return the value as is.
- Some(match self.ecx.read_immediate_raw(&op, /*force*/ false) {
+ Some(match self.ecx.read_immediate_raw(&op) {
Ok(Ok(imm)) => imm.into(),
_ => op,
})
@@ -520,8 +520,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
let left_ty = left.ty(self.local_decls, self.tcx);
let left_size = self.ecx.layout_of(left_ty).ok()?.size;
let right_size = r.layout.size;
- let r_bits = r.to_scalar().ok();
- let r_bits = r_bits.and_then(|r| r.to_bits(right_size).ok());
+ let r_bits = r.to_scalar().to_bits(right_size).ok();
if r_bits.map_or(false, |b| b >= left_size.bits() as u128) {
return None;
}
@@ -550,7 +549,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
// and use it to do const-prop here and everywhere else
// where it makes sense.
if let interpret::Operand::Immediate(interpret::Immediate::Scalar(
- ScalarMaybeUninit::Scalar(scalar),
+ scalar,
)) = *value
{
*operand = self.operand_from_scalar(
@@ -632,6 +631,14 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
if rvalue.needs_subst() {
return None;
}
+ if !rvalue
+ .ty(&self.ecx.frame().body.local_decls, *self.ecx.tcx)
+ .is_sized(self.ecx.tcx, self.param_env)
+ {
+ // the interpreter doesn't support unsized locals (only unsized arguments),
+ // but rustc does (in a kinda broken way), so we have to skip them here
+ return None;
+ }
if self.tcx.sess.mir_opt_level() >= 4 {
self.eval_rvalue_with_identities(rvalue, place)
@@ -649,21 +656,23 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
self.use_ecx(|this| match rvalue {
Rvalue::BinaryOp(op, box (left, right))
| Rvalue::CheckedBinaryOp(op, box (left, right)) => {
- let l = this.ecx.eval_operand(left, None);
- let r = this.ecx.eval_operand(right, None);
+ let l = this.ecx.eval_operand(left, None).and_then(|x| this.ecx.read_immediate(&x));
+ let r =
+ this.ecx.eval_operand(right, None).and_then(|x| this.ecx.read_immediate(&x));
let const_arg = match (l, r) {
- (Ok(ref x), Err(_)) | (Err(_), Ok(ref x)) => this.ecx.read_immediate(x)?,
- (Err(e), Err(_)) => return Err(e),
- (Ok(_), Ok(_)) => return this.ecx.eval_rvalue_into_place(rvalue, place),
+ (Ok(x), Err(_)) | (Err(_), Ok(x)) => x, // exactly one side is known
+ (Err(e), Err(_)) => return Err(e), // neither side is known
+ (Ok(_), Ok(_)) => return this.ecx.eval_rvalue_into_place(rvalue, place), // both sides are known
};
if !matches!(const_arg.layout.abi, abi::Abi::Scalar(..)) {
// We cannot handle Scalar Pair stuff.
- return this.ecx.eval_rvalue_into_place(rvalue, place);
+ // No point in calling `eval_rvalue_into_place`, since only one side is known
+ throw_machine_stop_str!("cannot optimize this")
}
- let arg_value = const_arg.to_scalar()?.to_bits(const_arg.layout.size)?;
+ let arg_value = const_arg.to_scalar().to_bits(const_arg.layout.size)?;
let dest = this.ecx.eval_place(place)?;
match op {
@@ -677,7 +686,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
BinOp::Mul if const_arg.layout.ty.is_integral() && arg_value == 0 => {
if let Rvalue::CheckedBinaryOp(_, _) = rvalue {
let val = Immediate::ScalarPair(
- const_arg.to_scalar()?.into(),
+ const_arg.to_scalar().into(),
Scalar::from_bool(false).into(),
);
this.ecx.write_immediate(val, &dest)
@@ -685,7 +694,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
this.ecx.write_immediate(*const_arg, &dest)
}
}
- _ => this.ecx.eval_rvalue_into_place(rvalue, place),
+ _ => throw_machine_stop_str!("cannot optimize this"),
}
}
_ => this.ecx.eval_rvalue_into_place(rvalue, place),
@@ -731,21 +740,18 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
}
// FIXME> figure out what to do when read_immediate_raw fails
- let imm = self.use_ecx(|this| this.ecx.read_immediate_raw(value, /*force*/ false));
+ let imm = self.use_ecx(|this| this.ecx.read_immediate_raw(value));
if let Some(Ok(imm)) = imm {
match *imm {
- interpret::Immediate::Scalar(ScalarMaybeUninit::Scalar(scalar)) => {
+ interpret::Immediate::Scalar(scalar) => {
*rval = Rvalue::Use(self.operand_from_scalar(
scalar,
value.layout.ty,
source_info.span,
));
}
- Immediate::ScalarPair(
- ScalarMaybeUninit::Scalar(_),
- ScalarMaybeUninit::Scalar(_),
- ) => {
+ Immediate::ScalarPair(..) => {
// Found a value represented as a pair. For now only do const-prop if the type
// of `rvalue` is also a tuple with two scalars.
// FIXME: enable the general case stated above ^.
@@ -800,13 +806,10 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
}
match **op {
- interpret::Operand::Immediate(Immediate::Scalar(ScalarMaybeUninit::Scalar(s))) => {
- s.try_to_int().is_ok()
+ interpret::Operand::Immediate(Immediate::Scalar(s)) => s.try_to_int().is_ok(),
+ interpret::Operand::Immediate(Immediate::ScalarPair(l, r)) => {
+ l.try_to_int().is_ok() && r.try_to_int().is_ok()
}
- interpret::Operand::Immediate(Immediate::ScalarPair(
- ScalarMaybeUninit::Scalar(l),
- ScalarMaybeUninit::Scalar(r),
- )) => l.try_to_int().is_ok() && r.try_to_int().is_ok(),
_ => false,
}
}
@@ -951,7 +954,7 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
}
fn visit_body(&mut self, body: &mut Body<'tcx>) {
- for (bb, data) in body.basic_blocks_mut().iter_enumerated_mut() {
+ for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() {
self.visit_basic_block_data(bb, data);
}
}
@@ -1063,26 +1066,28 @@ impl<'tcx> MutVisitor<'tcx> for ConstPropagator<'_, 'tcx> {
let source_info = terminator.source_info;
self.source_info = Some(source_info);
self.super_terminator(terminator, location);
+ // Do NOT early return in this function, it does some crucial fixup of the state at the end!
match &mut terminator.kind {
TerminatorKind::Assert { expected, ref mut cond, .. } => {
if let Some(ref value) = self.eval_operand(&cond) {
trace!("assertion on {:?} should be {:?}", value, expected);
- let expected = ScalarMaybeUninit::from(Scalar::from_bool(*expected));
- let value_const = self.ecx.read_scalar(&value).unwrap();
- if expected != value_const {
- // Poison all places this operand references so that further code
- // doesn't use the invalid value
- match cond {
- Operand::Move(ref place) | Operand::Copy(ref place) => {
- Self::remove_const(&mut self.ecx, place.local);
+ let expected = Scalar::from_bool(*expected);
+ // FIXME should be used use_ecx rather than a local match... but we have
+ // quite a few of these read_scalar/read_immediate that need fixing.
+ if let Ok(value_const) = self.ecx.read_scalar(&value) {
+ if expected != value_const {
+ // Poison all places this operand references so that further code
+ // doesn't use the invalid value
+ match cond {
+ Operand::Move(ref place) | Operand::Copy(ref place) => {
+ Self::remove_const(&mut self.ecx, place.local);
+ }
+ Operand::Constant(_) => {}
}
- Operand::Constant(_) => {}
- }
- } else {
- if self.should_const_prop(value) {
- if let ScalarMaybeUninit::Scalar(scalar) = value_const {
+ } else {
+ if self.should_const_prop(value) {
*cond = self.operand_from_scalar(
- scalar,
+ value_const,
self.tcx.types.bool,
source_info.span,
);
diff --git a/compiler/rustc_mir_transform/src/const_prop_lint.rs b/compiler/rustc_mir_transform/src/const_prop_lint.rs
index c2ea55af4..37e78f4ac 100644
--- a/compiler/rustc_mir_transform/src/const_prop_lint.rs
+++ b/compiler/rustc_mir_transform/src/const_prop_lint.rs
@@ -6,9 +6,9 @@ use crate::const_prop::ConstPropMachine;
use crate::const_prop::ConstPropMode;
use crate::MirLint;
use rustc_const_eval::const_eval::ConstEvalErr;
+use rustc_const_eval::interpret::Immediate;
use rustc_const_eval::interpret::{
- self, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, Scalar,
- ScalarMaybeUninit, StackPopCleanup,
+ self, InterpCx, InterpResult, LocalState, LocalValue, MemoryKind, OpTy, Scalar, StackPopCleanup,
};
use rustc_hir::def::DefKind;
use rustc_hir::HirId;
@@ -22,9 +22,7 @@ use rustc_middle::mir::{
};
use rustc_middle::ty::layout::{LayoutError, LayoutOf, LayoutOfHelpers, TyAndLayout};
use rustc_middle::ty::subst::{InternalSubsts, Subst};
-use rustc_middle::ty::{
- self, ConstInt, ConstKind, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitable,
-};
+use rustc_middle::ty::{self, ConstInt, Instance, ParamEnv, ScalarInt, Ty, TyCtxt, TypeVisitable};
use rustc_session::lint;
use rustc_span::Span;
use rustc_target::abi::{HasDataLayout, Size, TargetDataLayout};
@@ -106,7 +104,7 @@ impl<'tcx> MirLint<'tcx> for ConstProp {
let dummy_body = &Body::new(
body.source,
- body.basic_blocks().clone(),
+ (*body.basic_blocks).clone(),
body.source_scopes.clone(),
body.local_decls.clone(),
Default::default(),
@@ -230,7 +228,13 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
fn get_const(&self, place: Place<'tcx>) -> Option<OpTy<'tcx>> {
let op = match self.ecx.eval_place_to_op(place, None) {
- Ok(op) => op,
+ Ok(op) => {
+ if matches!(*op, interpret::Operand::Immediate(Immediate::Uninit)) {
+ // Make sure nobody accidentally uses this value.
+ return None;
+ }
+ op
+ }
Err(e) => {
trace!("get_const failed: {}", e);
return None;
@@ -239,7 +243,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
// Try to read the local as an immediate so that if it is representable as a scalar, we can
// handle it as such, but otherwise, just return the value as is.
- Some(match self.ecx.read_immediate_raw(&op, /*force*/ false) {
+ Some(match self.ecx.read_immediate_raw(&op) {
Ok(Ok(imm)) => imm.into(),
_ => op,
})
@@ -295,18 +299,15 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
let err = ConstEvalErr::new(&self.ecx, error, Some(c.span));
if let Some(lint_root) = self.lint_root(source_info) {
let lint_only = match c.literal {
- ConstantKind::Ty(ct) => match ct.kind() {
+ ConstantKind::Ty(ct) => ct.needs_subst(),
+ ConstantKind::Unevaluated(
+ ty::Unevaluated { def: _, substs: _, promoted: Some(_) },
+ _,
+ ) => {
// Promoteds must lint and not error as the user didn't ask for them
- ConstKind::Unevaluated(ty::Unevaluated {
- def: _,
- substs: _,
- promoted: Some(_),
- }) => true,
- // Out of backwards compatibility we cannot report hard errors in unused
- // generic functions using associated constants of the generic parameters.
- _ => c.literal.needs_subst(),
- },
- ConstantKind::Val(_, ty) => ty.needs_subst(),
+ true
+ }
+ ConstantKind::Unevaluated(..) | ConstantKind::Val(..) => c.needs_subst(),
};
if lint_only {
// Out of backwards compatibility we cannot report hard errors in unused
@@ -401,8 +402,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
let left_ty = left.ty(self.local_decls, self.tcx);
let left_size = self.ecx.layout_of(left_ty).ok()?.size;
let right_size = r.layout.size;
- let r_bits = r.to_scalar().ok();
- let r_bits = r_bits.and_then(|r| r.to_bits(right_size).ok());
+ let r_bits = r.to_scalar().to_bits(right_size).ok();
if r_bits.map_or(false, |b| b >= left_size.bits() as u128) {
debug!("check_binary_op: reporting assert for {:?}", source_info);
self.report_assert_as_lint(
@@ -517,6 +517,14 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
if rvalue.needs_subst() {
return None;
}
+ if !rvalue
+ .ty(&self.ecx.frame().body.local_decls, *self.ecx.tcx)
+ .is_sized(self.ecx.tcx, self.param_env)
+ {
+ // the interpreter doesn't support unsized locals (only unsized arguments),
+ // but rustc does (in a kinda broken way), so we have to skip them here
+ return None;
+ }
self.use_ecx(source_info, |this| this.ecx.eval_rvalue_into_place(rvalue, place))
}
@@ -524,7 +532,7 @@ impl<'mir, 'tcx> ConstPropagator<'mir, 'tcx> {
impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> {
fn visit_body(&mut self, body: &Body<'tcx>) {
- for (bb, data) in body.basic_blocks().iter_enumerated() {
+ for (bb, data) in body.basic_blocks.iter_enumerated() {
self.visit_basic_block_data(bb, data);
}
}
@@ -625,8 +633,12 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> {
TerminatorKind::Assert { expected, ref msg, ref cond, .. } => {
if let Some(ref value) = self.eval_operand(&cond, source_info) {
trace!("assertion on {:?} should be {:?}", value, expected);
- let expected = ScalarMaybeUninit::from(Scalar::from_bool(*expected));
- let value_const = self.ecx.read_scalar(&value).unwrap();
+ let expected = Scalar::from_bool(*expected);
+ let Ok(value_const) = self.ecx.read_scalar(&value) else {
+ // FIXME should be used use_ecx rather than a local match... but we have
+ // quite a few of these read_scalar/read_immediate that need fixing.
+ return
+ };
if expected != value_const {
enum DbgVal<T> {
Val(T),
@@ -643,9 +655,9 @@ impl<'tcx> Visitor<'tcx> for ConstPropagator<'_, 'tcx> {
let mut eval_to_int = |op| {
// This can be `None` if the lhs wasn't const propagated and we just
// triggered the assert on the value of the rhs.
- self.eval_operand(op, source_info).map_or(DbgVal::Underscore, |op| {
- DbgVal::Val(self.ecx.read_immediate(&op).unwrap().to_const_int())
- })
+ self.eval_operand(op, source_info)
+ .and_then(|op| self.ecx.read_immediate(&op).ok())
+ .map_or(DbgVal::Underscore, |op| DbgVal::Val(op.to_const_int()))
};
let msg = match msg {
AssertKind::DivisionByZero(op) => {
diff --git a/compiler/rustc_mir_transform/src/coverage/graph.rs b/compiler/rustc_mir_transform/src/coverage/graph.rs
index 759ea7cd3..782129be0 100644
--- a/compiler/rustc_mir_transform/src/coverage/graph.rs
+++ b/compiler/rustc_mir_transform/src/coverage/graph.rs
@@ -713,7 +713,7 @@ impl<
ShortCircuitPreorder {
body,
- visited: BitSet::new_empty(body.basic_blocks().len()),
+ visited: BitSet::new_empty(body.basic_blocks.len()),
worklist,
filtered_successors,
}
@@ -747,7 +747,7 @@ impl<
}
fn size_hint(&self) -> (usize, Option<usize>) {
- let size = self.body.basic_blocks().len() - self.visited.count();
+ let size = self.body.basic_blocks.len() - self.visited.count();
(size, Some(size))
}
}
diff --git a/compiler/rustc_mir_transform/src/coverage/mod.rs b/compiler/rustc_mir_transform/src/coverage/mod.rs
index 2619626a5..604810144 100644
--- a/compiler/rustc_mir_transform/src/coverage/mod.rs
+++ b/compiler/rustc_mir_transform/src/coverage/mod.rs
@@ -80,7 +80,7 @@ impl<'tcx> MirPass<'tcx> for InstrumentCoverage {
return;
}
- match mir_body.basic_blocks()[mir::START_BLOCK].terminator().kind {
+ match mir_body.basic_blocks[mir::START_BLOCK].terminator().kind {
TerminatorKind::Unreachable => {
trace!("InstrumentCoverage skipped for unreachable `START_BLOCK`");
return;
@@ -541,7 +541,7 @@ fn fn_sig_and_body<'tcx>(
// to HIR for it.
let hir_node = tcx.hir().get_if_local(def_id).expect("expected DefId is local");
let fn_body_id = hir::map::associated_body(hir_node).expect("HIR node is a function with body");
- (hir::map::fn_sig(hir_node), tcx.hir().body(fn_body_id))
+ (hir_node.fn_sig(), tcx.hir().body(fn_body_id))
}
fn get_body_span<'tcx>(
diff --git a/compiler/rustc_mir_transform/src/coverage/query.rs b/compiler/rustc_mir_transform/src/coverage/query.rs
index 9d02f58ae..dc1e68b25 100644
--- a/compiler/rustc_mir_transform/src/coverage/query.rs
+++ b/compiler/rustc_mir_transform/src/coverage/query.rs
@@ -84,7 +84,7 @@ impl CoverageVisitor {
}
fn visit_body(&mut self, body: &Body<'_>) {
- for bb_data in body.basic_blocks().iter() {
+ for bb_data in body.basic_blocks.iter() {
for statement in bb_data.statements.iter() {
if let StatementKind::Coverage(box ref coverage) = statement.kind {
if is_inlined(body, statement) {
@@ -138,7 +138,7 @@ fn coverageinfo<'tcx>(tcx: TyCtxt<'tcx>, instance_def: ty::InstanceDef<'tcx>) ->
fn covered_code_regions<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId) -> Vec<&'tcx CodeRegion> {
let body = mir_body(tcx, def_id);
- body.basic_blocks()
+ body.basic_blocks
.iter()
.flat_map(|data| {
data.statements.iter().filter_map(|statement| match statement.kind {
diff --git a/compiler/rustc_mir_transform/src/coverage/spans.rs b/compiler/rustc_mir_transform/src/coverage/spans.rs
index 423e78317..9f842c929 100644
--- a/compiler/rustc_mir_transform/src/coverage/spans.rs
+++ b/compiler/rustc_mir_transform/src/coverage/spans.rs
@@ -825,7 +825,7 @@ pub(super) fn filtered_statement_span(statement: &Statement<'_>) -> Option<Span>
// Retain spans from all other statements
StatementKind::FakeRead(box (_, _)) // Not including `ForGuardBinding`
- | StatementKind::CopyNonOverlapping(..)
+ | StatementKind::Intrinsic(..)
| StatementKind::Assign(_)
| StatementKind::SetDiscriminant { .. }
| StatementKind::Deinit(..)
diff --git a/compiler/rustc_mir_transform/src/coverage/tests.rs b/compiler/rustc_mir_transform/src/coverage/tests.rs
index 6380f0352..9c9ed5fa5 100644
--- a/compiler/rustc_mir_transform/src/coverage/tests.rs
+++ b/compiler/rustc_mir_transform/src/coverage/tests.rs
@@ -176,7 +176,7 @@ fn debug_basic_blocks<'tcx>(mir_body: &Body<'tcx>) -> String {
format!(
"{:?}",
mir_body
- .basic_blocks()
+ .basic_blocks
.iter_enumerated()
.map(|(bb, data)| {
let term = &data.terminator();
@@ -213,7 +213,7 @@ fn print_mir_graphviz(name: &str, mir_body: &Body<'_>) {
"digraph {} {{\n{}\n}}",
name,
mir_body
- .basic_blocks()
+ .basic_blocks
.iter_enumerated()
.map(|(bb, data)| {
format!(
@@ -653,7 +653,7 @@ fn test_traverse_coverage_with_loops() {
fn synthesize_body_span_from_terminators(mir_body: &Body<'_>) -> Span {
let mut some_span: Option<Span> = None;
- for (_, data) in mir_body.basic_blocks().iter_enumerated() {
+ for (_, data) in mir_body.basic_blocks.iter_enumerated() {
let term_span = data.terminator().source_info.span;
if let Some(span) = some_span.as_mut() {
*span = span.to(term_span);
diff --git a/compiler/rustc_mir_transform/src/dead_store_elimination.rs b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
index 9163672f5..3f3870cc7 100644
--- a/compiler/rustc_mir_transform/src/dead_store_elimination.rs
+++ b/compiler/rustc_mir_transform/src/dead_store_elimination.rs
@@ -52,7 +52,7 @@ pub fn eliminate<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, borrowed: &BitS
| StatementKind::StorageLive(_)
| StatementKind::StorageDead(_)
| StatementKind::Coverage(_)
- | StatementKind::CopyNonOverlapping(_)
+ | StatementKind::Intrinsic(_)
| StatementKind::Nop => (),
StatementKind::FakeRead(_) | StatementKind::AscribeUserType(_, _) => {
diff --git a/compiler/rustc_mir_transform/src/deaggregator.rs b/compiler/rustc_mir_transform/src/deaggregator.rs
index b93fe5879..fe272de20 100644
--- a/compiler/rustc_mir_transform/src/deaggregator.rs
+++ b/compiler/rustc_mir_transform/src/deaggregator.rs
@@ -6,10 +6,6 @@ use rustc_middle::ty::TyCtxt;
pub struct Deaggregator;
impl<'tcx> MirPass<'tcx> for Deaggregator {
- fn phase_change(&self) -> Option<MirPhase> {
- Some(MirPhase::Deaggregated)
- }
-
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
let basic_blocks = body.basic_blocks.as_mut_preserves_cfg();
for bb in basic_blocks {
diff --git a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
index d1977ed49..909116a77 100644
--- a/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
+++ b/compiler/rustc_mir_transform/src/deduplicate_blocks.rs
@@ -58,7 +58,7 @@ fn find_duplicates(body: &Body<'_>) -> FxHashMap<BasicBlock, BasicBlock> {
let mut duplicates = FxHashMap::default();
let bbs_to_go_through =
- body.basic_blocks().iter_enumerated().filter(|(_, bbd)| !bbd.is_cleanup).count();
+ body.basic_blocks.iter_enumerated().filter(|(_, bbd)| !bbd.is_cleanup).count();
let mut same_hashes =
FxHashMap::with_capacity_and_hasher(bbs_to_go_through, Default::default());
@@ -71,8 +71,7 @@ fn find_duplicates(body: &Body<'_>) -> FxHashMap<BasicBlock, BasicBlock> {
// When we see bb1, we see that it is a duplicate of bb3, and therefore insert it in the duplicates list
// with replacement bb3.
// When the duplicates are removed, we will end up with only bb3.
- for (bb, bbd) in body.basic_blocks().iter_enumerated().rev().filter(|(_, bbd)| !bbd.is_cleanup)
- {
+ for (bb, bbd) in body.basic_blocks.iter_enumerated().rev().filter(|(_, bbd)| !bbd.is_cleanup) {
// Basic blocks can get really big, so to avoid checking for duplicates in basic blocks
// that are unlikely to have duplicates, we stop early. The early bail number has been
// found experimentally by eprintln while compiling the crates in the rustc-perf suite.
diff --git a/compiler/rustc_mir_transform/src/deref_separator.rs b/compiler/rustc_mir_transform/src/deref_separator.rs
index a00bb16f7..7508df92d 100644
--- a/compiler/rustc_mir_transform/src/deref_separator.rs
+++ b/compiler/rustc_mir_transform/src/deref_separator.rs
@@ -28,8 +28,6 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> {
let mut last_len = 0;
let mut last_deref_idx = 0;
- let mut prev_temp: Option<Local> = None;
-
for (idx, elem) in place.projection[0..].iter().enumerate() {
if *elem == ProjectionElem::Deref {
last_deref_idx = idx;
@@ -39,14 +37,12 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> {
for (idx, (p_ref, p_elem)) in place.iter_projections().enumerate() {
if !p_ref.projection.is_empty() && p_elem == ProjectionElem::Deref {
let ty = p_ref.ty(&self.local_decls, self.tcx).ty;
- let temp = self.patcher.new_local_with_info(
+ let temp = self.patcher.new_internal_with_info(
ty,
self.local_decls[p_ref.local].source_info.span,
Some(Box::new(LocalInfo::DerefTemp)),
);
- self.patcher.add_statement(loc, StatementKind::StorageLive(temp));
-
// We are adding current p_ref's projections to our
// temp value, excluding projections we already covered.
let deref_place = Place::from(place_local)
@@ -66,22 +62,8 @@ impl<'tcx> MutVisitor<'tcx> for DerefChecker<'tcx> {
Place::from(temp).project_deeper(&place.projection[idx..], self.tcx);
*place = temp_place;
}
-
- // We are destroying the previous temp since it's no longer used.
- if let Some(prev_temp) = prev_temp {
- self.patcher.add_statement(loc, StatementKind::StorageDead(prev_temp));
- }
-
- prev_temp = Some(temp);
}
}
-
- // Since we won't be able to reach final temp, we destroy it outside the loop.
- if let Some(prev_temp) = prev_temp {
- let last_loc =
- Location { block: loc.block, statement_index: loc.statement_index + 1 };
- self.patcher.add_statement(last_loc, StatementKind::StorageDead(prev_temp));
- }
}
}
}
@@ -90,7 +72,7 @@ pub fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
let patch = MirPatch::new(body);
let mut checker = DerefChecker { tcx, patcher: patch, local_decls: body.local_decls.clone() };
- for (bb, data) in body.basic_blocks_mut().iter_enumerated_mut() {
+ for (bb, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() {
checker.visit_basic_block_data(bb, data);
}
@@ -100,6 +82,5 @@ pub fn deref_finder<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
impl<'tcx> MirPass<'tcx> for Derefer {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
deref_finder(tcx, body);
- body.phase = MirPhase::Derefered;
}
}
diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs
index 33572068f..9bc47613e 100644
--- a/compiler/rustc_mir_transform/src/dest_prop.rs
+++ b/compiler/rustc_mir_transform/src/dest_prop.rs
@@ -150,7 +150,7 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation {
def_id,
body.local_decls.len(),
relevant,
- body.basic_blocks().len()
+ body.basic_blocks.len()
);
if relevant > MAX_LOCALS {
warn!(
@@ -159,11 +159,11 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation {
);
return;
}
- if body.basic_blocks().len() > MAX_BLOCKS {
+ if body.basic_blocks.len() > MAX_BLOCKS {
warn!(
"too many blocks in {:?} ({}, max is {}), not optimizing",
def_id,
- body.basic_blocks().len(),
+ body.basic_blocks.len(),
MAX_BLOCKS
);
return;
@@ -537,7 +537,7 @@ impl<'a> Conflicts<'a> {
| StatementKind::FakeRead(..)
| StatementKind::AscribeUserType(..)
| StatementKind::Coverage(..)
- | StatementKind::CopyNonOverlapping(..)
+ | StatementKind::Intrinsic(..)
| StatementKind::Nop => {}
}
}
diff --git a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
index dba42f7af..32e738bbc 100644
--- a/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
+++ b/compiler/rustc_mir_transform/src/early_otherwise_branch.rs
@@ -104,8 +104,8 @@ impl<'tcx> MirPass<'tcx> for EarlyOtherwiseBranch {
let mut should_cleanup = false;
// Also consider newly generated bbs in the same pass
- for i in 0..body.basic_blocks().len() {
- let bbs = body.basic_blocks();
+ for i in 0..body.basic_blocks.len() {
+ let bbs = &*body.basic_blocks;
let parent = BasicBlock::from_usize(i);
let Some(opt_data) = evaluate_candidate(tcx, body, parent) else {
continue
@@ -316,7 +316,7 @@ fn evaluate_candidate<'tcx>(
body: &Body<'tcx>,
parent: BasicBlock,
) -> Option<OptimizationData<'tcx>> {
- let bbs = body.basic_blocks();
+ let bbs = &body.basic_blocks;
let TerminatorKind::SwitchInt {
targets,
switch_ty: parent_ty,
diff --git a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
index 44e3945d6..294af2455 100644
--- a/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_box_derefs.rs
@@ -69,10 +69,7 @@ impl<'tcx, 'a> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'tcx, 'a> {
let (unique_ty, nonnull_ty, ptr_ty) =
build_ptr_tys(tcx, base_ty.boxed_ty(), self.unique_did, self.nonnull_did);
- let ptr_local = self.patch.new_temp(ptr_ty, source_info.span);
- self.local_decls.push(LocalDecl::new(ptr_ty, source_info.span));
-
- self.patch.add_statement(location, StatementKind::StorageLive(ptr_local));
+ let ptr_local = self.patch.new_internal(ptr_ty, source_info.span);
self.patch.add_assign(
location,
@@ -84,11 +81,6 @@ impl<'tcx, 'a> MutVisitor<'tcx> for ElaborateBoxDerefVisitor<'tcx, 'a> {
);
place.local = ptr_local;
-
- self.patch.add_statement(
- Location { block: location.block, statement_index: location.statement_index + 1 },
- StatementKind::StorageDead(ptr_local),
- );
}
self.super_place(place, context, location);
@@ -115,34 +107,8 @@ impl<'tcx> MirPass<'tcx> for ElaborateBoxDerefs {
let mut visitor =
ElaborateBoxDerefVisitor { tcx, unique_did, nonnull_did, local_decls, patch };
- for (block, BasicBlockData { statements, terminator, .. }) in
- body.basic_blocks.as_mut().iter_enumerated_mut()
- {
- let mut index = 0;
- for statement in statements {
- let location = Location { block, statement_index: index };
- visitor.visit_statement(statement, location);
- index += 1;
- }
-
- if let Some(terminator) = terminator
- && !matches!(terminator.kind, TerminatorKind::Yield{..})
- {
- let location = Location { block, statement_index: index };
- visitor.visit_terminator(terminator, location);
- }
-
- let location = Location { block, statement_index: index };
- match terminator {
- // yielding into a box is handled when lowering generators
- Some(Terminator { kind: TerminatorKind::Yield { value, .. }, .. }) => {
- visitor.visit_operand(value, location);
- }
- Some(terminator) => {
- visitor.visit_terminator(terminator, location);
- }
- None => {}
- }
+ for (block, data) in body.basic_blocks.as_mut_preserves_cfg().iter_enumerated_mut() {
+ visitor.visit_basic_block_data(block, data);
}
visitor.patch.apply(body);
diff --git a/compiler/rustc_mir_transform/src/elaborate_drops.rs b/compiler/rustc_mir_transform/src/elaborate_drops.rs
index 9c1fcbaa6..65f4956d2 100644
--- a/compiler/rustc_mir_transform/src/elaborate_drops.rs
+++ b/compiler/rustc_mir_transform/src/elaborate_drops.rs
@@ -21,10 +21,6 @@ use std::fmt;
pub struct ElaborateDrops;
impl<'tcx> MirPass<'tcx> for ElaborateDrops {
- fn phase_change(&self) -> Option<MirPhase> {
- Some(MirPhase::DropsLowered)
- }
-
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
debug!("elaborate_drops({:?} @ {:?})", body.source, body.span);
@@ -89,13 +85,13 @@ fn find_dead_unwinds<'tcx>(
debug!("find_dead_unwinds({:?})", body.span);
// We only need to do this pass once, because unwind edges can only
// reach cleanup blocks, which can't have unwind edges themselves.
- let mut dead_unwinds = BitSet::new_empty(body.basic_blocks().len());
+ let mut dead_unwinds = BitSet::new_empty(body.basic_blocks.len());
let mut flow_inits = MaybeInitializedPlaces::new(tcx, body, &env)
.into_engine(tcx, body)
.pass_name("find_dead_unwinds")
.iterate_to_fixpoint()
.into_results_cursor(body);
- for (bb, bb_data) in body.basic_blocks().iter_enumerated() {
+ for (bb, bb_data) in body.basic_blocks.iter_enumerated() {
let place = match bb_data.terminator().kind {
TerminatorKind::Drop { ref place, unwind: Some(_), .. }
| TerminatorKind::DropAndReplace { ref place, unwind: Some(_), .. } => {
@@ -303,7 +299,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
}
fn collect_drop_flags(&mut self) {
- for (bb, data) in self.body.basic_blocks().iter_enumerated() {
+ for (bb, data) in self.body.basic_blocks.iter_enumerated() {
let terminator = data.terminator();
let place = match terminator.kind {
TerminatorKind::Drop { ref place, .. }
@@ -358,7 +354,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
}
fn elaborate_drops(&mut self) {
- for (bb, data) in self.body.basic_blocks().iter_enumerated() {
+ for (bb, data) in self.body.basic_blocks.iter_enumerated() {
let loc = Location { block: bb, statement_index: data.statements.len() };
let terminator = data.terminator();
@@ -515,7 +511,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
}
fn drop_flags_for_fn_rets(&mut self) {
- for (bb, data) in self.body.basic_blocks().iter_enumerated() {
+ for (bb, data) in self.body.basic_blocks.iter_enumerated() {
if let TerminatorKind::Call {
destination, target: Some(tgt), cleanup: Some(_), ..
} = data.terminator().kind
@@ -550,7 +546,7 @@ impl<'b, 'tcx> ElaborateDropsCtxt<'b, 'tcx> {
// drop flags by themselves, to avoid the drop flags being
// clobbered before they are read.
- for (bb, data) in self.body.basic_blocks().iter_enumerated() {
+ for (bb, data) in self.body.basic_blocks.iter_enumerated() {
debug!("drop_flags_for_locs({:?})", data);
for i in 0..(data.statements.len() + 1) {
debug!("drop_flag_for_locs: stmt {}", i);
diff --git a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs
index 7728fdaff..7522a50a8 100644
--- a/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs
+++ b/compiler/rustc_mir_transform/src/ffi_unwind_calls.rs
@@ -65,7 +65,7 @@ fn has_ffi_unwind_calls(tcx: TyCtxt<'_>, local_def_id: LocalDefId) -> bool {
let mut tainted = false;
- for block in body.basic_blocks() {
+ for block in body.basic_blocks.iter() {
if block.is_cleanup {
continue;
}
diff --git a/compiler/rustc_mir_transform/src/generator.rs b/compiler/rustc_mir_transform/src/generator.rs
index 91ecf3879..705cf776f 100644
--- a/compiler/rustc_mir_transform/src/generator.rs
+++ b/compiler/rustc_mir_transform/src/generator.rs
@@ -490,12 +490,12 @@ fn locals_live_across_suspend_points<'tcx>(
.iterate_to_fixpoint()
.into_results_cursor(body_ref);
- let mut storage_liveness_map = IndexVec::from_elem(None, body.basic_blocks());
+ let mut storage_liveness_map = IndexVec::from_elem(None, &body.basic_blocks);
let mut live_locals_at_suspension_points = Vec::new();
let mut source_info_at_suspension_points = Vec::new();
let mut live_locals_at_any_suspension_point = BitSet::new_empty(body.local_decls.len());
- for (block, data) in body.basic_blocks().iter_enumerated() {
+ for (block, data) in body.basic_blocks.iter_enumerated() {
if let TerminatorKind::Yield { .. } = data.terminator().kind {
let loc = Location { block, statement_index: data.statements.len() };
@@ -704,7 +704,7 @@ impl<'mir, 'tcx> rustc_mir_dataflow::ResultsVisitor<'mir, 'tcx>
impl StorageConflictVisitor<'_, '_, '_> {
fn apply_state(&mut self, flow_state: &BitSet<Local>, loc: Location) {
// Ignore unreachable blocks.
- if self.body.basic_blocks()[loc.block].terminator().kind == TerminatorKind::Unreachable {
+ if self.body.basic_blocks[loc.block].terminator().kind == TerminatorKind::Unreachable {
return;
}
@@ -886,7 +886,7 @@ fn elaborate_generator_drops<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
let mut elaborator = DropShimElaborator { body, patch: MirPatch::new(body), tcx, param_env };
- for (block, block_data) in body.basic_blocks().iter_enumerated() {
+ for (block, block_data) in body.basic_blocks.iter_enumerated() {
let (target, unwind, source_info) = match block_data.terminator() {
Terminator { source_info, kind: TerminatorKind::Drop { place, target, unwind } } => {
if let Some(local) = place.as_local() {
@@ -991,7 +991,7 @@ fn insert_panic_block<'tcx>(
body: &mut Body<'tcx>,
message: AssertMessage<'tcx>,
) -> BasicBlock {
- let assert_block = BasicBlock::new(body.basic_blocks().len());
+ let assert_block = BasicBlock::new(body.basic_blocks.len());
let term = TerminatorKind::Assert {
cond: Operand::Constant(Box::new(Constant {
span: body.span,
@@ -1021,7 +1021,7 @@ fn can_return<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>, param_env: ty::ParamEn
}
// If there's a return terminator the function may return.
- for block in body.basic_blocks() {
+ for block in body.basic_blocks.iter() {
if let TerminatorKind::Return = block.terminator().kind {
return true;
}
@@ -1038,7 +1038,7 @@ fn can_unwind<'tcx>(tcx: TyCtxt<'tcx>, body: &Body<'tcx>) -> bool {
}
// Unwinds can only start at certain terminators.
- for block in body.basic_blocks() {
+ for block in body.basic_blocks.iter() {
match block.terminator().kind {
// These never unwind.
TerminatorKind::Goto { .. }
@@ -1182,8 +1182,6 @@ fn create_cases<'tcx>(
transform: &TransformVisitor<'tcx>,
operation: Operation,
) -> Vec<(usize, BasicBlock)> {
- let tcx = transform.tcx;
-
let source_info = SourceInfo::outermost(body.span);
transform
@@ -1216,85 +1214,13 @@ fn create_cases<'tcx>(
if operation == Operation::Resume {
// Move the resume argument to the destination place of the `Yield` terminator
let resume_arg = Local::new(2); // 0 = return, 1 = self
-
- // handle `box yield` properly
- let box_place = if let [projection @ .., ProjectionElem::Deref] =
- &**point.resume_arg.projection
- {
- let box_place =
- Place::from(point.resume_arg.local).project_deeper(projection, tcx);
-
- let box_ty = box_place.ty(&body.local_decls, tcx).ty;
-
- if box_ty.is_box() { Some((box_place, box_ty)) } else { None }
- } else {
- None
- };
-
- if let Some((box_place, box_ty)) = box_place {
- let unique_did = box_ty
- .ty_adt_def()
- .expect("expected Box to be an Adt")
- .non_enum_variant()
- .fields[0]
- .did;
-
- let Some(nonnull_def) = tcx.type_of(unique_did).ty_adt_def() else {
- span_bug!(tcx.def_span(unique_did), "expected Box to contain Unique")
- };
-
- let nonnull_did = nonnull_def.non_enum_variant().fields[0].did;
-
- let (unique_ty, nonnull_ty, ptr_ty) =
- crate::elaborate_box_derefs::build_ptr_tys(
- tcx,
- box_ty.boxed_ty(),
- unique_did,
- nonnull_did,
- );
-
- let ptr_local = body.local_decls.push(LocalDecl::new(ptr_ty, body.span));
-
- statements.push(Statement {
- source_info,
- kind: StatementKind::StorageLive(ptr_local),
- });
-
- statements.push(Statement {
- source_info,
- kind: StatementKind::Assign(Box::new((
- Place::from(ptr_local),
- Rvalue::Use(Operand::Copy(box_place.project_deeper(
- &crate::elaborate_box_derefs::build_projection(
- unique_ty, nonnull_ty, ptr_ty,
- ),
- tcx,
- ))),
- ))),
- });
-
- statements.push(Statement {
- source_info,
- kind: StatementKind::Assign(Box::new((
- Place::from(ptr_local)
- .project_deeper(&[ProjectionElem::Deref], tcx),
- Rvalue::Use(Operand::Move(resume_arg.into())),
- ))),
- });
-
- statements.push(Statement {
- source_info,
- kind: StatementKind::StorageDead(ptr_local),
- });
- } else {
- statements.push(Statement {
- source_info,
- kind: StatementKind::Assign(Box::new((
- point.resume_arg,
- Rvalue::Use(Operand::Move(resume_arg.into())),
- ))),
- });
- }
+ statements.push(Statement {
+ source_info,
+ kind: StatementKind::Assign(Box::new((
+ point.resume_arg,
+ Rvalue::Use(Operand::Move(resume_arg.into())),
+ ))),
+ });
}
// Then jump to the real target
@@ -1314,10 +1240,6 @@ fn create_cases<'tcx>(
}
impl<'tcx> MirPass<'tcx> for StateTransform {
- fn phase_change(&self) -> Option<MirPhase> {
- Some(MirPhase::GeneratorsLowered)
- }
-
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
let Some(yield_ty) = body.yield_ty() else {
// This only applies to generators
@@ -1530,7 +1452,7 @@ impl<'tcx> Visitor<'tcx> for EnsureGeneratorFieldAssignmentsNeverAlias<'_> {
| StatementKind::Retag(..)
| StatementKind::AscribeUserType(..)
| StatementKind::Coverage(..)
- | StatementKind::CopyNonOverlapping(..)
+ | StatementKind::Intrinsic(..)
| StatementKind::Nop => {}
}
}
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs
index 76b1522f3..d00a384cb 100644
--- a/compiler/rustc_mir_transform/src/inline.rs
+++ b/compiler/rustc_mir_transform/src/inline.rs
@@ -8,8 +8,11 @@ use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}
use rustc_middle::mir::visit::*;
use rustc_middle::mir::*;
use rustc_middle::ty::subst::Subst;
-use rustc_middle::ty::{self, ConstKind, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
+use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt};
+use rustc_session::config::OptLevel;
+use rustc_span::def_id::DefId;
use rustc_span::{hygiene::ExpnKind, ExpnData, LocalExpnId, Span};
+use rustc_target::abi::VariantIdx;
use rustc_target::spec::abi::Abi;
use super::simplify::{remove_dead_blocks, CfgSimplifier};
@@ -43,8 +46,15 @@ impl<'tcx> MirPass<'tcx> for Inline {
return enabled;
}
- // rust-lang/rust#101004: reverted to old inlining decision logic
- sess.mir_opt_level() >= 3
+ match sess.mir_opt_level() {
+ 0 | 1 => false,
+ 2 => {
+ (sess.opts.optimize == OptLevel::Default
+ || sess.opts.optimize == OptLevel::Aggressive)
+ && sess.opts.incremental == None
+ }
+ _ => true,
+ }
}
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -85,7 +95,7 @@ fn inline<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) -> bool {
history: Vec::new(),
changed: false,
};
- let blocks = BasicBlock::new(0)..body.basic_blocks().next_index();
+ let blocks = BasicBlock::new(0)..body.basic_blocks.next_index();
this.process_blocks(body, blocks);
this.changed
}
@@ -95,8 +105,12 @@ struct Inliner<'tcx> {
param_env: ParamEnv<'tcx>,
/// Caller codegen attributes.
codegen_fn_attrs: &'tcx CodegenFnAttrs,
- /// Stack of inlined Instances.
- history: Vec<ty::Instance<'tcx>>,
+ /// Stack of inlined instances.
+ /// We only check the `DefId` and not the substs because we want to
+ /// avoid inlining cases of polymorphic recursion.
+ /// The number of `DefId`s is finite, so checking history is enough
+ /// to ensure that we do not loop endlessly while inlining.
+ history: Vec<DefId>,
/// Indicates that the caller body has been modified.
changed: bool,
}
@@ -124,7 +138,7 @@ impl<'tcx> Inliner<'tcx> {
Ok(new_blocks) => {
debug!("inlined {}", callsite.callee);
self.changed = true;
- self.history.push(callsite.callee);
+ self.history.push(callsite.callee.def_id());
self.process_blocks(caller_body, new_blocks);
self.history.pop();
}
@@ -203,9 +217,9 @@ impl<'tcx> Inliner<'tcx> {
}
}
- let old_blocks = caller_body.basic_blocks().next_index();
+ let old_blocks = caller_body.basic_blocks.next_index();
self.inline_call(caller_body, &callsite, callee_body);
- let new_blocks = old_blocks..caller_body.basic_blocks().next_index();
+ let new_blocks = old_blocks..caller_body.basic_blocks.next_index();
Ok(new_blocks)
}
@@ -300,7 +314,7 @@ impl<'tcx> Inliner<'tcx> {
return None;
}
- if self.history.contains(&callee) {
+ if self.history.contains(&callee.def_id()) {
return None;
}
@@ -395,124 +409,66 @@ impl<'tcx> Inliner<'tcx> {
// Give a bonus functions with a small number of blocks,
// We normally have two or three blocks for even
// very small functions.
- if callee_body.basic_blocks().len() <= 3 {
+ if callee_body.basic_blocks.len() <= 3 {
threshold += threshold / 4;
}
debug!(" final inline threshold = {}", threshold);
// FIXME: Give a bonus to functions with only a single caller
- let mut first_block = true;
- let mut cost = 0;
+ let diverges = matches!(
+ callee_body.basic_blocks[START_BLOCK].terminator().kind,
+ TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. }
+ );
+ if diverges && !matches!(callee_attrs.inline, InlineAttr::Always) {
+ return Err("callee diverges unconditionally");
+ }
- // Traverse the MIR manually so we can account for the effects of
- // inlining on the CFG.
+ let mut checker = CostChecker {
+ tcx: self.tcx,
+ param_env: self.param_env,
+ instance: callsite.callee,
+ callee_body,
+ cost: 0,
+ validation: Ok(()),
+ };
+
+ // Traverse the MIR manually so we can account for the effects of inlining on the CFG.
let mut work_list = vec![START_BLOCK];
- let mut visited = BitSet::new_empty(callee_body.basic_blocks().len());
+ let mut visited = BitSet::new_empty(callee_body.basic_blocks.len());
while let Some(bb) = work_list.pop() {
if !visited.insert(bb.index()) {
continue;
}
- let blk = &callee_body.basic_blocks()[bb];
- for stmt in &blk.statements {
- // Don't count StorageLive/StorageDead in the inlining cost.
- match stmt.kind {
- StatementKind::StorageLive(_)
- | StatementKind::StorageDead(_)
- | StatementKind::Deinit(_)
- | StatementKind::Nop => {}
- _ => cost += INSTR_COST,
- }
- }
- let term = blk.terminator();
- let mut is_drop = false;
- match term.kind {
- TerminatorKind::Drop { ref place, target, unwind }
- | TerminatorKind::DropAndReplace { ref place, target, unwind, .. } => {
- is_drop = true;
- work_list.push(target);
- // If the place doesn't actually need dropping, treat it like
- // a regular goto.
- let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty);
- if ty.needs_drop(tcx, self.param_env) {
- cost += CALL_PENALTY;
- if let Some(unwind) = unwind {
- cost += LANDINGPAD_PENALTY;
- work_list.push(unwind);
- }
- } else {
- cost += INSTR_COST;
- }
- }
+ let blk = &callee_body.basic_blocks[bb];
+ checker.visit_basic_block_data(bb, blk);
- TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. }
- if first_block =>
- {
- // If the function always diverges, don't inline
- // unless the cost is zero
- threshold = 0;
- }
-
- TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => {
- if let ty::FnDef(def_id, _) =
- *callsite.callee.subst_mir(self.tcx, &f.literal.ty()).kind()
- {
- // Don't give intrinsics the extra penalty for calls
- if tcx.is_intrinsic(def_id) {
- cost += INSTR_COST;
- } else {
- cost += CALL_PENALTY;
- }
- } else {
- cost += CALL_PENALTY;
- }
- if cleanup.is_some() {
- cost += LANDINGPAD_PENALTY;
- }
- }
- TerminatorKind::Assert { cleanup, .. } => {
- cost += CALL_PENALTY;
-
- if cleanup.is_some() {
- cost += LANDINGPAD_PENALTY;
- }
- }
- TerminatorKind::Resume => cost += RESUME_PENALTY,
- TerminatorKind::InlineAsm { cleanup, .. } => {
- cost += INSTR_COST;
+ let term = blk.terminator();
+ if let TerminatorKind::Drop { ref place, target, unwind }
+ | TerminatorKind::DropAndReplace { ref place, target, unwind, .. } = term.kind
+ {
+ work_list.push(target);
- if cleanup.is_some() {
- cost += LANDINGPAD_PENALTY;
+ // If the place doesn't actually need dropping, treat it like a regular goto.
+ let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty);
+ if ty.needs_drop(tcx, self.param_env) && let Some(unwind) = unwind {
+ work_list.push(unwind);
}
- }
- _ => cost += INSTR_COST,
- }
-
- if !is_drop {
- for succ in term.successors() {
- work_list.push(succ);
- }
+ } else {
+ work_list.extend(term.successors())
}
-
- first_block = false;
}
// Count up the cost of local variables and temps, if we know the size
// use that, otherwise we use a moderately-large dummy cost.
-
- let ptr_size = tcx.data_layout.pointer_size.bytes();
-
for v in callee_body.vars_and_temps_iter() {
- let ty = callsite.callee.subst_mir(self.tcx, &callee_body.local_decls[v].ty);
- // Cost of the var is the size in machine-words, if we know
- // it.
- if let Some(size) = type_size_of(tcx, self.param_env, ty) {
- cost += ((size + ptr_size - 1) / ptr_size) as usize;
- } else {
- cost += UNKNOWN_SIZE_COST;
- }
+ checker.visit_local_decl(v, &callee_body.local_decls[v]);
}
+ // Abort if type validation found anything fishy.
+ checker.validation?;
+
+ let cost = checker.cost;
if let InlineAttr::Always = callee_attrs.inline {
debug!("INLINING {:?} because inline(always) [cost={}]", callsite, cost);
Ok(())
@@ -585,7 +541,7 @@ impl<'tcx> Inliner<'tcx> {
args: &args,
new_locals: Local::new(caller_body.local_decls.len())..,
new_scopes: SourceScope::new(caller_body.source_scopes.len())..,
- new_blocks: BasicBlock::new(caller_body.basic_blocks().len())..,
+ new_blocks: BasicBlock::new(caller_body.basic_blocks.len())..,
destination: dest,
callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(),
callsite,
@@ -603,7 +559,9 @@ impl<'tcx> Inliner<'tcx> {
// If there are any locals without storage markers, give them storage only for the
// duration of the call.
for local in callee_body.vars_and_temps_iter() {
- if integrator.always_live_locals.contains(local) {
+ if !callee_body.local_decls[local].internal
+ && integrator.always_live_locals.contains(local)
+ {
let new_local = integrator.map_local(local);
caller_body[callsite.block].statements.push(Statement {
source_info: callsite.source_info,
@@ -616,7 +574,9 @@ impl<'tcx> Inliner<'tcx> {
// the slice once.
let mut n = 0;
for local in callee_body.vars_and_temps_iter().rev() {
- if integrator.always_live_locals.contains(local) {
+ if !callee_body.local_decls[local].internal
+ && integrator.always_live_locals.contains(local)
+ {
let new_local = integrator.map_local(local);
caller_body[block].statements.push(Statement {
source_info: callsite.source_info,
@@ -644,11 +604,11 @@ impl<'tcx> Inliner<'tcx> {
// `required_consts`, here we may not only have `ConstKind::Unevaluated`
// because we are calling `subst_and_normalize_erasing_regions`.
caller_body.required_consts.extend(
- callee_body.required_consts.iter().copied().filter(|&ct| {
- match ct.literal.const_for_ty() {
- Some(ct) => matches!(ct.kind(), ConstKind::Unevaluated(_)),
- None => true,
+ callee_body.required_consts.iter().copied().filter(|&ct| match ct.literal {
+ ConstantKind::Ty(_) => {
+ bug!("should never encounter ty::Unevaluated in `required_consts`")
}
+ ConstantKind::Val(..) | ConstantKind::Unevaluated(..) => true,
}),
);
}
@@ -782,6 +742,193 @@ fn type_size_of<'tcx>(
tcx.layout_of(param_env.and(ty)).ok().map(|layout| layout.size.bytes())
}
+/// Verify that the callee body is compatible with the caller.
+///
+/// This visitor mostly computes the inlining cost,
+/// but also needs to verify that types match because of normalization failure.
+struct CostChecker<'b, 'tcx> {
+ tcx: TyCtxt<'tcx>,
+ param_env: ParamEnv<'tcx>,
+ cost: usize,
+ callee_body: &'b Body<'tcx>,
+ instance: ty::Instance<'tcx>,
+ validation: Result<(), &'static str>,
+}
+
+impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> {
+ fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
+ // Don't count StorageLive/StorageDead in the inlining cost.
+ match statement.kind {
+ StatementKind::StorageLive(_)
+ | StatementKind::StorageDead(_)
+ | StatementKind::Deinit(_)
+ | StatementKind::Nop => {}
+ _ => self.cost += INSTR_COST,
+ }
+
+ self.super_statement(statement, location);
+ }
+
+ fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) {
+ let tcx = self.tcx;
+ match terminator.kind {
+ TerminatorKind::Drop { ref place, unwind, .. }
+ | TerminatorKind::DropAndReplace { ref place, unwind, .. } => {
+ // If the place doesn't actually need dropping, treat it like a regular goto.
+ let ty = self.instance.subst_mir(tcx, &place.ty(self.callee_body, tcx).ty);
+ if ty.needs_drop(tcx, self.param_env) {
+ self.cost += CALL_PENALTY;
+ if unwind.is_some() {
+ self.cost += LANDINGPAD_PENALTY;
+ }
+ } else {
+ self.cost += INSTR_COST;
+ }
+ }
+ TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => {
+ let fn_ty = self.instance.subst_mir(tcx, &f.literal.ty());
+ self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) {
+ // Don't give intrinsics the extra penalty for calls
+ INSTR_COST
+ } else {
+ CALL_PENALTY
+ };
+ if cleanup.is_some() {
+ self.cost += LANDINGPAD_PENALTY;
+ }
+ }
+ TerminatorKind::Assert { cleanup, .. } => {
+ self.cost += CALL_PENALTY;
+ if cleanup.is_some() {
+ self.cost += LANDINGPAD_PENALTY;
+ }
+ }
+ TerminatorKind::Resume => self.cost += RESUME_PENALTY,
+ TerminatorKind::InlineAsm { cleanup, .. } => {
+ self.cost += INSTR_COST;
+ if cleanup.is_some() {
+ self.cost += LANDINGPAD_PENALTY;
+ }
+ }
+ _ => self.cost += INSTR_COST,
+ }
+
+ self.super_terminator(terminator, location);
+ }
+
+ /// Count up the cost of local variables and temps, if we know the size
+ /// use that, otherwise we use a moderately-large dummy cost.
+ fn visit_local_decl(&mut self, local: Local, local_decl: &LocalDecl<'tcx>) {
+ let tcx = self.tcx;
+ let ptr_size = tcx.data_layout.pointer_size.bytes();
+
+ let ty = self.instance.subst_mir(tcx, &local_decl.ty);
+ // Cost of the var is the size in machine-words, if we know
+ // it.
+ if let Some(size) = type_size_of(tcx, self.param_env, ty) {
+ self.cost += ((size + ptr_size - 1) / ptr_size) as usize;
+ } else {
+ self.cost += UNKNOWN_SIZE_COST;
+ }
+
+ self.super_local_decl(local, local_decl)
+ }
+
+ /// This method duplicates code from MIR validation in an attempt to detect type mismatches due
+ /// to normalization failure.
+ fn visit_projection_elem(
+ &mut self,
+ local: Local,
+ proj_base: &[PlaceElem<'tcx>],
+ elem: PlaceElem<'tcx>,
+ context: PlaceContext,
+ location: Location,
+ ) {
+ if let ProjectionElem::Field(f, ty) = elem {
+ let parent = Place { local, projection: self.tcx.intern_place_elems(proj_base) };
+ let parent_ty = parent.ty(&self.callee_body.local_decls, self.tcx);
+ let check_equal = |this: &mut Self, f_ty| {
+ if !equal_up_to_regions(this.tcx, this.param_env, ty, f_ty) {
+ trace!(?ty, ?f_ty);
+ this.validation = Err("failed to normalize projection type");
+ return;
+ }
+ };
+
+ let kind = match parent_ty.ty.kind() {
+ &ty::Opaque(def_id, substs) => {
+ self.tcx.bound_type_of(def_id).subst(self.tcx, substs).kind()
+ }
+ kind => kind,
+ };
+
+ match kind {
+ ty::Tuple(fields) => {
+ let Some(f_ty) = fields.get(f.as_usize()) else {
+ self.validation = Err("malformed MIR");
+ return;
+ };
+ check_equal(self, *f_ty);
+ }
+ ty::Adt(adt_def, substs) => {
+ let var = parent_ty.variant_index.unwrap_or(VariantIdx::from_u32(0));
+ let Some(field) = adt_def.variant(var).fields.get(f.as_usize()) else {
+ self.validation = Err("malformed MIR");
+ return;
+ };
+ check_equal(self, field.ty(self.tcx, substs));
+ }
+ ty::Closure(_, substs) => {
+ let substs = substs.as_closure();
+ let Some(f_ty) = substs.upvar_tys().nth(f.as_usize()) else {
+ self.validation = Err("malformed MIR");
+ return;
+ };
+ check_equal(self, f_ty);
+ }
+ &ty::Generator(def_id, substs, _) => {
+ let f_ty = if let Some(var) = parent_ty.variant_index {
+ let gen_body = if def_id == self.callee_body.source.def_id() {
+ self.callee_body
+ } else {
+ self.tcx.optimized_mir(def_id)
+ };
+
+ let Some(layout) = gen_body.generator_layout() else {
+ self.validation = Err("malformed MIR");
+ return;
+ };
+
+ let Some(&local) = layout.variant_fields[var].get(f) else {
+ self.validation = Err("malformed MIR");
+ return;
+ };
+
+ let Some(&f_ty) = layout.field_tys.get(local) else {
+ self.validation = Err("malformed MIR");
+ return;
+ };
+
+ f_ty
+ } else {
+ let Some(f_ty) = substs.as_generator().prefix_tys().nth(f.index()) else {
+ self.validation = Err("malformed MIR");
+ return;
+ };
+
+ f_ty
+ };
+
+ check_equal(self, f_ty);
+ }
+ _ => self.validation = Err("malformed MIR"),
+ }
+ }
+
+ self.super_projection_elem(local, proj_base, elem, context, location);
+ }
+}
+
/**
* Integrator.
*
diff --git a/compiler/rustc_mir_transform/src/inline/cycle.rs b/compiler/rustc_mir_transform/src/inline/cycle.rs
index 7810218fd..b027f9492 100644
--- a/compiler/rustc_mir_transform/src/inline/cycle.rs
+++ b/compiler/rustc_mir_transform/src/inline/cycle.rs
@@ -153,7 +153,7 @@ pub(crate) fn mir_inliner_callees<'tcx>(
_ => tcx.instance_mir(instance),
};
let mut calls = FxIndexSet::default();
- for bb_data in body.basic_blocks() {
+ for bb_data in body.basic_blocks.iter() {
let terminator = bb_data.terminator();
if let TerminatorKind::Call { func, .. } = &terminator.kind {
let ty = func.ty(&body.local_decls, tcx);
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs
index d968a4885..e6fc85595 100644
--- a/compiler/rustc_mir_transform/src/lib.rs
+++ b/compiler/rustc_mir_transform/src/lib.rs
@@ -1,7 +1,7 @@
#![allow(rustc::potential_query_instability)]
#![feature(box_patterns)]
#![feature(let_chains)]
-#![feature(let_else)]
+#![cfg_attr(bootstrap, feature(let_else))]
#![feature(map_try_insert)]
#![feature(min_specialization)]
#![feature(never_type)]
@@ -10,6 +10,7 @@
#![feature(trusted_step)]
#![feature(try_blocks)]
#![feature(yeet_expr)]
+#![feature(if_let_guard)]
#![recursion_limit = "256"]
#[macro_use]
@@ -26,10 +27,14 @@ use rustc_hir::def_id::{DefId, LocalDefId};
use rustc_hir::intravisit::{self, Visitor};
use rustc_index::vec::IndexVec;
use rustc_middle::mir::visit::Visitor as _;
-use rustc_middle::mir::{traversal, Body, ConstQualifs, MirPass, MirPhase, Promoted};
+use rustc_middle::mir::{
+ traversal, AnalysisPhase, Body, ConstQualifs, Constant, LocalDecl, MirPass, MirPhase, Operand,
+ Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, Statement, StatementKind,
+ TerminatorKind,
+};
use rustc_middle::ty::query::Providers;
use rustc_middle::ty::{self, TyCtxt, TypeVisitable};
-use rustc_span::{Span, Symbol};
+use rustc_span::sym;
#[macro_use]
mod pass_manager;
@@ -139,6 +144,64 @@ pub fn provide(providers: &mut Providers) {
};
}
+fn remap_mir_for_const_eval_select<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ mut body: Body<'tcx>,
+ context: hir::Constness,
+) -> Body<'tcx> {
+ for bb in body.basic_blocks.as_mut().iter_mut() {
+ let terminator = bb.terminator.as_mut().expect("invalid terminator");
+ match terminator.kind {
+ TerminatorKind::Call {
+ func: Operand::Constant(box Constant { ref literal, .. }),
+ ref mut args,
+ destination,
+ target,
+ cleanup,
+ fn_span,
+ ..
+ } if let ty::FnDef(def_id, _) = *literal.ty().kind()
+ && tcx.item_name(def_id) == sym::const_eval_select
+ && tcx.is_intrinsic(def_id) =>
+ {
+ let [tupled_args, called_in_const, called_at_rt]: [_; 3] = std::mem::take(args).try_into().unwrap();
+ let ty = tupled_args.ty(&body.local_decls, tcx);
+ let fields = ty.tuple_fields();
+ let num_args = fields.len();
+ let func = if context == hir::Constness::Const { called_in_const } else { called_at_rt };
+ let (method, place): (fn(Place<'tcx>) -> Operand<'tcx>, Place<'tcx>) = match tupled_args {
+ Operand::Constant(_) => {
+ // there is no good way of extracting a tuple arg from a constant (const generic stuff)
+ // so we just create a temporary and deconstruct that.
+ let local = body.local_decls.push(LocalDecl::new(ty, fn_span));
+ bb.statements.push(Statement {
+ source_info: SourceInfo::outermost(fn_span),
+ kind: StatementKind::Assign(Box::new((local.into(), Rvalue::Use(tupled_args.clone())))),
+ });
+ (Operand::Move, local.into())
+ }
+ Operand::Move(place) => (Operand::Move, place),
+ Operand::Copy(place) => (Operand::Copy, place),
+ };
+ let place_elems = place.projection;
+ let arguments = (0..num_args).map(|x| {
+ let mut place_elems = place_elems.to_vec();
+ place_elems.push(ProjectionElem::Field(x.into(), fields[x]));
+ let projection = tcx.intern_place_elems(&place_elems);
+ let place = Place {
+ local: place.local,
+ projection,
+ };
+ method(place)
+ }).collect();
+ terminator.kind = TerminatorKind::Call { func, args: arguments, destination, target, cleanup, from_hir_call: false, fn_span };
+ }
+ _ => {}
+ }
+ }
+ body
+}
+
fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool {
let def_id = def_id.expect_local();
tcx.mir_keys(()).contains(&def_id)
@@ -159,14 +222,7 @@ fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> {
set: &'a mut FxIndexSet<LocalDefId>,
}
impl<'tcx> Visitor<'tcx> for GatherCtors<'_, 'tcx> {
- fn visit_variant_data(
- &mut self,
- v: &'tcx hir::VariantData<'tcx>,
- _: Symbol,
- _: &'tcx hir::Generics<'tcx>,
- _: hir::HirId,
- _: Span,
- ) {
+ fn visit_variant_data(&mut self, v: &'tcx hir::VariantData<'tcx>) {
if let hir::VariantData::Tuple(_, hir_id) = *v {
self.set.insert(self.tcx.hir().local_def_id(hir_id));
}
@@ -208,6 +264,8 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) ->
}
/// Make MIR ready for const evaluation. This is run on all MIR, not just on consts!
+/// FIXME(oli-obk): it's unclear whether we still need this phase (and its corresponding query).
+/// We used to have this for pre-miri MIR based const eval.
fn mir_const<'tcx>(
tcx: TyCtxt<'tcx>,
def: ty::WithOptConstParam<LocalDefId>,
@@ -243,7 +301,6 @@ fn mir_const<'tcx>(
// What we need to do constant evaluation.
&simplify::SimplifyCfg::new("initial"),
&rustc_peek::SanityCheck, // Just a lint
- &marker::PhaseChange(MirPhase::Const),
],
);
tcx.alloc_steal_mir(body)
@@ -330,7 +387,9 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -
.body_const_context(def.did)
.expect("mir_for_ctfe should not be used for runtime functions");
- let mut body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone();
+ let body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone();
+
+ let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::Const);
match context {
// Do not const prop functions, either they get executed at runtime or exported to metadata,
@@ -349,7 +408,10 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -
pm::run_passes(
tcx,
&mut body,
- &[&const_prop::ConstProp, &marker::PhaseChange(MirPhase::Optimized)],
+ &[
+ &const_prop::ConstProp,
+ &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)),
+ ],
);
}
}
@@ -389,38 +451,61 @@ fn mir_drops_elaborated_and_const_checked<'tcx>(
body.tainted_by_errors = Some(error_reported);
}
- // IMPORTANT
- pm::run_passes(tcx, &mut body, &[&remove_false_edges::RemoveFalseEdges]);
+ run_analysis_to_runtime_passes(tcx, &mut body);
+
+ tcx.alloc_steal_mir(body)
+}
+
+fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+ assert!(body.phase == MirPhase::Analysis(AnalysisPhase::Initial));
+ let did = body.source.def_id();
+
+ debug!("analysis_mir_cleanup({:?})", did);
+ run_analysis_cleanup_passes(tcx, body);
+ assert!(body.phase == MirPhase::Analysis(AnalysisPhase::PostCleanup));
// Do a little drop elaboration before const-checking if `const_precise_live_drops` is enabled.
if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, &body)) {
pm::run_passes(
tcx,
- &mut body,
+ body,
&[
- &simplify::SimplifyCfg::new("remove-false-edges"),
&remove_uninit_drops::RemoveUninitDrops,
+ &simplify::SimplifyCfg::new("remove-false-edges"),
],
);
check_consts::post_drop_elaboration::check_live_drops(tcx, &body); // FIXME: make this a MIR lint
}
- run_post_borrowck_cleanup_passes(tcx, &mut body);
- assert!(body.phase == MirPhase::Deaggregated);
- tcx.alloc_steal_mir(body)
+ debug!("runtime_mir_lowering({:?})", did);
+ run_runtime_lowering_passes(tcx, body);
+ assert!(body.phase == MirPhase::Runtime(RuntimePhase::Initial));
+
+ debug!("runtime_mir_cleanup({:?})", did);
+ run_runtime_cleanup_passes(tcx, body);
+ assert!(body.phase == MirPhase::Runtime(RuntimePhase::PostCleanup));
}
-/// After this series of passes, no lifetime analysis based on borrowing can be done.
-fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
- debug!("post_borrowck_cleanup({:?})", body.source.def_id());
+// FIXME(JakobDegen): Can we make these lists of passes consts?
- let post_borrowck_cleanup: &[&dyn MirPass<'tcx>] = &[
- // Remove all things only needed by analysis
+/// After this series of passes, no lifetime analysis based on borrowing can be done.
+fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+ let passes: &[&dyn MirPass<'tcx>] = &[
+ &remove_false_edges::RemoveFalseEdges,
&simplify_branches::SimplifyConstCondition::new("initial"),
&remove_noop_landing_pads::RemoveNoopLandingPads,
&cleanup_post_borrowck::CleanupNonCodegenStatements,
&simplify::SimplifyCfg::new("early-opt"),
&deref_separator::Derefer,
+ &marker::PhaseChange(MirPhase::Analysis(AnalysisPhase::PostCleanup)),
+ ];
+
+ pm::run_passes(tcx, body, passes);
+}
+
+/// Returns the sequence of passes that lowers analysis to runtime MIR.
+fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+ let passes: &[&dyn MirPass<'tcx>] = &[
// These next passes must be executed together
&add_call_guards::CriticalCallEdges,
&elaborate_drops::ElaborateDrops,
@@ -434,16 +519,27 @@ fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tc
// `AddRetag` needs to run after `ElaborateDrops`. Otherwise it should run fairly late,
// but before optimizations begin.
&elaborate_box_derefs::ElaborateBoxDerefs,
+ &generator::StateTransform,
&add_retag::AddRetag,
- &lower_intrinsics::LowerIntrinsics,
- &simplify::SimplifyCfg::new("elaborate-drops"),
- // `Deaggregator` is conceptually part of MIR building, some backends rely on it happening
- // and it can help optimizations.
+ // Deaggregator is necessary for const prop. We may want to consider implementing
+ // CTFE support for aggregates.
&deaggregator::Deaggregator,
&Lint(const_prop_lint::ConstProp),
+ &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Initial)),
+ ];
+ pm::run_passes_no_validate(tcx, body, passes);
+}
+
+/// Returns the sequence of passes that do the initial cleanup of runtime MIR.
+fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
+ let passes: &[&dyn MirPass<'tcx>] = &[
+ &elaborate_box_derefs::ElaborateBoxDerefs,
+ &lower_intrinsics::LowerIntrinsics,
+ &simplify::SimplifyCfg::new("elaborate-drops"),
+ &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::PostCleanup)),
];
- pm::run_passes(tcx, body, post_borrowck_cleanup);
+ pm::run_passes(tcx, body, passes);
}
fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -451,9 +547,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
WithMinOptLevel(1, x)
}
- // Lowering generator control-flow and variables has to happen before we do anything else
- // to them. We run some optimizations before that, because they may be harder to do on the state
- // machine than on MIR with async primitives.
+ // The main optimizations that we do on MIR.
pm::run_passes(
tcx,
body,
@@ -465,17 +559,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
&uninhabited_enum_branching::UninhabitedEnumBranching,
&o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")),
&inline::Inline,
- &generator::StateTransform,
- ],
- );
-
- assert!(body.phase == MirPhase::GeneratorsLowered);
-
- // The main optimizations that we do on MIR.
- pm::run_passes(
- tcx,
- body,
- &[
&remove_storage_markers::RemoveStorageMarkers,
&remove_zsts::RemoveZsts,
&const_goto::ConstGoto,
@@ -507,7 +590,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
&deduplicate_blocks::DeduplicateBlocks,
// Some cleanup necessary at least for LLVM and potentially other codegen backends.
&add_call_guards::CriticalCallEdges,
- &marker::PhaseChange(MirPhase::Optimized),
+ &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)),
// Dump the end result for testing and debugging purposes.
&dump_mir::Marker("PreCodegen"),
],
@@ -539,8 +622,9 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> {
Some(other) => panic!("do not use `optimized_mir` for constants: {:?}", other),
}
debug!("about to call mir_drops_elaborated...");
- let mut body =
+ let body =
tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).steal();
+ let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::NotConst);
debug!("body: {:#?}", body);
run_optimization_passes(tcx, &mut body);
@@ -566,7 +650,7 @@ fn promoted_mir<'tcx>(
if let Some(error_reported) = tainted_by_errors {
body.tainted_by_errors = Some(error_reported);
}
- run_post_borrowck_cleanup_passes(tcx, body);
+ run_analysis_to_runtime_passes(tcx, body);
}
debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR");
diff --git a/compiler/rustc_mir_transform/src/lower_intrinsics.rs b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
index b7ba61651..9892580e6 100644
--- a/compiler/rustc_mir_transform/src/lower_intrinsics.rs
+++ b/compiler/rustc_mir_transform/src/lower_intrinsics.rs
@@ -46,12 +46,31 @@ impl<'tcx> MirPass<'tcx> for LowerIntrinsics {
let mut args = args.drain(..);
block.statements.push(Statement {
source_info: terminator.source_info,
- kind: StatementKind::CopyNonOverlapping(Box::new(
- rustc_middle::mir::CopyNonOverlapping {
- src: args.next().unwrap(),
- dst: args.next().unwrap(),
- count: args.next().unwrap(),
- },
+ kind: StatementKind::Intrinsic(Box::new(
+ NonDivergingIntrinsic::CopyNonOverlapping(
+ rustc_middle::mir::CopyNonOverlapping {
+ src: args.next().unwrap(),
+ dst: args.next().unwrap(),
+ count: args.next().unwrap(),
+ },
+ ),
+ )),
+ });
+ assert_eq!(
+ args.next(),
+ None,
+ "Extra argument for copy_non_overlapping intrinsic"
+ );
+ drop(args);
+ terminator.kind = TerminatorKind::Goto { target };
+ }
+ sym::assume => {
+ let target = target.unwrap();
+ let mut args = args.drain(..);
+ block.statements.push(Statement {
+ source_info: terminator.source_info,
+ kind: StatementKind::Intrinsic(Box::new(
+ NonDivergingIntrinsic::Assume(args.next().unwrap()),
)),
});
assert_eq!(
diff --git a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
index 22b6dead9..3957cd92c 100644
--- a/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
+++ b/compiler/rustc_mir_transform/src/multiple_return_terminators.rs
@@ -15,7 +15,7 @@ impl<'tcx> MirPass<'tcx> for MultipleReturnTerminators {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
// find basic blocks with no statement and a return terminator
- let mut bbs_simple_returns = BitSet::new_empty(body.basic_blocks().len());
+ let mut bbs_simple_returns = BitSet::new_empty(body.basic_blocks.len());
let def_id = body.source.def_id();
let bbs = body.basic_blocks_mut();
for idx in bbs.indices() {
diff --git a/compiler/rustc_mir_transform/src/normalize_array_len.rs b/compiler/rustc_mir_transform/src/normalize_array_len.rs
index c0217a105..a159e6171 100644
--- a/compiler/rustc_mir_transform/src/normalize_array_len.rs
+++ b/compiler/rustc_mir_transform/src/normalize_array_len.rs
@@ -21,10 +21,10 @@ impl<'tcx> MirPass<'tcx> for NormalizeArrayLen {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
// early returns for edge cases of highly unrolled functions
- if body.basic_blocks().len() > MAX_NUM_BLOCKS {
+ if body.basic_blocks.len() > MAX_NUM_BLOCKS {
return;
}
- if body.local_decls().len() > MAX_NUM_LOCALS {
+ if body.local_decls.len() > MAX_NUM_LOCALS {
return;
}
normalize_array_len_calls(tcx, body)
diff --git a/compiler/rustc_mir_transform/src/nrvo.rs b/compiler/rustc_mir_transform/src/nrvo.rs
index bb063915f..4291e81c7 100644
--- a/compiler/rustc_mir_transform/src/nrvo.rs
+++ b/compiler/rustc_mir_transform/src/nrvo.rs
@@ -53,10 +53,10 @@ impl<'tcx> MirPass<'tcx> for RenameReturnPlace {
def_id, returned_local
);
- RenameToReturnPlace { tcx, to_rename: returned_local }.visit_body(body);
+ RenameToReturnPlace { tcx, to_rename: returned_local }.visit_body_preserves_cfg(body);
// Clean up the `NOP`s we inserted for statements made useless by our renaming.
- for block_data in body.basic_blocks_mut() {
+ for block_data in body.basic_blocks.as_mut_preserves_cfg() {
block_data.statements.retain(|stmt| stmt.kind != mir::StatementKind::Nop);
}
@@ -89,7 +89,7 @@ fn local_eligible_for_nrvo(body: &mut mir::Body<'_>) -> Option<Local> {
}
let mut copied_to_return_place = None;
- for block in body.basic_blocks().indices() {
+ for block in body.basic_blocks.indices() {
// Look for blocks with a `Return` terminator.
if !matches!(body[block].terminator().kind, mir::TerminatorKind::Return) {
continue;
@@ -122,7 +122,7 @@ fn find_local_assigned_to_return_place(
body: &mut mir::Body<'_>,
) -> Option<Local> {
let mut block = start;
- let mut seen = HybridBitSet::new_empty(body.basic_blocks().len());
+ let mut seen = HybridBitSet::new_empty(body.basic_blocks.len());
// Iterate as long as `block` has exactly one predecessor that we have not yet visited.
while seen.insert(block) {
diff --git a/compiler/rustc_mir_transform/src/pass_manager.rs b/compiler/rustc_mir_transform/src/pass_manager.rs
index e27d4ab16..67dae7146 100644
--- a/compiler/rustc_mir_transform/src/pass_manager.rs
+++ b/compiler/rustc_mir_transform/src/pass_manager.rs
@@ -1,6 +1,6 @@
use std::borrow::Cow;
-use rustc_middle::mir::{self, Body, MirPhase};
+use rustc_middle::mir::{self, Body, MirPhase, RuntimePhase};
use rustc_middle::ty::TyCtxt;
use rustc_session::Session;
@@ -72,48 +72,62 @@ where
}
}
+/// Run the sequence of passes without validating the MIR after each pass. The MIR is still
+/// validated at the end.
+pub fn run_passes_no_validate<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ body: &mut Body<'tcx>,
+ passes: &[&dyn MirPass<'tcx>],
+) {
+ run_passes_inner(tcx, body, passes, false);
+}
+
pub fn run_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn MirPass<'tcx>]) {
+ run_passes_inner(tcx, body, passes, true);
+}
+
+fn run_passes_inner<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ body: &mut Body<'tcx>,
+ passes: &[&dyn MirPass<'tcx>],
+ validate_each: bool,
+) {
let start_phase = body.phase;
let mut cnt = 0;
- let validate = tcx.sess.opts.unstable_opts.validate_mir;
+ let validate = validate_each & tcx.sess.opts.unstable_opts.validate_mir;
let overridden_passes = &tcx.sess.opts.unstable_opts.mir_enable_passes;
trace!(?overridden_passes);
- if validate {
- validate_body(tcx, body, format!("start of phase transition from {:?}", start_phase));
- }
-
for pass in passes {
let name = pass.name();
- if let Some((_, polarity)) = overridden_passes.iter().rev().find(|(s, _)| s == &*name) {
- trace!(
- pass = %name,
- "{} as requested by flag",
- if *polarity { "Running" } else { "Not running" },
- );
- if !polarity {
- continue;
- }
- } else {
- if !pass.is_enabled(&tcx.sess) {
- continue;
- }
- }
- let dump_enabled = pass.is_mir_dump_enabled();
+ // Gather information about what we should be doing for this pass
+ let overridden =
+ overridden_passes.iter().rev().find(|(s, _)| s == &*name).map(|(_name, polarity)| {
+ trace!(
+ pass = %name,
+ "{} as requested by flag",
+ if *polarity { "Running" } else { "Not running" },
+ );
+ *polarity
+ });
+ let is_enabled = overridden.unwrap_or_else(|| pass.is_enabled(&tcx.sess));
+ let new_phase = pass.phase_change();
+ let dump_enabled = (is_enabled && pass.is_mir_dump_enabled()) || new_phase.is_some();
+ let validate = (validate && is_enabled)
+ || new_phase == Some(MirPhase::Runtime(RuntimePhase::Optimized));
if dump_enabled {
dump_mir(tcx, body, start_phase, &name, cnt, false);
}
-
- pass.run_pass(tcx, body);
-
+ if is_enabled {
+ pass.run_pass(tcx, body);
+ }
if dump_enabled {
dump_mir(tcx, body, start_phase, &name, cnt, true);
cnt += 1;
}
-
if let Some(new_phase) = pass.phase_change() {
if body.phase >= new_phase {
panic!("Invalid MIR phase transition from {:?} to {:?}", body.phase, new_phase);
@@ -121,15 +135,10 @@ pub fn run_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, passes: &[&dyn
body.phase = new_phase;
}
-
if validate {
- validate_body(tcx, body, format!("after pass {}", pass.name()));
+ validate_body(tcx, body, format!("after pass {}", name));
}
}
-
- if validate || body.phase == MirPhase::Optimized {
- validate_body(tcx, body, format!("end of phase transition to {:?}", body.phase));
- }
}
pub fn validate_body<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>, when: String) {
@@ -144,7 +153,7 @@ pub fn dump_mir<'tcx>(
cnt: usize,
is_after: bool,
) {
- let phase_index = phase as u32;
+ let phase_index = phase.phase_index();
mir::dump_mir(
tcx,
diff --git a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
index 5c441c5b1..f1bbf2ea7 100644
--- a/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
+++ b/compiler/rustc_mir_transform/src/remove_noop_landing_pads.rs
@@ -51,7 +51,7 @@ impl RemoveNoopLandingPads {
StatementKind::Assign { .. }
| StatementKind::SetDiscriminant { .. }
| StatementKind::Deinit(..)
- | StatementKind::CopyNonOverlapping(..)
+ | StatementKind::Intrinsic(..)
| StatementKind::Retag { .. } => {
return false;
}
@@ -94,7 +94,7 @@ impl RemoveNoopLandingPads {
let mut jumps_folded = 0;
let mut landing_pads_removed = 0;
- let mut nop_landing_pads = BitSet::new_empty(body.basic_blocks().len());
+ let mut nop_landing_pads = BitSet::new_empty(body.basic_blocks.len());
// This is a post-order traversal, so that if A post-dominates B
// then A will be visited before B.
diff --git a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
index 96b715402..78b6f714a 100644
--- a/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
+++ b/compiler/rustc_mir_transform/src/remove_uninit_drops.rs
@@ -35,7 +35,7 @@ impl<'tcx> MirPass<'tcx> for RemoveUninitDrops {
.into_results_cursor(body);
let mut to_remove = vec![];
- for (bb, block) in body.basic_blocks().iter_enumerated() {
+ for (bb, block) in body.basic_blocks.iter_enumerated() {
let terminator = block.terminator();
let (TerminatorKind::Drop { place, .. } | TerminatorKind::DropAndReplace { place, .. })
= &terminator.kind
diff --git a/compiler/rustc_mir_transform/src/required_consts.rs b/compiler/rustc_mir_transform/src/required_consts.rs
index 827ce0c02..cc75947d9 100644
--- a/compiler/rustc_mir_transform/src/required_consts.rs
+++ b/compiler/rustc_mir_transform/src/required_consts.rs
@@ -1,5 +1,5 @@
use rustc_middle::mir::visit::Visitor;
-use rustc_middle::mir::{Constant, Location};
+use rustc_middle::mir::{Constant, ConstantKind, Location};
use rustc_middle::ty::ConstKind;
pub struct RequiredConstsVisitor<'a, 'tcx> {
@@ -15,8 +15,13 @@ impl<'a, 'tcx> RequiredConstsVisitor<'a, 'tcx> {
impl<'tcx> Visitor<'tcx> for RequiredConstsVisitor<'_, 'tcx> {
fn visit_constant(&mut self, constant: &Constant<'tcx>, _: Location) {
let literal = constant.literal;
- if let Some(ct) = literal.const_for_ty() && let ConstKind::Unevaluated(_) = ct.kind() {
- self.required_consts.push(*constant);
+ match literal {
+ ConstantKind::Ty(c) => match c.kind() {
+ ConstKind::Param(_) => {}
+ _ => bug!("only ConstKind::Param should be encountered here, got {:#?}", c),
+ },
+ ConstantKind::Unevaluated(..) => self.required_consts.push(*constant),
+ ConstantKind::Val(..) => {}
}
}
}
diff --git a/compiler/rustc_mir_transform/src/reveal_all.rs b/compiler/rustc_mir_transform/src/reveal_all.rs
index 4919ad400..abe6cb285 100644
--- a/compiler/rustc_mir_transform/src/reveal_all.rs
+++ b/compiler/rustc_mir_transform/src/reveal_all.rs
@@ -19,7 +19,7 @@ impl<'tcx> MirPass<'tcx> for RevealAll {
}
let param_env = tcx.param_env_reveal_all_normalized(body.source.def_id());
- RevealAllVisitor { tcx, param_env }.visit_body(body);
+ RevealAllVisitor { tcx, param_env }.visit_body_preserves_cfg(body);
}
}
diff --git a/compiler/rustc_mir_transform/src/separate_const_switch.rs b/compiler/rustc_mir_transform/src/separate_const_switch.rs
index 925eb10a1..2f116aaa9 100644
--- a/compiler/rustc_mir_transform/src/separate_const_switch.rs
+++ b/compiler/rustc_mir_transform/src/separate_const_switch.rs
@@ -62,7 +62,7 @@ impl<'tcx> MirPass<'tcx> for SeparateConstSwitch {
pub fn separate_const_switch(body: &mut Body<'_>) -> usize {
let mut new_blocks: SmallVec<[(BasicBlock, BasicBlock); 6]> = SmallVec::new();
let predecessors = body.basic_blocks.predecessors();
- 'block_iter: for (block_id, block) in body.basic_blocks().iter_enumerated() {
+ 'block_iter: for (block_id, block) in body.basic_blocks.iter_enumerated() {
if let TerminatorKind::SwitchInt {
discr: Operand::Copy(switch_place) | Operand::Move(switch_place),
..
@@ -90,7 +90,7 @@ pub fn separate_const_switch(body: &mut Body<'_>) -> usize {
let mut predecessors_left = predecessors[block_id].len();
'predec_iter: for predecessor_id in predecessors[block_id].iter().copied() {
- let predecessor = &body.basic_blocks()[predecessor_id];
+ let predecessor = &body.basic_blocks[predecessor_id];
// First we make sure the predecessor jumps
// in a reasonable way
@@ -249,7 +249,7 @@ fn is_likely_const<'tcx>(mut tracked_place: Place<'tcx>, block: &BasicBlockData<
| StatementKind::AscribeUserType(_, _)
| StatementKind::Coverage(_)
| StatementKind::StorageDead(_)
- | StatementKind::CopyNonOverlapping(_)
+ | StatementKind::Intrinsic(_)
| StatementKind::Nop => {}
}
}
@@ -317,7 +317,7 @@ fn find_determining_place<'tcx>(
| StatementKind::Retag(_, _)
| StatementKind::AscribeUserType(_, _)
| StatementKind::Coverage(_)
- | StatementKind::CopyNonOverlapping(_)
+ | StatementKind::Intrinsic(_)
| StatementKind::Nop => {}
// If the discriminant is set, it is always set
diff --git a/compiler/rustc_mir_transform/src/shim.rs b/compiler/rustc_mir_transform/src/shim.rs
index 3620e94be..882136200 100644
--- a/compiler/rustc_mir_transform/src/shim.rs
+++ b/compiler/rustc_mir_transform/src/shim.rs
@@ -4,7 +4,7 @@ use rustc_hir::lang_items::LangItem;
use rustc_middle::mir::*;
use rustc_middle::ty::query::Providers;
use rustc_middle::ty::subst::{InternalSubsts, Subst};
-use rustc_middle::ty::{self, EarlyBinder, Ty, TyCtxt};
+use rustc_middle::ty::{self, EarlyBinder, GeneratorSubsts, Ty, TyCtxt};
use rustc_target::abi::VariantIdx;
use rustc_index::vec::{Idx, IndexVec};
@@ -17,8 +17,8 @@ use std::iter;
use crate::util::expand_aggregate;
use crate::{
- abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, marker, pass_manager as pm,
- remove_noop_landing_pads, simplify,
+ abort_unwinding_calls, add_call_guards, add_moves_for_packed_drops, deref_separator, marker,
+ pass_manager as pm, remove_noop_landing_pads, simplify,
};
use rustc_middle::mir::patch::MirPatch;
use rustc_mir_dataflow::elaborate_drops::{self, DropElaborator, DropFlagMode, DropStyle};
@@ -92,11 +92,12 @@ fn make_shim<'tcx>(tcx: TyCtxt<'tcx>, instance: ty::InstanceDef<'tcx>) -> Body<'
&mut result,
&[
&add_moves_for_packed_drops::AddMovesForPackedDrops,
+ &deref_separator::Derefer,
&remove_noop_landing_pads::RemoveNoopLandingPads,
&simplify::SimplifyCfg::new("make_shim"),
&add_call_guards::CriticalCallEdges,
&abort_unwinding_calls::AbortUnwindingCalls,
- &marker::PhaseChange(MirPhase::Const),
+ &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)),
],
);
@@ -322,6 +323,9 @@ fn build_clone_shim<'tcx>(tcx: TyCtxt<'tcx>, def_id: DefId, self_ty: Ty<'tcx>) -
builder.tuple_like_shim(dest, src, substs.as_closure().upvar_tys())
}
ty::Tuple(..) => builder.tuple_like_shim(dest, src, self_ty.tuple_fields()),
+ ty::Generator(gen_def_id, substs, hir::Movability::Movable) => {
+ builder.generator_shim(dest, src, *gen_def_id, substs.as_generator())
+ }
_ => bug!("clone shim for `{:?}` which is not `Copy` and is not an aggregate", self_ty),
};
@@ -387,7 +391,7 @@ impl<'tcx> CloneShimBuilder<'tcx> {
/// offset=0 will give you the index of the next BasicBlock,
/// offset=1 will give the index of the next-to-next block,
/// offset=-1 will give you the index of the last-created block
- fn block_index_offset(&mut self, offset: usize) -> BasicBlock {
+ fn block_index_offset(&self, offset: usize) -> BasicBlock {
BasicBlock::new(self.blocks.len() + offset)
}
@@ -460,49 +464,106 @@ impl<'tcx> CloneShimBuilder<'tcx> {
);
}
- fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I)
+ fn clone_fields<I>(
+ &mut self,
+ dest: Place<'tcx>,
+ src: Place<'tcx>,
+ target: BasicBlock,
+ mut unwind: BasicBlock,
+ tys: I,
+ ) -> BasicBlock
where
I: IntoIterator<Item = Ty<'tcx>>,
{
- let mut previous_field = None;
+ // For an iterator of length n, create 2*n + 1 blocks.
for (i, ity) in tys.into_iter().enumerate() {
+ // Each iteration creates two blocks, referred to here as block 2*i and block 2*i + 1.
+ //
+ // Block 2*i attempts to clone the field. If successful it branches to 2*i + 2 (the
+ // next clone block). If unsuccessful it branches to the previous unwind block, which
+ // is initially the `unwind` argument passed to this function.
+ //
+ // Block 2*i + 1 is the unwind block for this iteration. It drops the cloned value
+ // created by block 2*i. We store this block in `unwind` so that the next clone block
+ // will unwind to it if cloning fails.
+
let field = Field::new(i);
let src_field = self.tcx.mk_place_field(src, field, ity);
let dest_field = self.tcx.mk_place_field(dest, field, ity);
- // #(2i + 1) is the cleanup block for the previous clone operation
- let cleanup_block = self.block_index_offset(1);
- // #(2i + 2) is the next cloning block
- // (or the Return terminator if this is the last block)
+ let next_unwind = self.block_index_offset(1);
let next_block = self.block_index_offset(2);
+ self.make_clone_call(dest_field, src_field, ity, next_block, unwind);
+ self.block(
+ vec![],
+ TerminatorKind::Drop { place: dest_field, target: unwind, unwind: None },
+ true,
+ );
+ unwind = next_unwind;
+ }
+ // If all clones succeed then we end up here.
+ self.block(vec![], TerminatorKind::Goto { target }, false);
+ unwind
+ }
- // BB #(2i)
- // `dest.i = Clone::clone(&src.i);`
- // Goto #(2i + 2) if ok, #(2i + 1) if unwinding happens.
- self.make_clone_call(dest_field, src_field, ity, next_block, cleanup_block);
-
- // BB #(2i + 1) (cleanup)
- if let Some((previous_field, previous_cleanup)) = previous_field.take() {
- // Drop previous field and goto previous cleanup block.
- self.block(
- vec![],
- TerminatorKind::Drop {
- place: previous_field,
- target: previous_cleanup,
- unwind: None,
- },
- true,
- );
- } else {
- // Nothing to drop, just resume.
- self.block(vec![], TerminatorKind::Resume, true);
- }
+ fn tuple_like_shim<I>(&mut self, dest: Place<'tcx>, src: Place<'tcx>, tys: I)
+ where
+ I: IntoIterator<Item = Ty<'tcx>>,
+ {
+ self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
+ let unwind = self.block(vec![], TerminatorKind::Resume, true);
+ let target = self.block(vec![], TerminatorKind::Return, false);
- previous_field = Some((dest_field, cleanup_block));
- }
+ let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, tys);
+ }
- self.block(vec![], TerminatorKind::Return, false);
+ fn generator_shim(
+ &mut self,
+ dest: Place<'tcx>,
+ src: Place<'tcx>,
+ gen_def_id: DefId,
+ substs: GeneratorSubsts<'tcx>,
+ ) {
+ self.block(vec![], TerminatorKind::Goto { target: self.block_index_offset(3) }, false);
+ let unwind = self.block(vec![], TerminatorKind::Resume, true);
+ // This will get overwritten with a switch once we know the target blocks
+ let switch = self.block(vec![], TerminatorKind::Unreachable, false);
+ let unwind = self.clone_fields(dest, src, switch, unwind, substs.upvar_tys());
+ let target = self.block(vec![], TerminatorKind::Return, false);
+ let unreachable = self.block(vec![], TerminatorKind::Unreachable, false);
+ let mut cases = Vec::with_capacity(substs.state_tys(gen_def_id, self.tcx).count());
+ for (index, state_tys) in substs.state_tys(gen_def_id, self.tcx).enumerate() {
+ let variant_index = VariantIdx::new(index);
+ let dest = self.tcx.mk_place_downcast_unnamed(dest, variant_index);
+ let src = self.tcx.mk_place_downcast_unnamed(src, variant_index);
+ let clone_block = self.block_index_offset(1);
+ let start_block = self.block(
+ vec![self.make_statement(StatementKind::SetDiscriminant {
+ place: Box::new(Place::return_place()),
+ variant_index,
+ })],
+ TerminatorKind::Goto { target: clone_block },
+ false,
+ );
+ cases.push((index as u128, start_block));
+ let _final_cleanup_block = self.clone_fields(dest, src, target, unwind, state_tys);
+ }
+ let discr_ty = substs.discr_ty(self.tcx);
+ let temp = self.make_place(Mutability::Mut, discr_ty);
+ let rvalue = Rvalue::Discriminant(src);
+ let statement = self.make_statement(StatementKind::Assign(Box::new((temp, rvalue))));
+ match &mut self.blocks[switch] {
+ BasicBlockData { statements, terminator: Some(Terminator { kind, .. }), .. } => {
+ statements.push(statement);
+ *kind = TerminatorKind::SwitchInt {
+ discr: Operand::Move(temp),
+ switch_ty: discr_ty,
+ targets: SwitchTargets::new(cases.into_iter(), unreachable),
+ };
+ }
+ BasicBlockData { terminator: None, .. } => unreachable!(),
+ }
}
}
diff --git a/compiler/rustc_mir_transform/src/simplify.rs b/compiler/rustc_mir_transform/src/simplify.rs
index 180f4c7dc..57d372fda 100644
--- a/compiler/rustc_mir_transform/src/simplify.rs
+++ b/compiler/rustc_mir_transform/src/simplify.rs
@@ -74,7 +74,7 @@ pub struct CfgSimplifier<'a, 'tcx> {
impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
pub fn new(body: &'a mut Body<'tcx>) -> Self {
- let mut pred_count = IndexVec::from_elem(0u32, body.basic_blocks());
+ let mut pred_count = IndexVec::from_elem(0u32, &body.basic_blocks);
// we can't use mir.predecessors() here because that counts
// dead blocks, which we don't want to.
@@ -263,7 +263,7 @@ impl<'a, 'tcx> CfgSimplifier<'a, 'tcx> {
pub fn remove_dead_blocks<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
let reachable = traversal::reachable_as_bitset(body);
- let num_blocks = body.basic_blocks().len();
+ let num_blocks = body.basic_blocks.len();
if num_blocks == reachable.count() {
return;
}
@@ -412,7 +412,7 @@ pub fn simplify_locals<'tcx>(body: &mut Body<'tcx>, tcx: TyCtxt<'tcx>) {
if map.iter().any(Option::is_none) {
// Update references to all vars and tmps now
let mut updater = LocalUpdater { map, tcx };
- updater.visit_body(body);
+ updater.visit_body_preserves_cfg(body);
body.local_decls.shrink_to_fit();
}
@@ -499,7 +499,7 @@ impl UsedLocals {
impl<'tcx> Visitor<'tcx> for UsedLocals {
fn visit_statement(&mut self, statement: &Statement<'tcx>, location: Location) {
match statement.kind {
- StatementKind::CopyNonOverlapping(..)
+ StatementKind::Intrinsic(..)
| StatementKind::Retag(..)
| StatementKind::Coverage(..)
| StatementKind::FakeRead(..)
@@ -548,7 +548,7 @@ fn remove_unused_definitions(used_locals: &mut UsedLocals, body: &mut Body<'_>)
while modified {
modified = false;
- for data in body.basic_blocks_mut() {
+ for data in body.basic_blocks.as_mut_preserves_cfg() {
// Remove unnecessary StorageLive and StorageDead annotations.
data.statements.retain(|statement| {
let keep = match &statement.kind {
diff --git a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs
index bbfaace70..321d8c63b 100644
--- a/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs
+++ b/compiler/rustc_mir_transform/src/simplify_comparison_integral.rs
@@ -151,7 +151,7 @@ struct OptimizationFinder<'a, 'tcx> {
impl<'tcx> OptimizationFinder<'_, 'tcx> {
fn find_optimizations(&self) -> Vec<OptimizationInfo<'tcx>> {
self.body
- .basic_blocks()
+ .basic_blocks
.iter_enumerated()
.filter_map(|(bb_idx, bb)| {
// find switch
diff --git a/compiler/rustc_mir_transform/src/simplify_try.rs b/compiler/rustc_mir_transform/src/simplify_try.rs
index d52f1261b..baeb620ef 100644
--- a/compiler/rustc_mir_transform/src/simplify_try.rs
+++ b/compiler/rustc_mir_transform/src/simplify_try.rs
@@ -596,7 +596,7 @@ struct SimplifyBranchSameOptimizationFinder<'a, 'tcx> {
impl<'tcx> SimplifyBranchSameOptimizationFinder<'_, 'tcx> {
fn find(&self) -> Vec<SimplifyBranchSameOptimization> {
self.body
- .basic_blocks()
+ .basic_blocks
.iter_enumerated()
.filter_map(|(bb_idx, bb)| {
let (discr_switched_on, targets_and_values) = match &bb.terminator().kind {
@@ -632,7 +632,7 @@ impl<'tcx> SimplifyBranchSameOptimizationFinder<'_, 'tcx> {
let mut iter_bbs_reachable = targets_and_values
.iter()
- .map(|target_and_value| (target_and_value, &self.body.basic_blocks()[target_and_value.target]))
+ .map(|target_and_value| (target_and_value, &self.body.basic_blocks[target_and_value.target]))
.filter(|(_, bb)| {
// Reaching `unreachable` is UB so assume it doesn't happen.
bb.terminator().kind != TerminatorKind::Unreachable
diff --git a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs
index 30be64f5b..96ea15f1b 100644
--- a/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs
+++ b/compiler/rustc_mir_transform/src/uninhabited_enum_branching.rs
@@ -79,7 +79,7 @@ fn ensure_otherwise_unreachable<'tcx>(
targets: &SwitchTargets,
) -> Option<BasicBlockData<'tcx>> {
let otherwise = targets.otherwise();
- let bb = &body.basic_blocks()[otherwise];
+ let bb = &body.basic_blocks[otherwise];
if bb.terminator().kind == TerminatorKind::Unreachable
&& bb.statements.iter().all(|s| matches!(&s.kind, StatementKind::StorageDead(_)))
{
@@ -102,10 +102,10 @@ impl<'tcx> MirPass<'tcx> for UninhabitedEnumBranching {
fn run_pass(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
trace!("UninhabitedEnumBranching starting for {:?}", body.source);
- for bb in body.basic_blocks().indices() {
+ for bb in body.basic_blocks.indices() {
trace!("processing block {:?}", bb);
- let Some(discriminant_ty) = get_switched_on_type(&body.basic_blocks()[bb], tcx, body) else {
+ let Some(discriminant_ty) = get_switched_on_type(&body.basic_blocks[bb], tcx, body) else {
continue;
};
diff --git a/compiler/rustc_mir_transform/src/unreachable_prop.rs b/compiler/rustc_mir_transform/src/unreachable_prop.rs
index f916ca362..95fda2eaf 100644
--- a/compiler/rustc_mir_transform/src/unreachable_prop.rs
+++ b/compiler/rustc_mir_transform/src/unreachable_prop.rs
@@ -12,9 +12,8 @@ pub struct UnreachablePropagation;
impl MirPass<'_> for UnreachablePropagation {
fn is_enabled(&self, sess: &rustc_session::Session) -> bool {
- // Enable only under -Zmir-opt-level=4 as in some cases (check the deeply-nested-opt
- // perf benchmark) LLVM may spend quite a lot of time optimizing the generated code.
- sess.mir_opt_level() >= 4
+ // Enable only under -Zmir-opt-level=2 as this can make programs less debuggable.
+ sess.mir_opt_level() >= 2
}
fn run_pass<'tcx>(&self, tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) {
@@ -38,7 +37,19 @@ impl MirPass<'_> for UnreachablePropagation {
}
}
+ // We do want do keep some unreachable blocks, but make them empty.
+ for bb in unreachable_blocks {
+ if !tcx.consider_optimizing(|| {
+ format!("UnreachablePropagation {:?} ", body.source.def_id())
+ }) {
+ break;
+ }
+
+ body.basic_blocks_mut()[bb].statements.clear();
+ }
+
let replaced = !replacements.is_empty();
+
for (bb, terminator_kind) in replacements {
if !tcx.consider_optimizing(|| {
format!("UnreachablePropagation {:?} ", body.source.def_id())
@@ -57,42 +68,55 @@ impl MirPass<'_> for UnreachablePropagation {
fn remove_successors<'tcx, F>(
terminator_kind: &TerminatorKind<'tcx>,
- predicate: F,
+ is_unreachable: F,
) -> Option<TerminatorKind<'tcx>>
where
F: Fn(BasicBlock) -> bool,
{
- let terminator = match *terminator_kind {
- TerminatorKind::Goto { target } if predicate(target) => TerminatorKind::Unreachable,
- TerminatorKind::SwitchInt { ref discr, switch_ty, ref targets } => {
+ let terminator = match terminator_kind {
+ // This will unconditionally run into an unreachable and is therefore unreachable as well.
+ TerminatorKind::Goto { target } if is_unreachable(*target) => TerminatorKind::Unreachable,
+ TerminatorKind::SwitchInt { targets, discr, switch_ty } => {
let otherwise = targets.otherwise();
- let original_targets_len = targets.iter().len() + 1;
- let (mut values, mut targets): (Vec<_>, Vec<_>) =
- targets.iter().filter(|(_, bb)| !predicate(*bb)).unzip();
+ // If all targets are unreachable, we can be unreachable as well.
+ if targets.all_targets().iter().all(|bb| is_unreachable(*bb)) {
+ TerminatorKind::Unreachable
+ } else if is_unreachable(otherwise) {
+ // If there are multiple targets, don't delete unreachable branches (like an unreachable otherwise)
+ // unless otherwise is unreachable, in which case deleting a normal branch causes it to be merged with
+ // the otherwise, keeping its unreachable.
+ // This looses information about reachability causing worse codegen.
+ // For example (see src/test/codegen/match-optimizes-away.rs)
+ //
+ // pub enum Two { A, B }
+ // pub fn identity(x: Two) -> Two {
+ // match x {
+ // Two::A => Two::A,
+ // Two::B => Two::B,
+ // }
+ // }
+ //
+ // This generates a `switchInt() -> [0: 0, 1: 1, otherwise: unreachable]`, which allows us or LLVM to
+ // turn it into just `x` later. Without the unreachable, such a transformation would be illegal.
+ // If the otherwise branch is unreachable, we can delete all other unreacahble targets, as they will
+ // still point to the unreachable and therefore not lose reachability information.
+ let reachable_iter = targets.iter().filter(|(_, bb)| !is_unreachable(*bb));
- if !predicate(otherwise) {
- targets.push(otherwise);
- } else {
- values.pop();
- }
+ let new_targets = SwitchTargets::new(reachable_iter, otherwise);
- let retained_targets_len = targets.len();
+ // No unreachable branches were removed.
+ if new_targets.all_targets().len() == targets.all_targets().len() {
+ return None;
+ }
- if targets.is_empty() {
- TerminatorKind::Unreachable
- } else if targets.len() == 1 {
- TerminatorKind::Goto { target: targets[0] }
- } else if original_targets_len != retained_targets_len {
TerminatorKind::SwitchInt {
discr: discr.clone(),
- switch_ty,
- targets: SwitchTargets::new(
- values.iter().copied().zip(targets.iter().copied()),
- *targets.last().unwrap(),
- ),
+ switch_ty: *switch_ty,
+ targets: new_targets,
}
} else {
+ // If the otherwise branch is reachable, we don't want to delete any unreachable branches.
return None;
}
}