diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src/inline.rs')
| -rw-r--r-- | compiler/rustc_mir_transform/src/inline.rs | 89 |
1 files changed, 43 insertions, 46 deletions
diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index 6e6d6566f..f0cb317f4 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -10,7 +10,7 @@ use rustc_middle::mir::*; use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; use rustc_session::config::OptLevel; use rustc_span::{hygiene::ExpnKind, ExpnData, LocalExpnId, Span}; -use rustc_target::abi::VariantIdx; +use rustc_target::abi::{FieldIdx, FIRST_VARIANT}; use rustc_target::spec::abi::Abi; use crate::simplify::{remove_dead_blocks, CfgSimplifier}; @@ -270,7 +270,9 @@ impl<'tcx> Inliner<'tcx> { | InstanceDef::FnPtrShim(..) | InstanceDef::ClosureOnceShim { .. } | InstanceDef::DropGlue(..) - | InstanceDef::CloneShim(..) => return Ok(()), + | InstanceDef::CloneShim(..) + | InstanceDef::ThreadLocalShim(..) + | InstanceDef::FnPtrAddrShim(..) => return Ok(()), } if self.tcx.is_constructor(callee_def_id) { @@ -424,13 +426,6 @@ impl<'tcx> Inliner<'tcx> { debug!(" final inline threshold = {}", threshold); // FIXME: Give a bonus to functions with only a single caller - let diverges = matches!( - callee_body.basic_blocks[START_BLOCK].terminator().kind, - TerminatorKind::Unreachable | TerminatorKind::Call { target: None, .. } - ); - if diverges && !matches!(callee_attrs.inline, InlineAttr::Always) { - return Err("callee diverges unconditionally"); - } let mut checker = CostChecker { tcx: self.tcx, @@ -453,14 +448,12 @@ impl<'tcx> Inliner<'tcx> { checker.visit_basic_block_data(bb, blk); let term = blk.terminator(); - if let TerminatorKind::Drop { ref place, target, unwind } - | TerminatorKind::DropAndReplace { ref place, target, unwind, .. } = term.kind - { + if let TerminatorKind::Drop { ref place, target, unwind } = term.kind { work_list.push(target); // If the place doesn't actually need dropping, treat it like a regular goto. let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty); - if ty.needs_drop(tcx, self.param_env) && let Some(unwind) = unwind { + if ty.needs_drop(tcx, self.param_env) && let UnwindAction::Cleanup(unwind) = unwind { work_list.push(unwind); } } else if callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set @@ -507,7 +500,7 @@ impl<'tcx> Inliner<'tcx> { ) { let terminator = caller_body[callsite.block].terminator.take().unwrap(); match terminator.kind { - TerminatorKind::Call { args, destination, cleanup, .. } => { + TerminatorKind::Call { args, destination, unwind, .. } => { // If the call is something like `a[*i] = f(i)`, where // `i : &mut usize`, then just duplicating the `a[*i]` // Place could result in two different locations if `f` @@ -578,7 +571,7 @@ impl<'tcx> Inliner<'tcx> { destination: destination_local, callsite_scope: caller_body.source_scopes[callsite.source_info.scope].clone(), callsite, - cleanup_block: cleanup, + cleanup_block: unwind, in_cleanup_block: false, tcx: self.tcx, expn_data, @@ -708,7 +701,7 @@ impl<'tcx> Inliner<'tcx> { // The `tmp0`, `tmp1`, and `tmp2` in our example above. let tuple_tmp_args = tuple_tys.iter().enumerate().map(|(i, ty)| { // This is e.g., `tuple_tmp.0` in our example above. - let tuple_field = Operand::Move(tcx.mk_place_field(tuple, Field::new(i), ty)); + let tuple_field = Operand::Move(tcx.mk_place_field(tuple, FieldIdx::new(i), ty)); // Spill to a local to make e.g., `tmp0`. self.create_temp_if_necessary(tuple_field, callsite, caller_body) @@ -815,20 +808,19 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { fn visit_terminator(&mut self, terminator: &Terminator<'tcx>, location: Location) { let tcx = self.tcx; match terminator.kind { - TerminatorKind::Drop { ref place, unwind, .. } - | TerminatorKind::DropAndReplace { ref place, unwind, .. } => { + TerminatorKind::Drop { ref place, unwind, .. } => { // If the place doesn't actually need dropping, treat it like a regular goto. let ty = self.instance.subst_mir(tcx, &place.ty(self.callee_body, tcx).ty); if ty.needs_drop(tcx, self.param_env) { self.cost += CALL_PENALTY; - if unwind.is_some() { + if let UnwindAction::Cleanup(_) = unwind { self.cost += LANDINGPAD_PENALTY; } } else { self.cost += INSTR_COST; } } - TerminatorKind::Call { func: Operand::Constant(ref f), cleanup, .. } => { + TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { let fn_ty = self.instance.subst_mir(tcx, &f.literal.ty()); self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { // Don't give intrinsics the extra penalty for calls @@ -836,20 +828,20 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } else { CALL_PENALTY }; - if cleanup.is_some() { + if let UnwindAction::Cleanup(_) = unwind { self.cost += LANDINGPAD_PENALTY; } } - TerminatorKind::Assert { cleanup, .. } => { + TerminatorKind::Assert { unwind, .. } => { self.cost += CALL_PENALTY; - if cleanup.is_some() { + if let UnwindAction::Cleanup(_) = unwind { self.cost += LANDINGPAD_PENALTY; } } TerminatorKind::Resume => self.cost += RESUME_PENALTY, - TerminatorKind::InlineAsm { cleanup, .. } => { + TerminatorKind::InlineAsm { unwind, .. } => { self.cost += INSTR_COST; - if cleanup.is_some() { + if let UnwindAction::Cleanup(_) = unwind { self.cost += LANDINGPAD_PENALTY; } } @@ -914,8 +906,8 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { check_equal(self, *f_ty); } ty::Adt(adt_def, substs) => { - let var = parent_ty.variant_index.unwrap_or(VariantIdx::from_u32(0)); - let Some(field) = adt_def.variant(var).fields.get(f.as_usize()) else { + let var = parent_ty.variant_index.unwrap_or(FIRST_VARIANT); + let Some(field) = adt_def.variant(var).fields.get(f) else { self.validation = Err("malformed MIR"); return; }; @@ -987,7 +979,7 @@ struct Integrator<'a, 'tcx> { destination: Local, callsite_scope: SourceScopeData<'tcx>, callsite: &'a CallSite<'tcx>, - cleanup_block: Option<BasicBlock>, + cleanup_block: UnwindAction, in_cleanup_block: bool, tcx: TyCtxt<'tcx>, expn_data: LocalExpnId, @@ -1022,18 +1014,21 @@ impl Integrator<'_, '_> { new } - fn map_unwind(&self, unwind: Option<BasicBlock>) -> Option<BasicBlock> { + fn map_unwind(&self, unwind: UnwindAction) -> UnwindAction { if self.in_cleanup_block { - if unwind.is_some() { - bug!("cleanup on cleanup block"); + match unwind { + UnwindAction::Cleanup(_) | UnwindAction::Continue => { + bug!("cleanup on cleanup block"); + } + UnwindAction::Unreachable | UnwindAction::Terminate => return unwind, } - return unwind; } match unwind { - Some(target) => Some(self.map_block(target)), + UnwindAction::Unreachable | UnwindAction::Terminate => unwind, + UnwindAction::Cleanup(target) => UnwindAction::Cleanup(self.map_block(target)), // Add an unwind edge to the original call's cleanup block - None => self.cleanup_block, + UnwindAction::Continue => self.cleanup_block, } } } @@ -1120,20 +1115,19 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { *tgt = self.map_block(*tgt); } } - TerminatorKind::Drop { ref mut target, ref mut unwind, .. } - | TerminatorKind::DropAndReplace { ref mut target, ref mut unwind, .. } => { + TerminatorKind::Drop { ref mut target, ref mut unwind, .. } => { *target = self.map_block(*target); *unwind = self.map_unwind(*unwind); } - TerminatorKind::Call { ref mut target, ref mut cleanup, .. } => { + TerminatorKind::Call { ref mut target, ref mut unwind, .. } => { if let Some(ref mut tgt) = *target { *tgt = self.map_block(*tgt); } - *cleanup = self.map_unwind(*cleanup); + *unwind = self.map_unwind(*unwind); } - TerminatorKind::Assert { ref mut target, ref mut cleanup, .. } => { + TerminatorKind::Assert { ref mut target, ref mut unwind, .. } => { *target = self.map_block(*target); - *cleanup = self.map_unwind(*cleanup); + *unwind = self.map_unwind(*unwind); } TerminatorKind::Return => { terminator.kind = if let Some(tgt) = self.callsite.target { @@ -1143,11 +1137,14 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { } } TerminatorKind::Resume => { - if let Some(tgt) = self.cleanup_block { - terminator.kind = TerminatorKind::Goto { target: tgt } - } + terminator.kind = match self.cleanup_block { + UnwindAction::Cleanup(tgt) => TerminatorKind::Goto { target: tgt }, + UnwindAction::Continue => TerminatorKind::Resume, + UnwindAction::Unreachable => TerminatorKind::Unreachable, + UnwindAction::Terminate => TerminatorKind::Terminate, + }; } - TerminatorKind::Abort => {} + TerminatorKind::Terminate => {} TerminatorKind::Unreachable => {} TerminatorKind::FalseEdge { ref mut real_target, ref mut imaginary_target } => { *real_target = self.map_block(*real_target); @@ -1158,11 +1155,11 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { { bug!("False unwinds should have been removed before inlining") } - TerminatorKind::InlineAsm { ref mut destination, ref mut cleanup, .. } => { + TerminatorKind::InlineAsm { ref mut destination, ref mut unwind, .. } => { if let Some(ref mut tgt) = *destination { *tgt = self.map_block(*tgt); } - *cleanup = self.map_unwind(*cleanup); + *unwind = self.map_unwind(*unwind); } } } |