From 9835e2ae736235810b4ea1c162ca5e65c547e770 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Sat, 18 May 2024 04:49:50 +0200 Subject: Merging upstream version 1.71.1+dfsg1. Signed-off-by: Daniel Baumann --- compiler/rustc_mir_transform/src/inline.rs | 139 ++++++++++++++--------------- 1 file changed, 66 insertions(+), 73 deletions(-) (limited to 'compiler/rustc_mir_transform/src/inline.rs') diff --git a/compiler/rustc_mir_transform/src/inline.rs b/compiler/rustc_mir_transform/src/inline.rs index f0cb317f4..233de6dc6 100644 --- a/compiler/rustc_mir_transform/src/inline.rs +++ b/compiler/rustc_mir_transform/src/inline.rs @@ -3,13 +3,13 @@ use crate::deref_separator::deref_finder; use rustc_attr::InlineAttr; use rustc_hir::def_id::DefId; use rustc_index::bit_set::BitSet; -use rustc_index::vec::Idx; +use rustc_index::Idx; use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; use rustc_middle::mir::visit::*; use rustc_middle::mir::*; +use rustc_middle::ty::TypeVisitableExt; use rustc_middle::ty::{self, Instance, InstanceDef, ParamEnv, Ty, TyCtxt}; use rustc_session::config::OptLevel; -use rustc_span::{hygiene::ExpnKind, ExpnData, LocalExpnId, Span}; use rustc_target::abi::{FieldIdx, FIRST_VARIANT}; use rustc_target::spec::abi::Abi; @@ -26,8 +26,6 @@ const CALL_PENALTY: usize = 25; const LANDINGPAD_PENALTY: usize = 50; const RESUME_PENALTY: usize = 45; -const UNKNOWN_SIZE_COST: usize = 10; - const TOP_DOWN_DEPTH_LIMIT: usize = 5; pub struct Inline; @@ -169,8 +167,20 @@ impl<'tcx> Inliner<'tcx> { ) -> Result, &'static str> { let callee_attrs = self.tcx.codegen_fn_attrs(callsite.callee.def_id()); self.check_codegen_attributes(callsite, callee_attrs)?; + + let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); + let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; + let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; + for arg in args { + if !arg.ty(&caller_body.local_decls, self.tcx).is_sized(self.tcx, self.param_env) { + // We do not allow inlining functions with unsized params. Inlining these functions + // could create unsized locals, which are unsound and being phased out. + return Err("Call has unsized argument"); + } + } + self.check_mir_is_available(caller_body, &callsite.callee)?; - let callee_body = self.tcx.instance_mir(callsite.callee.def); + let callee_body = try_instance_mir(self.tcx, callsite.callee.def)?; self.check_mir_body(callsite, callee_body, callee_attrs)?; if !self.tcx.consider_optimizing(|| { @@ -182,7 +192,7 @@ impl<'tcx> Inliner<'tcx> { let Ok(callee_body) = callsite.callee.try_subst_mir_and_normalize_erasing_regions( self.tcx, self.param_env, - callee_body.clone(), + ty::EarlyBinder(callee_body.clone()), ) else { return Err("failed to normalize callee body"); }; @@ -190,9 +200,6 @@ impl<'tcx> Inliner<'tcx> { // Check call signature compatibility. // Normally, this shouldn't be required, but trait normalization failure can create a // validation ICE. - let terminator = caller_body[callsite.block].terminator.as_ref().unwrap(); - let TerminatorKind::Call { args, destination, .. } = &terminator.kind else { bug!() }; - let destination_ty = destination.ty(&caller_body.local_decls, self.tcx).ty; let output_type = callee_body.return_ty(); if !util::is_subtype(self.tcx, self.param_env, output_type, destination_ty) { trace!(?output_type, ?destination_ty); @@ -350,14 +357,8 @@ impl<'tcx> Inliner<'tcx> { callsite: &CallSite<'tcx>, callee_attrs: &CodegenFnAttrs, ) -> Result<(), &'static str> { - match callee_attrs.inline { - InlineAttr::Never => return Err("never inline hint"), - InlineAttr::Always | InlineAttr::Hint => {} - InlineAttr::None => { - if self.tcx.sess.mir_opt_level() <= 2 { - return Err("at mir-opt-level=2, only #[inline] is inlined"); - } - } + if let InlineAttr::Never = callee_attrs.inline { + return Err("never inline hint"); } // Only inline local functions if they would be eligible for cross-crate @@ -436,6 +437,10 @@ impl<'tcx> Inliner<'tcx> { validation: Ok(()), }; + for var_debug_info in callee_body.var_debug_info.iter() { + checker.visit_var_debug_info(var_debug_info); + } + // Traverse the MIR manually so we can account for the effects of inlining on the CFG. let mut work_list = vec![START_BLOCK]; let mut visited = BitSet::new_empty(callee_body.basic_blocks.len()); @@ -448,14 +453,16 @@ impl<'tcx> Inliner<'tcx> { checker.visit_basic_block_data(bb, blk); let term = blk.terminator(); - if let TerminatorKind::Drop { ref place, target, unwind } = term.kind { + if let TerminatorKind::Drop { ref place, target, unwind, replace: _ } = term.kind { work_list.push(target); // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = callsite.callee.subst_mir(self.tcx, &place.ty(callee_body, tcx).ty); + let ty = callsite + .callee + .subst_mir(self.tcx, ty::EarlyBinder(&place.ty(callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) && let UnwindAction::Cleanup(unwind) = unwind { - work_list.push(unwind); - } + work_list.push(unwind); + } } else if callee_attrs.instruction_set != self.codegen_fn_attrs.instruction_set && matches!(term.kind, TerminatorKind::InlineAsm { .. }) { @@ -470,12 +477,6 @@ impl<'tcx> Inliner<'tcx> { } } - // Count up the cost of local variables and temps, if we know the size - // use that, otherwise we use a moderately-large dummy cost. - for v in callee_body.vars_and_temps_iter() { - checker.visit_local_decl(v, &callee_body.local_decls[v]); - } - // Abort if type validation found anything fishy. checker.validation?; @@ -553,16 +554,6 @@ impl<'tcx> Inliner<'tcx> { // Copy the arguments if needed. let args: Vec<_> = self.make_call_args(args, &callsite, caller_body, &callee_body); - let mut expn_data = ExpnData::default( - ExpnKind::Inlined, - callsite.source_info.span, - self.tcx.sess.edition(), - None, - None, - ); - expn_data.def_site = callee_body.span; - let expn_data = - self.tcx.with_stable_hashing_context(|hcx| LocalExpnId::fresh(expn_data, hcx)); let mut integrator = Integrator { args: &args, new_locals: Local::new(caller_body.local_decls.len()).., @@ -574,7 +565,6 @@ impl<'tcx> Inliner<'tcx> { cleanup_block: unwind, in_cleanup_block: false, tcx: self.tcx, - expn_data, always_live_locals: BitSet::new_filled(callee_body.local_decls.len()), }; @@ -770,14 +760,6 @@ impl<'tcx> Inliner<'tcx> { } } -fn type_size_of<'tcx>( - tcx: TyCtxt<'tcx>, - param_env: ty::ParamEnv<'tcx>, - ty: Ty<'tcx>, -) -> Option { - tcx.layout_of(param_env.and(ty)).ok().map(|layout| layout.size.bytes()) -} - /// Verify that the callee body is compatible with the caller. /// /// This visitor mostly computes the inlining cost, @@ -810,7 +792,9 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { match terminator.kind { TerminatorKind::Drop { ref place, unwind, .. } => { // If the place doesn't actually need dropping, treat it like a regular goto. - let ty = self.instance.subst_mir(tcx, &place.ty(self.callee_body, tcx).ty); + let ty = self + .instance + .subst_mir(tcx, ty::EarlyBinder(&place.ty(self.callee_body, tcx).ty)); if ty.needs_drop(tcx, self.param_env) { self.cost += CALL_PENALTY; if let UnwindAction::Cleanup(_) = unwind { @@ -821,7 +805,7 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { } } TerminatorKind::Call { func: Operand::Constant(ref f), unwind, .. } => { - let fn_ty = self.instance.subst_mir(tcx, &f.literal.ty()); + let fn_ty = self.instance.subst_mir(tcx, ty::EarlyBinder(&f.literal.ty())); self.cost += if let ty::FnDef(def_id, _) = *fn_ty.kind() && tcx.is_intrinsic(def_id) { // Don't give intrinsics the extra penalty for calls INSTR_COST @@ -851,24 +835,6 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { self.super_terminator(terminator, location); } - /// Count up the cost of local variables and temps, if we know the size - /// use that, otherwise we use a moderately-large dummy cost. - fn visit_local_decl(&mut self, local: Local, local_decl: &LocalDecl<'tcx>) { - let tcx = self.tcx; - let ptr_size = tcx.data_layout.pointer_size.bytes(); - - let ty = self.instance.subst_mir(tcx, &local_decl.ty); - // Cost of the var is the size in machine-words, if we know - // it. - if let Some(size) = type_size_of(tcx, self.param_env, ty) { - self.cost += ((size + ptr_size - 1) / ptr_size) as usize; - } else { - self.cost += UNKNOWN_SIZE_COST; - } - - self.super_local_decl(local, local_decl) - } - /// This method duplicates code from MIR validation in an attempt to detect type mismatches due /// to normalization failure. fn visit_projection_elem( @@ -883,7 +849,16 @@ impl<'tcx> Visitor<'tcx> for CostChecker<'_, 'tcx> { let parent = Place { local, projection: self.tcx.mk_place_elems(proj_base) }; let parent_ty = parent.ty(&self.callee_body.local_decls, self.tcx); let check_equal = |this: &mut Self, f_ty| { - if !util::is_equal_up_to_subtyping(this.tcx, this.param_env, ty, f_ty) { + // Fast path if there is nothing to substitute. + if ty == f_ty { + return; + } + let ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder(&ty)); + let f_ty = this.instance.subst_mir(this.tcx, ty::EarlyBinder(&f_ty)); + if ty == f_ty { + return; + } + if !util::is_subtype(this.tcx, this.param_env, ty, f_ty) { trace!(?ty, ?f_ty); this.validation = Err("failed to normalize projection type"); return; @@ -982,7 +957,6 @@ struct Integrator<'a, 'tcx> { cleanup_block: UnwindAction, in_cleanup_block: bool, tcx: TyCtxt<'tcx>, - expn_data: LocalExpnId, always_live_locals: BitSet, } @@ -1068,11 +1042,6 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { *scope = self.map_scope(*scope); } - fn visit_span(&mut self, span: &mut Span) { - // Make sure that all spans track the fact that they were inlined. - *span = span.fresh_expansion(self.expn_data); - } - fn visit_basic_block_data(&mut self, block: BasicBlock, data: &mut BasicBlockData<'tcx>) { self.in_cleanup_block = data.is_cleanup; self.super_basic_block_data(block, data); @@ -1164,3 +1133,27 @@ impl<'tcx> MutVisitor<'tcx> for Integrator<'_, 'tcx> { } } } + +#[instrument(skip(tcx), level = "debug")] +fn try_instance_mir<'tcx>( + tcx: TyCtxt<'tcx>, + instance: InstanceDef<'tcx>, +) -> Result<&'tcx Body<'tcx>, &'static str> { + match instance { + ty::InstanceDef::DropGlue(_, Some(ty)) => match ty.kind() { + ty::Adt(def, substs) => { + let fields = def.all_fields(); + for field in fields { + let field_ty = field.ty(tcx, substs); + if field_ty.has_param() && field_ty.has_projections() { + return Err("cannot build drop shim for polymorphic type"); + } + } + + Ok(tcx.instance_mir(instance)) + } + _ => Ok(tcx.instance_mir(instance)), + }, + _ => Ok(tcx.instance_mir(instance)), + } +} -- cgit v1.2.3