diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src/lib.rs')
| -rw-r--r-- | compiler/rustc_mir_transform/src/lib.rs | 180 |
1 files changed, 132 insertions, 48 deletions
diff --git a/compiler/rustc_mir_transform/src/lib.rs b/compiler/rustc_mir_transform/src/lib.rs index d968a4885..e6fc85595 100644 --- a/compiler/rustc_mir_transform/src/lib.rs +++ b/compiler/rustc_mir_transform/src/lib.rs @@ -1,7 +1,7 @@ #![allow(rustc::potential_query_instability)] #![feature(box_patterns)] #![feature(let_chains)] -#![feature(let_else)] +#![cfg_attr(bootstrap, feature(let_else))] #![feature(map_try_insert)] #![feature(min_specialization)] #![feature(never_type)] @@ -10,6 +10,7 @@ #![feature(trusted_step)] #![feature(try_blocks)] #![feature(yeet_expr)] +#![feature(if_let_guard)] #![recursion_limit = "256"] #[macro_use] @@ -26,10 +27,14 @@ use rustc_hir::def_id::{DefId, LocalDefId}; use rustc_hir::intravisit::{self, Visitor}; use rustc_index::vec::IndexVec; use rustc_middle::mir::visit::Visitor as _; -use rustc_middle::mir::{traversal, Body, ConstQualifs, MirPass, MirPhase, Promoted}; +use rustc_middle::mir::{ + traversal, AnalysisPhase, Body, ConstQualifs, Constant, LocalDecl, MirPass, MirPhase, Operand, + Place, ProjectionElem, Promoted, RuntimePhase, Rvalue, SourceInfo, Statement, StatementKind, + TerminatorKind, +}; use rustc_middle::ty::query::Providers; use rustc_middle::ty::{self, TyCtxt, TypeVisitable}; -use rustc_span::{Span, Symbol}; +use rustc_span::sym; #[macro_use] mod pass_manager; @@ -139,6 +144,64 @@ pub fn provide(providers: &mut Providers) { }; } +fn remap_mir_for_const_eval_select<'tcx>( + tcx: TyCtxt<'tcx>, + mut body: Body<'tcx>, + context: hir::Constness, +) -> Body<'tcx> { + for bb in body.basic_blocks.as_mut().iter_mut() { + let terminator = bb.terminator.as_mut().expect("invalid terminator"); + match terminator.kind { + TerminatorKind::Call { + func: Operand::Constant(box Constant { ref literal, .. }), + ref mut args, + destination, + target, + cleanup, + fn_span, + .. + } if let ty::FnDef(def_id, _) = *literal.ty().kind() + && tcx.item_name(def_id) == sym::const_eval_select + && tcx.is_intrinsic(def_id) => + { + let [tupled_args, called_in_const, called_at_rt]: [_; 3] = std::mem::take(args).try_into().unwrap(); + let ty = tupled_args.ty(&body.local_decls, tcx); + let fields = ty.tuple_fields(); + let num_args = fields.len(); + let func = if context == hir::Constness::Const { called_in_const } else { called_at_rt }; + let (method, place): (fn(Place<'tcx>) -> Operand<'tcx>, Place<'tcx>) = match tupled_args { + Operand::Constant(_) => { + // there is no good way of extracting a tuple arg from a constant (const generic stuff) + // so we just create a temporary and deconstruct that. + let local = body.local_decls.push(LocalDecl::new(ty, fn_span)); + bb.statements.push(Statement { + source_info: SourceInfo::outermost(fn_span), + kind: StatementKind::Assign(Box::new((local.into(), Rvalue::Use(tupled_args.clone())))), + }); + (Operand::Move, local.into()) + } + Operand::Move(place) => (Operand::Move, place), + Operand::Copy(place) => (Operand::Copy, place), + }; + let place_elems = place.projection; + let arguments = (0..num_args).map(|x| { + let mut place_elems = place_elems.to_vec(); + place_elems.push(ProjectionElem::Field(x.into(), fields[x])); + let projection = tcx.intern_place_elems(&place_elems); + let place = Place { + local: place.local, + projection, + }; + method(place) + }).collect(); + terminator.kind = TerminatorKind::Call { func, args: arguments, destination, target, cleanup, from_hir_call: false, fn_span }; + } + _ => {} + } + } + body +} + fn is_mir_available(tcx: TyCtxt<'_>, def_id: DefId) -> bool { let def_id = def_id.expect_local(); tcx.mir_keys(()).contains(&def_id) @@ -159,14 +222,7 @@ fn mir_keys(tcx: TyCtxt<'_>, (): ()) -> FxIndexSet<LocalDefId> { set: &'a mut FxIndexSet<LocalDefId>, } impl<'tcx> Visitor<'tcx> for GatherCtors<'_, 'tcx> { - fn visit_variant_data( - &mut self, - v: &'tcx hir::VariantData<'tcx>, - _: Symbol, - _: &'tcx hir::Generics<'tcx>, - _: hir::HirId, - _: Span, - ) { + fn visit_variant_data(&mut self, v: &'tcx hir::VariantData<'tcx>) { if let hir::VariantData::Tuple(_, hir_id) = *v { self.set.insert(self.tcx.hir().local_def_id(hir_id)); } @@ -208,6 +264,8 @@ fn mir_const_qualif(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) -> } /// Make MIR ready for const evaluation. This is run on all MIR, not just on consts! +/// FIXME(oli-obk): it's unclear whether we still need this phase (and its corresponding query). +/// We used to have this for pre-miri MIR based const eval. fn mir_const<'tcx>( tcx: TyCtxt<'tcx>, def: ty::WithOptConstParam<LocalDefId>, @@ -243,7 +301,6 @@ fn mir_const<'tcx>( // What we need to do constant evaluation. &simplify::SimplifyCfg::new("initial"), &rustc_peek::SanityCheck, // Just a lint - &marker::PhaseChange(MirPhase::Const), ], ); tcx.alloc_steal_mir(body) @@ -330,7 +387,9 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) - .body_const_context(def.did) .expect("mir_for_ctfe should not be used for runtime functions"); - let mut body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone(); + let body = tcx.mir_drops_elaborated_and_const_checked(def).borrow().clone(); + + let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::Const); match context { // Do not const prop functions, either they get executed at runtime or exported to metadata, @@ -349,7 +408,10 @@ fn inner_mir_for_ctfe(tcx: TyCtxt<'_>, def: ty::WithOptConstParam<LocalDefId>) - pm::run_passes( tcx, &mut body, - &[&const_prop::ConstProp, &marker::PhaseChange(MirPhase::Optimized)], + &[ + &const_prop::ConstProp, + &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)), + ], ); } } @@ -389,38 +451,61 @@ fn mir_drops_elaborated_and_const_checked<'tcx>( body.tainted_by_errors = Some(error_reported); } - // IMPORTANT - pm::run_passes(tcx, &mut body, &[&remove_false_edges::RemoveFalseEdges]); + run_analysis_to_runtime_passes(tcx, &mut body); + + tcx.alloc_steal_mir(body) +} + +fn run_analysis_to_runtime_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + assert!(body.phase == MirPhase::Analysis(AnalysisPhase::Initial)); + let did = body.source.def_id(); + + debug!("analysis_mir_cleanup({:?})", did); + run_analysis_cleanup_passes(tcx, body); + assert!(body.phase == MirPhase::Analysis(AnalysisPhase::PostCleanup)); // Do a little drop elaboration before const-checking if `const_precise_live_drops` is enabled. if check_consts::post_drop_elaboration::checking_enabled(&ConstCx::new(tcx, &body)) { pm::run_passes( tcx, - &mut body, + body, &[ - &simplify::SimplifyCfg::new("remove-false-edges"), &remove_uninit_drops::RemoveUninitDrops, + &simplify::SimplifyCfg::new("remove-false-edges"), ], ); check_consts::post_drop_elaboration::check_live_drops(tcx, &body); // FIXME: make this a MIR lint } - run_post_borrowck_cleanup_passes(tcx, &mut body); - assert!(body.phase == MirPhase::Deaggregated); - tcx.alloc_steal_mir(body) + debug!("runtime_mir_lowering({:?})", did); + run_runtime_lowering_passes(tcx, body); + assert!(body.phase == MirPhase::Runtime(RuntimePhase::Initial)); + + debug!("runtime_mir_cleanup({:?})", did); + run_runtime_cleanup_passes(tcx, body); + assert!(body.phase == MirPhase::Runtime(RuntimePhase::PostCleanup)); } -/// After this series of passes, no lifetime analysis based on borrowing can be done. -fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { - debug!("post_borrowck_cleanup({:?})", body.source.def_id()); +// FIXME(JakobDegen): Can we make these lists of passes consts? - let post_borrowck_cleanup: &[&dyn MirPass<'tcx>] = &[ - // Remove all things only needed by analysis +/// After this series of passes, no lifetime analysis based on borrowing can be done. +fn run_analysis_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let passes: &[&dyn MirPass<'tcx>] = &[ + &remove_false_edges::RemoveFalseEdges, &simplify_branches::SimplifyConstCondition::new("initial"), &remove_noop_landing_pads::RemoveNoopLandingPads, &cleanup_post_borrowck::CleanupNonCodegenStatements, &simplify::SimplifyCfg::new("early-opt"), &deref_separator::Derefer, + &marker::PhaseChange(MirPhase::Analysis(AnalysisPhase::PostCleanup)), + ]; + + pm::run_passes(tcx, body, passes); +} + +/// Returns the sequence of passes that lowers analysis to runtime MIR. +fn run_runtime_lowering_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let passes: &[&dyn MirPass<'tcx>] = &[ // These next passes must be executed together &add_call_guards::CriticalCallEdges, &elaborate_drops::ElaborateDrops, @@ -434,16 +519,27 @@ fn run_post_borrowck_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tc // `AddRetag` needs to run after `ElaborateDrops`. Otherwise it should run fairly late, // but before optimizations begin. &elaborate_box_derefs::ElaborateBoxDerefs, + &generator::StateTransform, &add_retag::AddRetag, - &lower_intrinsics::LowerIntrinsics, - &simplify::SimplifyCfg::new("elaborate-drops"), - // `Deaggregator` is conceptually part of MIR building, some backends rely on it happening - // and it can help optimizations. + // Deaggregator is necessary for const prop. We may want to consider implementing + // CTFE support for aggregates. &deaggregator::Deaggregator, &Lint(const_prop_lint::ConstProp), + &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Initial)), + ]; + pm::run_passes_no_validate(tcx, body, passes); +} + +/// Returns the sequence of passes that do the initial cleanup of runtime MIR. +fn run_runtime_cleanup_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { + let passes: &[&dyn MirPass<'tcx>] = &[ + &elaborate_box_derefs::ElaborateBoxDerefs, + &lower_intrinsics::LowerIntrinsics, + &simplify::SimplifyCfg::new("elaborate-drops"), + &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::PostCleanup)), ]; - pm::run_passes(tcx, body, post_borrowck_cleanup); + pm::run_passes(tcx, body, passes); } fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { @@ -451,9 +547,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { WithMinOptLevel(1, x) } - // Lowering generator control-flow and variables has to happen before we do anything else - // to them. We run some optimizations before that, because they may be harder to do on the state - // machine than on MIR with async primitives. + // The main optimizations that we do on MIR. pm::run_passes( tcx, body, @@ -465,17 +559,6 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &uninhabited_enum_branching::UninhabitedEnumBranching, &o1(simplify::SimplifyCfg::new("after-uninhabited-enum-branching")), &inline::Inline, - &generator::StateTransform, - ], - ); - - assert!(body.phase == MirPhase::GeneratorsLowered); - - // The main optimizations that we do on MIR. - pm::run_passes( - tcx, - body, - &[ &remove_storage_markers::RemoveStorageMarkers, &remove_zsts::RemoveZsts, &const_goto::ConstGoto, @@ -507,7 +590,7 @@ fn run_optimization_passes<'tcx>(tcx: TyCtxt<'tcx>, body: &mut Body<'tcx>) { &deduplicate_blocks::DeduplicateBlocks, // Some cleanup necessary at least for LLVM and potentially other codegen backends. &add_call_guards::CriticalCallEdges, - &marker::PhaseChange(MirPhase::Optimized), + &marker::PhaseChange(MirPhase::Runtime(RuntimePhase::Optimized)), // Dump the end result for testing and debugging purposes. &dump_mir::Marker("PreCodegen"), ], @@ -539,8 +622,9 @@ fn inner_optimized_mir(tcx: TyCtxt<'_>, did: LocalDefId) -> Body<'_> { Some(other) => panic!("do not use `optimized_mir` for constants: {:?}", other), } debug!("about to call mir_drops_elaborated..."); - let mut body = + let body = tcx.mir_drops_elaborated_and_const_checked(ty::WithOptConstParam::unknown(did)).steal(); + let mut body = remap_mir_for_const_eval_select(tcx, body, hir::Constness::NotConst); debug!("body: {:#?}", body); run_optimization_passes(tcx, &mut body); @@ -566,7 +650,7 @@ fn promoted_mir<'tcx>( if let Some(error_reported) = tainted_by_errors { body.tainted_by_errors = Some(error_reported); } - run_post_borrowck_cleanup_passes(tcx, body); + run_analysis_to_runtime_passes(tcx, body); } debug_assert!(!promoted.has_free_regions(), "Free regions in promoted MIR"); |