diff options
Diffstat (limited to 'compiler/rustc_mir_transform/src/dest_prop.rs')
| -rw-r--r-- | compiler/rustc_mir_transform/src/dest_prop.rs | 180 |
1 files changed, 96 insertions, 84 deletions
diff --git a/compiler/rustc_mir_transform/src/dest_prop.rs b/compiler/rustc_mir_transform/src/dest_prop.rs index 97485c4f5..08e296a83 100644 --- a/compiler/rustc_mir_transform/src/dest_prop.rs +++ b/compiler/rustc_mir_transform/src/dest_prop.rs @@ -129,18 +129,16 @@ use std::collections::hash_map::{Entry, OccupiedEntry}; +use crate::simplify::remove_dead_blocks; use crate::MirPass; use rustc_data_structures::fx::FxHashMap; use rustc_index::bit_set::BitSet; +use rustc_middle::mir::visit::{MutVisitor, PlaceContext, Visitor}; use rustc_middle::mir::{dump_mir, PassWhere}; use rustc_middle::mir::{ traversal, BasicBlock, Body, InlineAsmOperand, Local, LocalKind, Location, Operand, Place, Rvalue, Statement, StatementKind, TerminatorKind, }; -use rustc_middle::mir::{ - visit::{MutVisitor, PlaceContext, Visitor}, - ProjectionElem, -}; use rustc_middle::ty::TyCtxt; use rustc_mir_dataflow::impls::MaybeLiveLocals; use rustc_mir_dataflow::{Analysis, ResultsCursor}; @@ -238,6 +236,12 @@ impl<'tcx> MirPass<'tcx> for DestinationPropagation { apply_merges(body, tcx, &merges, &merged_locals); } + if round_count != 0 { + // Merging can introduce overlap between moved arguments and/or call destination in an + // unreachable code, which validator considers to be ill-formed. + remove_dead_blocks(tcx, body); + } + trace!(round_count); } } @@ -359,40 +363,45 @@ struct FilterInformation<'a, 'body, 'alloc, 'tcx> { // through these methods, and not directly. impl<'alloc> Candidates<'alloc> { /// Just `Vec::retain`, but the condition is inverted and we add debugging output - fn vec_remove_debug( + fn vec_filter_candidates( src: Local, v: &mut Vec<Local>, - mut f: impl FnMut(Local) -> bool, + mut f: impl FnMut(Local) -> CandidateFilter, at: Location, ) { v.retain(|dest| { let remove = f(*dest); - if remove { + if remove == CandidateFilter::Remove { trace!("eliminating {:?} => {:?} due to conflict at {:?}", src, dest, at); } - !remove + remove == CandidateFilter::Keep }); } - /// `vec_remove_debug` but for an `Entry` - fn entry_remove( + /// `vec_filter_candidates` but for an `Entry` + fn entry_filter_candidates( mut entry: OccupiedEntry<'_, Local, Vec<Local>>, p: Local, - f: impl FnMut(Local) -> bool, + f: impl FnMut(Local) -> CandidateFilter, at: Location, ) { let candidates = entry.get_mut(); - Self::vec_remove_debug(p, candidates, f, at); + Self::vec_filter_candidates(p, candidates, f, at); if candidates.len() == 0 { entry.remove(); } } - /// Removes all candidates `(p, q)` or `(q, p)` where `p` is the indicated local and `f(q)` is true. - fn remove_candidates_if(&mut self, p: Local, mut f: impl FnMut(Local) -> bool, at: Location) { + /// For all candidates `(p, q)` or `(q, p)` removes the candidate if `f(q)` says to do so + fn filter_candidates_by( + &mut self, + p: Local, + mut f: impl FnMut(Local) -> CandidateFilter, + at: Location, + ) { // Cover the cases where `p` appears as a `src` if let Entry::Occupied(entry) = self.c.entry(p) { - Self::entry_remove(entry, p, &mut f, at); + Self::entry_filter_candidates(entry, p, &mut f, at); } // And the cases where `p` appears as a `dest` let Some(srcs) = self.reverse.get_mut(&p) else { @@ -401,18 +410,31 @@ impl<'alloc> Candidates<'alloc> { // We use `retain` here to remove the elements from the reverse set if we've removed the // matching candidate in the forward set. srcs.retain(|src| { - if !f(*src) { + if f(*src) == CandidateFilter::Keep { return true; } let Entry::Occupied(entry) = self.c.entry(*src) else { return false; }; - Self::entry_remove(entry, *src, |dest| dest == p, at); + Self::entry_filter_candidates( + entry, + *src, + |dest| { + if dest == p { CandidateFilter::Remove } else { CandidateFilter::Keep } + }, + at, + ); false }); } } +#[derive(Copy, Clone, PartialEq, Eq)] +enum CandidateFilter { + Keep, + Remove, +} + impl<'a, 'body, 'alloc, 'tcx> FilterInformation<'a, 'body, 'alloc, 'tcx> { /// Filters the set of candidates to remove those that conflict. /// @@ -460,7 +482,7 @@ impl<'a, 'body, 'alloc, 'tcx> FilterInformation<'a, 'body, 'alloc, 'tcx> { for (i, statement) in data.statements.iter().enumerate().rev() { self.at = Location { block, statement_index: i }; self.live.seek_after_primary_effect(self.at); - self.get_statement_write_info(&statement.kind); + self.write_info.for_statement(&statement.kind, self.body); self.apply_conflicts(); } } @@ -469,80 +491,59 @@ impl<'a, 'body, 'alloc, 'tcx> FilterInformation<'a, 'body, 'alloc, 'tcx> { fn apply_conflicts(&mut self) { let writes = &self.write_info.writes; for p in writes { - self.candidates.remove_candidates_if( + let other_skip = self.write_info.skip_pair.and_then(|(a, b)| { + if a == *p { + Some(b) + } else if b == *p { + Some(a) + } else { + None + } + }); + self.candidates.filter_candidates_by( *p, - // It is possible that a local may be live for less than the - // duration of a statement This happens in the case of function - // calls or inline asm. Because of this, we also mark locals as - // conflicting when both of them are written to in the same - // statement. - |q| self.live.contains(q) || writes.contains(&q), + |q| { + if Some(q) == other_skip { + return CandidateFilter::Keep; + } + // It is possible that a local may be live for less than the + // duration of a statement This happens in the case of function + // calls or inline asm. Because of this, we also mark locals as + // conflicting when both of them are written to in the same + // statement. + if self.live.contains(q) || writes.contains(&q) { + CandidateFilter::Remove + } else { + CandidateFilter::Keep + } + }, self.at, ); } } - - /// Gets the write info for the `statement`. - fn get_statement_write_info(&mut self, statement: &StatementKind<'tcx>) { - self.write_info.writes.clear(); - match statement { - StatementKind::Assign(box (lhs, rhs)) => match rhs { - Rvalue::Use(op) => { - if !lhs.is_indirect() { - self.get_assign_use_write_info(*lhs, op); - return; - } - } - _ => (), - }, - _ => (), - } - - self.write_info.for_statement(statement); - } - - fn get_assign_use_write_info(&mut self, lhs: Place<'tcx>, rhs: &Operand<'tcx>) { - // We register the writes for the operand unconditionally - self.write_info.add_operand(rhs); - // However, we cannot do the same thing for the `lhs` as that would always block the - // optimization. Instead, we consider removing candidates manually. - let Some(rhs) = rhs.place() else { - self.write_info.add_place(lhs); - return; - }; - // Find out which candidate pair we should skip, if any - let Some((src, dest)) = places_to_candidate_pair(lhs, rhs, self.body) else { - self.write_info.add_place(lhs); - return; - }; - self.candidates.remove_candidates_if( - lhs.local, - |other| { - // Check if this is the candidate pair that should not be removed - if (lhs.local == src && other == dest) || (lhs.local == dest && other == src) { - return false; - } - // Otherwise, do the "standard" thing - self.live.contains(other) - }, - self.at, - ) - } } /// Describes where a statement/terminator writes to #[derive(Default, Debug)] struct WriteInfo { writes: Vec<Local>, + /// If this pair of locals is a candidate pair, completely skip processing it during this + /// statement. All other candidates are unaffected. + skip_pair: Option<(Local, Local)>, } impl WriteInfo { - fn for_statement<'tcx>(&mut self, statement: &StatementKind<'tcx>) { + fn for_statement<'tcx>(&mut self, statement: &StatementKind<'tcx>, body: &Body<'tcx>) { + self.reset(); match statement { StatementKind::Assign(box (lhs, rhs)) => { self.add_place(*lhs); match rhs { - Rvalue::Use(op) | Rvalue::Repeat(op, _) => { + Rvalue::Use(op) => { + self.add_operand(op); + self.consider_skipping_for_assign_use(*lhs, op, body); + } + Rvalue::Repeat(op, _) => { self.add_operand(op); } Rvalue::Cast(_, op, _) @@ -586,8 +587,22 @@ impl WriteInfo { } } + fn consider_skipping_for_assign_use<'tcx>( + &mut self, + lhs: Place<'tcx>, + rhs: &Operand<'tcx>, + body: &Body<'tcx>, + ) { + let Some(rhs) = rhs.place() else { + return + }; + if let Some(pair) = places_to_candidate_pair(lhs, rhs, body) { + self.skip_pair = Some(pair); + } + } + fn for_terminator<'tcx>(&mut self, terminator: &TerminatorKind<'tcx>) { - self.writes.clear(); + self.reset(); match terminator { TerminatorKind::SwitchInt { discr: op, .. } | TerminatorKind::Assert { cond: op, .. } => { @@ -643,7 +658,7 @@ impl WriteInfo { } } - fn add_place<'tcx>(&mut self, place: Place<'tcx>) { + fn add_place(&mut self, place: Place<'_>) { self.writes.push(place.local); } @@ -657,15 +672,16 @@ impl WriteInfo { Operand::Copy(_) | Operand::Constant(_) => (), } } + + fn reset(&mut self) { + self.writes.clear(); + self.skip_pair = None; + } } ///////////////////////////////////////////////////// // Candidate accumulation -fn is_constant<'tcx>(place: Place<'tcx>) -> bool { - place.projection.iter().all(|p| !matches!(p, ProjectionElem::Deref | ProjectionElem::Index(_))) -} - /// If the pair of places is being considered for merging, returns the candidate which would be /// merged in order to accomplish this. /// @@ -741,10 +757,6 @@ impl<'tcx> Visitor<'tcx> for FindAssignments<'_, '_, 'tcx> { Rvalue::Use(Operand::Copy(rhs) | Operand::Move(rhs)), )) = &statement.kind { - if !is_constant(*lhs) || !is_constant(*rhs) { - return; - } - let Some((src, dest)) = places_to_candidate_pair(*lhs, *rhs, self.body) else { return; }; |