summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_codegen_ssa/src/mir
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-17 12:02:58 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-17 12:02:58 +0000
commit698f8c2f01ea549d77d7dc3338a12e04c11057b9 (patch)
tree173a775858bd501c378080a10dca74132f05bc50 /compiler/rustc_codegen_ssa/src/mir
parentInitial commit. (diff)
downloadrustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.tar.xz
rustc-698f8c2f01ea549d77d7dc3338a12e04c11057b9.zip
Adding upstream version 1.64.0+dfsg1.upstream/1.64.0+dfsg1
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'compiler/rustc_codegen_ssa/src/mir')
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/analyze.rs368
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/block.rs1654
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/constant.rs90
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs55
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/debuginfo.rs418
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/intrinsic.rs636
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/mod.rs410
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/operand.rs461
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/place.rs549
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/rvalue.rs729
-rw-r--r--compiler/rustc_codegen_ssa/src/mir/statement.rs102
11 files changed, 5472 insertions, 0 deletions
diff --git a/compiler/rustc_codegen_ssa/src/mir/analyze.rs b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
new file mode 100644
index 000000000..24da48ead
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/analyze.rs
@@ -0,0 +1,368 @@
+//! An analysis to determine which locals require allocas and
+//! which do not.
+
+use super::FunctionCx;
+use crate::traits::*;
+use rustc_data_structures::graph::dominators::Dominators;
+use rustc_index::bit_set::BitSet;
+use rustc_index::vec::IndexVec;
+use rustc_middle::mir::traversal;
+use rustc_middle::mir::visit::{MutatingUseContext, NonMutatingUseContext, PlaceContext, Visitor};
+use rustc_middle::mir::{self, Location, TerminatorKind};
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+
+pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ fx: &FunctionCx<'a, 'tcx, Bx>,
+) -> BitSet<mir::Local> {
+ let mir = fx.mir;
+ let dominators = mir.basic_blocks.dominators();
+ let locals = mir
+ .local_decls
+ .iter()
+ .map(|decl| {
+ let ty = fx.monomorphize(decl.ty);
+ let layout = fx.cx.spanned_layout_of(ty, decl.source_info.span);
+ if layout.is_zst() {
+ LocalKind::ZST
+ } else if fx.cx.is_backend_immediate(layout) || fx.cx.is_backend_scalar_pair(layout) {
+ LocalKind::Unused
+ } else {
+ LocalKind::Memory
+ }
+ })
+ .collect();
+
+ let mut analyzer = LocalAnalyzer { fx, dominators, locals };
+
+ // Arguments get assigned to by means of the function being called
+ for arg in mir.args_iter() {
+ analyzer.assign(arg, mir::START_BLOCK.start_location());
+ }
+
+ // If there exists a local definition that dominates all uses of that local,
+ // the definition should be visited first. Traverse blocks in an order that
+ // is a topological sort of dominance partial order.
+ for (bb, data) in traversal::reverse_postorder(&mir) {
+ analyzer.visit_basic_block_data(bb, data);
+ }
+
+ let mut non_ssa_locals = BitSet::new_empty(analyzer.locals.len());
+ for (local, kind) in analyzer.locals.iter_enumerated() {
+ if matches!(kind, LocalKind::Memory) {
+ non_ssa_locals.insert(local);
+ }
+ }
+
+ non_ssa_locals
+}
+
+#[derive(Copy, Clone, PartialEq, Eq)]
+enum LocalKind {
+ ZST,
+ /// A local that requires an alloca.
+ Memory,
+ /// A scalar or a scalar pair local that is neither defined nor used.
+ Unused,
+ /// A scalar or a scalar pair local with a single definition that dominates all uses.
+ SSA(mir::Location),
+}
+
+struct LocalAnalyzer<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
+ fx: &'mir FunctionCx<'a, 'tcx, Bx>,
+ dominators: Dominators<mir::BasicBlock>,
+ locals: IndexVec<mir::Local, LocalKind>,
+}
+
+impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> LocalAnalyzer<'mir, 'a, 'tcx, Bx> {
+ fn assign(&mut self, local: mir::Local, location: Location) {
+ let kind = &mut self.locals[local];
+ match *kind {
+ LocalKind::ZST => {}
+ LocalKind::Memory => {}
+ LocalKind::Unused => {
+ *kind = LocalKind::SSA(location);
+ }
+ LocalKind::SSA(_) => {
+ *kind = LocalKind::Memory;
+ }
+ }
+ }
+
+ fn process_place(
+ &mut self,
+ place_ref: &mir::PlaceRef<'tcx>,
+ context: PlaceContext,
+ location: Location,
+ ) {
+ let cx = self.fx.cx;
+
+ if let Some((place_base, elem)) = place_ref.last_projection() {
+ let mut base_context = if context.is_mutating_use() {
+ PlaceContext::MutatingUse(MutatingUseContext::Projection)
+ } else {
+ PlaceContext::NonMutatingUse(NonMutatingUseContext::Projection)
+ };
+
+ // Allow uses of projections that are ZSTs or from scalar fields.
+ let is_consume = matches!(
+ context,
+ PlaceContext::NonMutatingUse(
+ NonMutatingUseContext::Copy | NonMutatingUseContext::Move,
+ )
+ );
+ if is_consume {
+ let base_ty = place_base.ty(self.fx.mir, cx.tcx());
+ let base_ty = self.fx.monomorphize(base_ty);
+
+ // ZSTs don't require any actual memory access.
+ let elem_ty = base_ty.projection_ty(cx.tcx(), self.fx.monomorphize(elem)).ty;
+ let span = self.fx.mir.local_decls[place_ref.local].source_info.span;
+ if cx.spanned_layout_of(elem_ty, span).is_zst() {
+ return;
+ }
+
+ if let mir::ProjectionElem::Field(..) = elem {
+ let layout = cx.spanned_layout_of(base_ty.ty, span);
+ if cx.is_backend_immediate(layout) || cx.is_backend_scalar_pair(layout) {
+ // Recurse with the same context, instead of `Projection`,
+ // potentially stopping at non-operand projections,
+ // which would trigger `not_ssa` on locals.
+ base_context = context;
+ }
+ }
+ }
+
+ if let mir::ProjectionElem::Deref = elem {
+ // Deref projections typically only read the pointer.
+ base_context = PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy);
+ }
+
+ self.process_place(&place_base, base_context, location);
+ // HACK(eddyb) this emulates the old `visit_projection_elem`, this
+ // entire `visit_place`-like `process_place` method should be rewritten,
+ // now that we have moved to the "slice of projections" representation.
+ if let mir::ProjectionElem::Index(local) = elem {
+ self.visit_local(
+ local,
+ PlaceContext::NonMutatingUse(NonMutatingUseContext::Copy),
+ location,
+ );
+ }
+ } else {
+ self.visit_local(place_ref.local, context, location);
+ }
+ }
+}
+
+impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
+ for LocalAnalyzer<'mir, 'a, 'tcx, Bx>
+{
+ fn visit_assign(
+ &mut self,
+ place: &mir::Place<'tcx>,
+ rvalue: &mir::Rvalue<'tcx>,
+ location: Location,
+ ) {
+ debug!("visit_assign(place={:?}, rvalue={:?})", place, rvalue);
+
+ if let Some(local) = place.as_local() {
+ self.assign(local, location);
+ if self.locals[local] != LocalKind::Memory {
+ let decl_span = self.fx.mir.local_decls[local].source_info.span;
+ if !self.fx.rvalue_creates_operand(rvalue, decl_span) {
+ self.locals[local] = LocalKind::Memory;
+ }
+ }
+ } else {
+ self.visit_place(place, PlaceContext::MutatingUse(MutatingUseContext::Store), location);
+ }
+
+ self.visit_rvalue(rvalue, location);
+ }
+
+ fn visit_place(&mut self, place: &mir::Place<'tcx>, context: PlaceContext, location: Location) {
+ debug!("visit_place(place={:?}, context={:?})", place, context);
+ self.process_place(&place.as_ref(), context, location);
+ }
+
+ fn visit_local(&mut self, local: mir::Local, context: PlaceContext, location: Location) {
+ match context {
+ PlaceContext::MutatingUse(MutatingUseContext::Call)
+ | PlaceContext::MutatingUse(MutatingUseContext::Yield) => {
+ self.assign(local, location);
+ }
+
+ PlaceContext::NonUse(_) | PlaceContext::MutatingUse(MutatingUseContext::Retag) => {}
+
+ PlaceContext::NonMutatingUse(
+ NonMutatingUseContext::Copy | NonMutatingUseContext::Move,
+ ) => match &mut self.locals[local] {
+ LocalKind::ZST => {}
+ LocalKind::Memory => {}
+ LocalKind::SSA(def) if def.dominates(location, &self.dominators) => {}
+ // Reads from uninitialized variables (e.g., in dead code, after
+ // optimizations) require locals to be in (uninitialized) memory.
+ // N.B., there can be uninitialized reads of a local visited after
+ // an assignment to that local, if they happen on disjoint paths.
+ kind @ (LocalKind::Unused | LocalKind::SSA(_)) => {
+ *kind = LocalKind::Memory;
+ }
+ },
+
+ PlaceContext::MutatingUse(
+ MutatingUseContext::Store
+ | MutatingUseContext::Deinit
+ | MutatingUseContext::SetDiscriminant
+ | MutatingUseContext::AsmOutput
+ | MutatingUseContext::Borrow
+ | MutatingUseContext::AddressOf
+ | MutatingUseContext::Projection,
+ )
+ | PlaceContext::NonMutatingUse(
+ NonMutatingUseContext::Inspect
+ | NonMutatingUseContext::SharedBorrow
+ | NonMutatingUseContext::UniqueBorrow
+ | NonMutatingUseContext::ShallowBorrow
+ | NonMutatingUseContext::AddressOf
+ | NonMutatingUseContext::Projection,
+ ) => {
+ self.locals[local] = LocalKind::Memory;
+ }
+
+ PlaceContext::MutatingUse(MutatingUseContext::Drop) => {
+ let kind = &mut self.locals[local];
+ if *kind != LocalKind::Memory {
+ let ty = self.fx.mir.local_decls[local].ty;
+ let ty = self.fx.monomorphize(ty);
+ if self.fx.cx.type_needs_drop(ty) {
+ // Only need the place if we're actually dropping it.
+ *kind = LocalKind::Memory;
+ }
+ }
+ }
+ }
+ }
+}
+
+#[derive(Copy, Clone, Debug, PartialEq, Eq)]
+pub enum CleanupKind {
+ NotCleanup,
+ Funclet,
+ Internal { funclet: mir::BasicBlock },
+}
+
+impl CleanupKind {
+ pub fn funclet_bb(self, for_bb: mir::BasicBlock) -> Option<mir::BasicBlock> {
+ match self {
+ CleanupKind::NotCleanup => None,
+ CleanupKind::Funclet => Some(for_bb),
+ CleanupKind::Internal { funclet } => Some(funclet),
+ }
+ }
+}
+
+pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKind> {
+ fn discover_masters<'tcx>(
+ result: &mut IndexVec<mir::BasicBlock, CleanupKind>,
+ mir: &mir::Body<'tcx>,
+ ) {
+ for (bb, data) in mir.basic_blocks().iter_enumerated() {
+ match data.terminator().kind {
+ TerminatorKind::Goto { .. }
+ | TerminatorKind::Resume
+ | TerminatorKind::Abort
+ | TerminatorKind::Return
+ | TerminatorKind::GeneratorDrop
+ | TerminatorKind::Unreachable
+ | TerminatorKind::SwitchInt { .. }
+ | TerminatorKind::Yield { .. }
+ | TerminatorKind::FalseEdge { .. }
+ | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ }
+ TerminatorKind::Call { cleanup: unwind, .. }
+ | TerminatorKind::InlineAsm { cleanup: unwind, .. }
+ | TerminatorKind::Assert { cleanup: unwind, .. }
+ | TerminatorKind::DropAndReplace { unwind, .. }
+ | TerminatorKind::Drop { unwind, .. } => {
+ if let Some(unwind) = unwind {
+ debug!(
+ "cleanup_kinds: {:?}/{:?} registering {:?} as funclet",
+ bb, data, unwind
+ );
+ result[unwind] = CleanupKind::Funclet;
+ }
+ }
+ }
+ }
+ }
+
+ fn propagate<'tcx>(result: &mut IndexVec<mir::BasicBlock, CleanupKind>, mir: &mir::Body<'tcx>) {
+ let mut funclet_succs = IndexVec::from_elem(None, mir.basic_blocks());
+
+ let mut set_successor = |funclet: mir::BasicBlock, succ| match funclet_succs[funclet] {
+ ref mut s @ None => {
+ debug!("set_successor: updating successor of {:?} to {:?}", funclet, succ);
+ *s = Some(succ);
+ }
+ Some(s) => {
+ if s != succ {
+ span_bug!(
+ mir.span,
+ "funclet {:?} has 2 parents - {:?} and {:?}",
+ funclet,
+ s,
+ succ
+ );
+ }
+ }
+ };
+
+ for (bb, data) in traversal::reverse_postorder(mir) {
+ let funclet = match result[bb] {
+ CleanupKind::NotCleanup => continue,
+ CleanupKind::Funclet => bb,
+ CleanupKind::Internal { funclet } => funclet,
+ };
+
+ debug!(
+ "cleanup_kinds: {:?}/{:?}/{:?} propagating funclet {:?}",
+ bb, data, result[bb], funclet
+ );
+
+ for succ in data.terminator().successors() {
+ let kind = result[succ];
+ debug!("cleanup_kinds: propagating {:?} to {:?}/{:?}", funclet, succ, kind);
+ match kind {
+ CleanupKind::NotCleanup => {
+ result[succ] = CleanupKind::Internal { funclet };
+ }
+ CleanupKind::Funclet => {
+ if funclet != succ {
+ set_successor(funclet, succ);
+ }
+ }
+ CleanupKind::Internal { funclet: succ_funclet } => {
+ if funclet != succ_funclet {
+ // `succ` has 2 different funclet going into it, so it must
+ // be a funclet by itself.
+
+ debug!(
+ "promoting {:?} to a funclet and updating {:?}",
+ succ, succ_funclet
+ );
+ result[succ] = CleanupKind::Funclet;
+ set_successor(succ_funclet, succ);
+ set_successor(funclet, succ);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ let mut result = IndexVec::from_elem(CleanupKind::NotCleanup, mir.basic_blocks());
+
+ discover_masters(&mut result, mir);
+ propagate(&mut result, mir);
+ debug!("cleanup_kinds: result={:?}", result);
+ result
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs
new file mode 100644
index 000000000..3eee58d9d
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/block.rs
@@ -0,0 +1,1654 @@
+use super::operand::OperandRef;
+use super::operand::OperandValue::{Immediate, Pair, Ref};
+use super::place::PlaceRef;
+use super::{FunctionCx, LocalRef};
+
+use crate::base;
+use crate::common::{self, IntPredicate};
+use crate::meth;
+use crate::traits::*;
+use crate::MemFlags;
+
+use rustc_ast as ast;
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
+use rustc_hir::lang_items::LangItem;
+use rustc_index::vec::Idx;
+use rustc_middle::mir::AssertKind;
+use rustc_middle::mir::{self, SwitchTargets};
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
+use rustc_middle::ty::{self, Instance, Ty, TypeVisitable};
+use rustc_span::source_map::Span;
+use rustc_span::{sym, Symbol};
+use rustc_symbol_mangling::typeid::typeid_for_fnabi;
+use rustc_target::abi::call::{ArgAbi, FnAbi, PassMode};
+use rustc_target::abi::{self, HasDataLayout, WrappingRange};
+use rustc_target::spec::abi::Abi;
+
+/// Used by `FunctionCx::codegen_terminator` for emitting common patterns
+/// e.g., creating a basic block, calling a function, etc.
+struct TerminatorCodegenHelper<'tcx> {
+ bb: mir::BasicBlock,
+ terminator: &'tcx mir::Terminator<'tcx>,
+ funclet_bb: Option<mir::BasicBlock>,
+}
+
+impl<'a, 'tcx> TerminatorCodegenHelper<'tcx> {
+ /// Returns the appropriate `Funclet` for the current funclet, if on MSVC,
+ /// either already previously cached, or newly created, by `landing_pad_for`.
+ fn funclet<'b, Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &'b mut FunctionCx<'a, 'tcx, Bx>,
+ ) -> Option<&'b Bx::Funclet> {
+ let funclet_bb = self.funclet_bb?;
+ if base::wants_msvc_seh(fx.cx.tcx().sess) {
+ // If `landing_pad_for` hasn't been called yet to create the `Funclet`,
+ // it has to be now. This may not seem necessary, as RPO should lead
+ // to all the unwind edges being visited (and so to `landing_pad_for`
+ // getting called for them), before building any of the blocks inside
+ // the funclet itself - however, if MIR contains edges that end up not
+ // being needed in the LLVM IR after monomorphization, the funclet may
+ // be unreachable, and we don't have yet a way to skip building it in
+ // such an eventuality (which may be a better solution than this).
+ if fx.funclets[funclet_bb].is_none() {
+ fx.landing_pad_for(funclet_bb);
+ }
+
+ Some(
+ fx.funclets[funclet_bb]
+ .as_ref()
+ .expect("landing_pad_for didn't also create funclets entry"),
+ )
+ } else {
+ None
+ }
+ }
+
+ fn lltarget<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ target: mir::BasicBlock,
+ ) -> (Bx::BasicBlock, bool) {
+ let span = self.terminator.source_info.span;
+ let lltarget = fx.llbb(target);
+ let target_funclet = fx.cleanup_kinds[target].funclet_bb(target);
+ match (self.funclet_bb, target_funclet) {
+ (None, None) => (lltarget, false),
+ (Some(f), Some(t_f)) if f == t_f || !base::wants_msvc_seh(fx.cx.tcx().sess) => {
+ (lltarget, false)
+ }
+ // jump *into* cleanup - need a landing pad if GNU, cleanup pad if MSVC
+ (None, Some(_)) => (fx.landing_pad_for(target), false),
+ (Some(_), None) => span_bug!(span, "{:?} - jump out of cleanup?", self.terminator),
+ (Some(_), Some(_)) => (fx.landing_pad_for(target), true),
+ }
+ }
+
+ /// Create a basic block.
+ fn llblock<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ target: mir::BasicBlock,
+ ) -> Bx::BasicBlock {
+ let (lltarget, is_cleanupret) = self.lltarget(fx, target);
+ if is_cleanupret {
+ // MSVC cross-funclet jump - need a trampoline
+
+ debug!("llblock: creating cleanup trampoline for {:?}", target);
+ let name = &format!("{:?}_cleanup_trampoline_{:?}", self.bb, target);
+ let trampoline = Bx::append_block(fx.cx, fx.llfn, name);
+ let mut trampoline_bx = Bx::build(fx.cx, trampoline);
+ trampoline_bx.cleanup_ret(self.funclet(fx).unwrap(), Some(lltarget));
+ trampoline
+ } else {
+ lltarget
+ }
+ }
+
+ fn funclet_br<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ bx: &mut Bx,
+ target: mir::BasicBlock,
+ ) {
+ let (lltarget, is_cleanupret) = self.lltarget(fx, target);
+ if is_cleanupret {
+ // micro-optimization: generate a `ret` rather than a jump
+ // to a trampoline.
+ bx.cleanup_ret(self.funclet(fx).unwrap(), Some(lltarget));
+ } else {
+ bx.br(lltarget);
+ }
+ }
+
+ /// Call `fn_ptr` of `fn_abi` with the arguments `llargs`, the optional
+ /// return destination `destination` and the cleanup function `cleanup`.
+ fn do_call<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ bx: &mut Bx,
+ fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
+ fn_ptr: Bx::Value,
+ llargs: &[Bx::Value],
+ destination: Option<(ReturnDest<'tcx, Bx::Value>, mir::BasicBlock)>,
+ cleanup: Option<mir::BasicBlock>,
+ copied_constant_arguments: &[PlaceRef<'tcx, <Bx as BackendTypes>::Value>],
+ ) {
+ // If there is a cleanup block and the function we're calling can unwind, then
+ // do an invoke, otherwise do a call.
+ let fn_ty = bx.fn_decl_backend_type(&fn_abi);
+
+ let unwind_block = if let Some(cleanup) = cleanup.filter(|_| fn_abi.can_unwind) {
+ Some(self.llblock(fx, cleanup))
+ } else if fx.mir[self.bb].is_cleanup
+ && fn_abi.can_unwind
+ && !base::wants_msvc_seh(fx.cx.tcx().sess)
+ {
+ // Exception must not propagate out of the execution of a cleanup (doing so
+ // can cause undefined behaviour). We insert a double unwind guard for
+ // functions that can potentially unwind to protect against this.
+ //
+ // This is not necessary for SEH which does not use successive unwinding
+ // like Itanium EH. EH frames in SEH are different from normal function
+ // frames and SEH will abort automatically if an exception tries to
+ // propagate out from cleanup.
+ Some(fx.double_unwind_guard())
+ } else {
+ None
+ };
+
+ if let Some(unwind_block) = unwind_block {
+ let ret_llbb = if let Some((_, target)) = destination {
+ fx.llbb(target)
+ } else {
+ fx.unreachable_block()
+ };
+ let invokeret =
+ bx.invoke(fn_ty, fn_ptr, &llargs, ret_llbb, unwind_block, self.funclet(fx));
+ bx.apply_attrs_callsite(&fn_abi, invokeret);
+ if fx.mir[self.bb].is_cleanup {
+ bx.do_not_inline(invokeret);
+ }
+
+ if let Some((ret_dest, target)) = destination {
+ bx.switch_to_block(fx.llbb(target));
+ fx.set_debug_loc(bx, self.terminator.source_info);
+ for tmp in copied_constant_arguments {
+ bx.lifetime_end(tmp.llval, tmp.layout.size);
+ }
+ fx.store_return(bx, ret_dest, &fn_abi.ret, invokeret);
+ }
+ } else {
+ let llret = bx.call(fn_ty, fn_ptr, &llargs, self.funclet(fx));
+ bx.apply_attrs_callsite(&fn_abi, llret);
+ if fx.mir[self.bb].is_cleanup {
+ // Cleanup is always the cold path. Don't inline
+ // drop glue. Also, when there is a deeply-nested
+ // struct, there are "symmetry" issues that cause
+ // exponential inlining - see issue #41696.
+ bx.do_not_inline(llret);
+ }
+
+ if let Some((ret_dest, target)) = destination {
+ for tmp in copied_constant_arguments {
+ bx.lifetime_end(tmp.llval, tmp.layout.size);
+ }
+ fx.store_return(bx, ret_dest, &fn_abi.ret, llret);
+ self.funclet_br(fx, bx, target);
+ } else {
+ bx.unreachable();
+ }
+ }
+ }
+
+ /// Generates inline assembly with optional `destination` and `cleanup`.
+ fn do_inlineasm<Bx: BuilderMethods<'a, 'tcx>>(
+ &self,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ bx: &mut Bx,
+ template: &[InlineAsmTemplatePiece],
+ operands: &[InlineAsmOperandRef<'tcx, Bx>],
+ options: InlineAsmOptions,
+ line_spans: &[Span],
+ destination: Option<mir::BasicBlock>,
+ cleanup: Option<mir::BasicBlock>,
+ instance: Instance<'_>,
+ ) {
+ if let Some(cleanup) = cleanup {
+ let ret_llbb = if let Some(target) = destination {
+ fx.llbb(target)
+ } else {
+ fx.unreachable_block()
+ };
+
+ bx.codegen_inline_asm(
+ template,
+ &operands,
+ options,
+ line_spans,
+ instance,
+ Some((ret_llbb, self.llblock(fx, cleanup), self.funclet(fx))),
+ );
+ } else {
+ bx.codegen_inline_asm(template, &operands, options, line_spans, instance, None);
+
+ if let Some(target) = destination {
+ self.funclet_br(fx, bx, target);
+ } else {
+ bx.unreachable();
+ }
+ }
+ }
+}
+
+/// Codegen implementations for some terminator variants.
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ /// Generates code for a `Resume` terminator.
+ fn codegen_resume_terminator(&mut self, helper: TerminatorCodegenHelper<'tcx>, mut bx: Bx) {
+ if let Some(funclet) = helper.funclet(self) {
+ bx.cleanup_ret(funclet, None);
+ } else {
+ let slot = self.get_personality_slot(&mut bx);
+ let lp0 = slot.project_field(&mut bx, 0);
+ let lp0 = bx.load_operand(lp0).immediate();
+ let lp1 = slot.project_field(&mut bx, 1);
+ let lp1 = bx.load_operand(lp1).immediate();
+ slot.storage_dead(&mut bx);
+
+ let mut lp = bx.const_undef(self.landing_pad_type());
+ lp = bx.insert_value(lp, lp0, 0);
+ lp = bx.insert_value(lp, lp1, 1);
+ bx.resume(lp);
+ }
+ }
+
+ fn codegen_switchint_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ discr: &mir::Operand<'tcx>,
+ switch_ty: Ty<'tcx>,
+ targets: &SwitchTargets,
+ ) {
+ let discr = self.codegen_operand(&mut bx, &discr);
+ // `switch_ty` is redundant, sanity-check that.
+ assert_eq!(discr.layout.ty, switch_ty);
+ let mut target_iter = targets.iter();
+ if target_iter.len() == 1 {
+ // If there are two targets (one conditional, one fallback), emit br instead of switch
+ let (test_value, target) = target_iter.next().unwrap();
+ let lltrue = helper.llblock(self, target);
+ let llfalse = helper.llblock(self, targets.otherwise());
+ if switch_ty == bx.tcx().types.bool {
+ // Don't generate trivial icmps when switching on bool
+ match test_value {
+ 0 => bx.cond_br(discr.immediate(), llfalse, lltrue),
+ 1 => bx.cond_br(discr.immediate(), lltrue, llfalse),
+ _ => bug!(),
+ }
+ } else {
+ let switch_llty = bx.immediate_backend_type(bx.layout_of(switch_ty));
+ let llval = bx.const_uint_big(switch_llty, test_value);
+ let cmp = bx.icmp(IntPredicate::IntEQ, discr.immediate(), llval);
+ bx.cond_br(cmp, lltrue, llfalse);
+ }
+ } else {
+ bx.switch(
+ discr.immediate(),
+ helper.llblock(self, targets.otherwise()),
+ target_iter.map(|(value, target)| (value, helper.llblock(self, target))),
+ );
+ }
+ }
+
+ fn codegen_return_terminator(&mut self, mut bx: Bx) {
+ // Call `va_end` if this is the definition of a C-variadic function.
+ if self.fn_abi.c_variadic {
+ // The `VaList` "spoofed" argument is just after all the real arguments.
+ let va_list_arg_idx = self.fn_abi.args.len();
+ match self.locals[mir::Local::new(1 + va_list_arg_idx)] {
+ LocalRef::Place(va_list) => {
+ bx.va_end(va_list.llval);
+ }
+ _ => bug!("C-variadic function must have a `VaList` place"),
+ }
+ }
+ if self.fn_abi.ret.layout.abi.is_uninhabited() {
+ // Functions with uninhabited return values are marked `noreturn`,
+ // so we should make sure that we never actually do.
+ // We play it safe by using a well-defined `abort`, but we could go for immediate UB
+ // if that turns out to be helpful.
+ bx.abort();
+ // `abort` does not terminate the block, so we still need to generate
+ // an `unreachable` terminator after it.
+ bx.unreachable();
+ return;
+ }
+ let llval = match self.fn_abi.ret.mode {
+ PassMode::Ignore | PassMode::Indirect { .. } => {
+ bx.ret_void();
+ return;
+ }
+
+ PassMode::Direct(_) | PassMode::Pair(..) => {
+ let op = self.codegen_consume(&mut bx, mir::Place::return_place().as_ref());
+ if let Ref(llval, _, align) = op.val {
+ bx.load(bx.backend_type(op.layout), llval, align)
+ } else {
+ op.immediate_or_packed_pair(&mut bx)
+ }
+ }
+
+ PassMode::Cast(cast_ty) => {
+ let op = match self.locals[mir::RETURN_PLACE] {
+ LocalRef::Operand(Some(op)) => op,
+ LocalRef::Operand(None) => bug!("use of return before def"),
+ LocalRef::Place(cg_place) => OperandRef {
+ val: Ref(cg_place.llval, None, cg_place.align),
+ layout: cg_place.layout,
+ },
+ LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
+ };
+ let llslot = match op.val {
+ Immediate(_) | Pair(..) => {
+ let scratch = PlaceRef::alloca(&mut bx, self.fn_abi.ret.layout);
+ op.val.store(&mut bx, scratch);
+ scratch.llval
+ }
+ Ref(llval, _, align) => {
+ assert_eq!(align, op.layout.align.abi, "return place is unaligned!");
+ llval
+ }
+ };
+ let ty = bx.cast_backend_type(&cast_ty);
+ let addr = bx.pointercast(llslot, bx.type_ptr_to(ty));
+ bx.load(ty, addr, self.fn_abi.ret.layout.align.abi)
+ }
+ };
+ bx.ret(llval);
+ }
+
+ fn codegen_drop_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ location: mir::Place<'tcx>,
+ target: mir::BasicBlock,
+ unwind: Option<mir::BasicBlock>,
+ ) {
+ let ty = location.ty(self.mir, bx.tcx()).ty;
+ let ty = self.monomorphize(ty);
+ let drop_fn = Instance::resolve_drop_in_place(bx.tcx(), ty);
+
+ if let ty::InstanceDef::DropGlue(_, None) = drop_fn.def {
+ // we don't actually need to drop anything.
+ helper.funclet_br(self, &mut bx, target);
+ return;
+ }
+
+ let place = self.codegen_place(&mut bx, location.as_ref());
+ let (args1, args2);
+ let mut args = if let Some(llextra) = place.llextra {
+ args2 = [place.llval, llextra];
+ &args2[..]
+ } else {
+ args1 = [place.llval];
+ &args1[..]
+ };
+ let (drop_fn, fn_abi) = match ty.kind() {
+ // FIXME(eddyb) perhaps move some of this logic into
+ // `Instance::resolve_drop_in_place`?
+ ty::Dynamic(..) => {
+ let virtual_drop = Instance {
+ def: ty::InstanceDef::Virtual(drop_fn.def_id(), 0),
+ substs: drop_fn.substs,
+ };
+ let fn_abi = bx.fn_abi_of_instance(virtual_drop, ty::List::empty());
+ let vtable = args[1];
+ args = &args[..1];
+ (
+ meth::VirtualIndex::from_index(ty::COMMON_VTABLE_ENTRIES_DROPINPLACE)
+ .get_fn(&mut bx, vtable, ty, &fn_abi),
+ fn_abi,
+ )
+ }
+ _ => (bx.get_fn_addr(drop_fn), bx.fn_abi_of_instance(drop_fn, ty::List::empty())),
+ };
+ helper.do_call(
+ self,
+ &mut bx,
+ fn_abi,
+ drop_fn,
+ args,
+ Some((ReturnDest::Nothing, target)),
+ unwind,
+ &[],
+ );
+ }
+
+ fn codegen_assert_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ terminator: &mir::Terminator<'tcx>,
+ cond: &mir::Operand<'tcx>,
+ expected: bool,
+ msg: &mir::AssertMessage<'tcx>,
+ target: mir::BasicBlock,
+ cleanup: Option<mir::BasicBlock>,
+ ) {
+ let span = terminator.source_info.span;
+ let cond = self.codegen_operand(&mut bx, cond).immediate();
+ let mut const_cond = bx.const_to_opt_u128(cond, false).map(|c| c == 1);
+
+ // This case can currently arise only from functions marked
+ // with #[rustc_inherit_overflow_checks] and inlined from
+ // another crate (mostly core::num generic/#[inline] fns),
+ // while the current crate doesn't use overflow checks.
+ // NOTE: Unlike binops, negation doesn't have its own
+ // checked operation, just a comparison with the minimum
+ // value, so we have to check for the assert message.
+ if !bx.check_overflow() {
+ if let AssertKind::OverflowNeg(_) = *msg {
+ const_cond = Some(expected);
+ }
+ }
+
+ // Don't codegen the panic block if success if known.
+ if const_cond == Some(expected) {
+ helper.funclet_br(self, &mut bx, target);
+ return;
+ }
+
+ // Pass the condition through llvm.expect for branch hinting.
+ let cond = bx.expect(cond, expected);
+
+ // Create the failure block and the conditional branch to it.
+ let lltarget = helper.llblock(self, target);
+ let panic_block = bx.append_sibling_block("panic");
+ if expected {
+ bx.cond_br(cond, lltarget, panic_block);
+ } else {
+ bx.cond_br(cond, panic_block, lltarget);
+ }
+
+ // After this point, bx is the block for the call to panic.
+ bx.switch_to_block(panic_block);
+ self.set_debug_loc(&mut bx, terminator.source_info);
+
+ // Get the location information.
+ let location = self.get_caller_location(&mut bx, terminator.source_info).immediate();
+
+ // Put together the arguments to the panic entry point.
+ let (lang_item, args) = match msg {
+ AssertKind::BoundsCheck { ref len, ref index } => {
+ let len = self.codegen_operand(&mut bx, len).immediate();
+ let index = self.codegen_operand(&mut bx, index).immediate();
+ // It's `fn panic_bounds_check(index: usize, len: usize)`,
+ // and `#[track_caller]` adds an implicit third argument.
+ (LangItem::PanicBoundsCheck, vec![index, len, location])
+ }
+ _ => {
+ let msg = bx.const_str(msg.description());
+ // It's `pub fn panic(expr: &str)`, with the wide reference being passed
+ // as two arguments, and `#[track_caller]` adds an implicit third argument.
+ (LangItem::Panic, vec![msg.0, msg.1, location])
+ }
+ };
+
+ let (fn_abi, llfn) = common::build_langcall(&bx, Some(span), lang_item);
+
+ // Codegen the actual panic invoke/call.
+ helper.do_call(self, &mut bx, fn_abi, llfn, &args, None, cleanup, &[]);
+ }
+
+ fn codegen_abort_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ terminator: &mir::Terminator<'tcx>,
+ ) {
+ let span = terminator.source_info.span;
+ self.set_debug_loc(&mut bx, terminator.source_info);
+
+ // Obtain the panic entry point.
+ let (fn_abi, llfn) = common::build_langcall(&bx, Some(span), LangItem::PanicNoUnwind);
+
+ // Codegen the actual panic invoke/call.
+ helper.do_call(self, &mut bx, fn_abi, llfn, &[], None, None, &[]);
+ }
+
+ /// Returns `true` if this is indeed a panic intrinsic and codegen is done.
+ fn codegen_panic_intrinsic(
+ &mut self,
+ helper: &TerminatorCodegenHelper<'tcx>,
+ bx: &mut Bx,
+ intrinsic: Option<Symbol>,
+ instance: Option<Instance<'tcx>>,
+ source_info: mir::SourceInfo,
+ target: Option<mir::BasicBlock>,
+ cleanup: Option<mir::BasicBlock>,
+ ) -> bool {
+ // Emit a panic or a no-op for `assert_*` intrinsics.
+ // These are intrinsics that compile to panics so that we can get a message
+ // which mentions the offending type, even from a const context.
+ #[derive(Debug, PartialEq)]
+ enum AssertIntrinsic {
+ Inhabited,
+ ZeroValid,
+ UninitValid,
+ }
+ let panic_intrinsic = intrinsic.and_then(|i| match i {
+ sym::assert_inhabited => Some(AssertIntrinsic::Inhabited),
+ sym::assert_zero_valid => Some(AssertIntrinsic::ZeroValid),
+ sym::assert_uninit_valid => Some(AssertIntrinsic::UninitValid),
+ _ => None,
+ });
+ if let Some(intrinsic) = panic_intrinsic {
+ use AssertIntrinsic::*;
+
+ let ty = instance.unwrap().substs.type_at(0);
+ let layout = bx.layout_of(ty);
+ let do_panic = match intrinsic {
+ Inhabited => layout.abi.is_uninhabited(),
+ ZeroValid => !bx.tcx().permits_zero_init(layout),
+ UninitValid => !bx.tcx().permits_uninit_init(layout),
+ };
+ if do_panic {
+ let msg_str = with_no_visible_paths!({
+ with_no_trimmed_paths!({
+ if layout.abi.is_uninhabited() {
+ // Use this error even for the other intrinsics as it is more precise.
+ format!("attempted to instantiate uninhabited type `{}`", ty)
+ } else if intrinsic == ZeroValid {
+ format!("attempted to zero-initialize type `{}`, which is invalid", ty)
+ } else {
+ format!(
+ "attempted to leave type `{}` uninitialized, which is invalid",
+ ty
+ )
+ }
+ })
+ });
+ let msg = bx.const_str(&msg_str);
+ let location = self.get_caller_location(bx, source_info).immediate();
+
+ // Obtain the panic entry point.
+ let (fn_abi, llfn) =
+ common::build_langcall(bx, Some(source_info.span), LangItem::Panic);
+
+ // Codegen the actual panic invoke/call.
+ helper.do_call(
+ self,
+ bx,
+ fn_abi,
+ llfn,
+ &[msg.0, msg.1, location],
+ target.as_ref().map(|bb| (ReturnDest::Nothing, *bb)),
+ cleanup,
+ &[],
+ );
+ } else {
+ // a NOP
+ let target = target.unwrap();
+ helper.funclet_br(self, bx, target)
+ }
+ true
+ } else {
+ false
+ }
+ }
+
+ fn codegen_call_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ terminator: &mir::Terminator<'tcx>,
+ func: &mir::Operand<'tcx>,
+ args: &[mir::Operand<'tcx>],
+ destination: mir::Place<'tcx>,
+ target: Option<mir::BasicBlock>,
+ cleanup: Option<mir::BasicBlock>,
+ fn_span: Span,
+ ) {
+ let source_info = terminator.source_info;
+ let span = source_info.span;
+
+ // Create the callee. This is a fn ptr or zero-sized and hence a kind of scalar.
+ let callee = self.codegen_operand(&mut bx, func);
+
+ let (instance, mut llfn) = match *callee.layout.ty.kind() {
+ ty::FnDef(def_id, substs) => (
+ Some(
+ ty::Instance::resolve(bx.tcx(), ty::ParamEnv::reveal_all(), def_id, substs)
+ .unwrap()
+ .unwrap()
+ .polymorphize(bx.tcx()),
+ ),
+ None,
+ ),
+ ty::FnPtr(_) => (None, Some(callee.immediate())),
+ _ => bug!("{} is not callable", callee.layout.ty),
+ };
+ let def = instance.map(|i| i.def);
+
+ if let Some(ty::InstanceDef::DropGlue(_, None)) = def {
+ // Empty drop glue; a no-op.
+ let target = target.unwrap();
+ helper.funclet_br(self, &mut bx, target);
+ return;
+ }
+
+ // FIXME(eddyb) avoid computing this if possible, when `instance` is
+ // available - right now `sig` is only needed for getting the `abi`
+ // and figuring out how many extra args were passed to a C-variadic `fn`.
+ let sig = callee.layout.ty.fn_sig(bx.tcx());
+ let abi = sig.abi();
+
+ // Handle intrinsics old codegen wants Expr's for, ourselves.
+ let intrinsic = match def {
+ Some(ty::InstanceDef::Intrinsic(def_id)) => Some(bx.tcx().item_name(def_id)),
+ _ => None,
+ };
+
+ let extra_args = &args[sig.inputs().skip_binder().len()..];
+ let extra_args = bx.tcx().mk_type_list(extra_args.iter().map(|op_arg| {
+ let op_ty = op_arg.ty(self.mir, bx.tcx());
+ self.monomorphize(op_ty)
+ }));
+
+ let fn_abi = match instance {
+ Some(instance) => bx.fn_abi_of_instance(instance, extra_args),
+ None => bx.fn_abi_of_fn_ptr(sig, extra_args),
+ };
+
+ if intrinsic == Some(sym::transmute) {
+ if let Some(target) = target {
+ self.codegen_transmute(&mut bx, &args[0], destination);
+ helper.funclet_br(self, &mut bx, target);
+ } else {
+ // If we are trying to transmute to an uninhabited type,
+ // it is likely there is no allotted destination. In fact,
+ // transmuting to an uninhabited type is UB, which means
+ // we can do what we like. Here, we declare that transmuting
+ // into an uninhabited type is impossible, so anything following
+ // it must be unreachable.
+ assert_eq!(fn_abi.ret.layout.abi, abi::Abi::Uninhabited);
+ bx.unreachable();
+ }
+ return;
+ }
+
+ if self.codegen_panic_intrinsic(
+ &helper,
+ &mut bx,
+ intrinsic,
+ instance,
+ source_info,
+ target,
+ cleanup,
+ ) {
+ return;
+ }
+
+ // The arguments we'll be passing. Plus one to account for outptr, if used.
+ let arg_count = fn_abi.args.len() + fn_abi.ret.is_indirect() as usize;
+ let mut llargs = Vec::with_capacity(arg_count);
+
+ // Prepare the return value destination
+ let ret_dest = if target.is_some() {
+ let is_intrinsic = intrinsic.is_some();
+ self.make_return_dest(&mut bx, destination, &fn_abi.ret, &mut llargs, is_intrinsic)
+ } else {
+ ReturnDest::Nothing
+ };
+
+ if intrinsic == Some(sym::caller_location) {
+ if let Some(target) = target {
+ let location = self
+ .get_caller_location(&mut bx, mir::SourceInfo { span: fn_span, ..source_info });
+
+ if let ReturnDest::IndirectOperand(tmp, _) = ret_dest {
+ location.val.store(&mut bx, tmp);
+ }
+ self.store_return(&mut bx, ret_dest, &fn_abi.ret, location.immediate());
+ helper.funclet_br(self, &mut bx, target);
+ }
+ return;
+ }
+
+ match intrinsic {
+ None | Some(sym::drop_in_place) => {}
+ Some(sym::copy_nonoverlapping) => unreachable!(),
+ Some(intrinsic) => {
+ let dest = match ret_dest {
+ _ if fn_abi.ret.is_indirect() => llargs[0],
+ ReturnDest::Nothing => {
+ bx.const_undef(bx.type_ptr_to(bx.arg_memory_ty(&fn_abi.ret)))
+ }
+ ReturnDest::IndirectOperand(dst, _) | ReturnDest::Store(dst) => dst.llval,
+ ReturnDest::DirectOperand(_) => {
+ bug!("Cannot use direct operand with an intrinsic call")
+ }
+ };
+
+ let args: Vec<_> = args
+ .iter()
+ .enumerate()
+ .map(|(i, arg)| {
+ // The indices passed to simd_shuffle* in the
+ // third argument must be constant. This is
+ // checked by const-qualification, which also
+ // promotes any complex rvalues to constants.
+ if i == 2 && intrinsic.as_str().starts_with("simd_shuffle") {
+ if let mir::Operand::Constant(constant) = arg {
+ let c = self.eval_mir_constant(constant);
+ let (llval, ty) = self.simd_shuffle_indices(
+ &bx,
+ constant.span,
+ self.monomorphize(constant.ty()),
+ c,
+ );
+ return OperandRef {
+ val: Immediate(llval),
+ layout: bx.layout_of(ty),
+ };
+ } else {
+ span_bug!(span, "shuffle indices must be constant");
+ }
+ }
+
+ self.codegen_operand(&mut bx, arg)
+ })
+ .collect();
+
+ Self::codegen_intrinsic_call(
+ &mut bx,
+ *instance.as_ref().unwrap(),
+ &fn_abi,
+ &args,
+ dest,
+ span,
+ );
+
+ if let ReturnDest::IndirectOperand(dst, _) = ret_dest {
+ self.store_return(&mut bx, ret_dest, &fn_abi.ret, dst.llval);
+ }
+
+ if let Some(target) = target {
+ helper.funclet_br(self, &mut bx, target);
+ } else {
+ bx.unreachable();
+ }
+
+ return;
+ }
+ }
+
+ // Split the rust-call tupled arguments off.
+ let (first_args, untuple) = if abi == Abi::RustCall && !args.is_empty() {
+ let (tup, args) = args.split_last().unwrap();
+ (args, Some(tup))
+ } else {
+ (args, None)
+ };
+
+ let mut copied_constant_arguments = vec![];
+ 'make_args: for (i, arg) in first_args.iter().enumerate() {
+ let mut op = self.codegen_operand(&mut bx, arg);
+
+ if let (0, Some(ty::InstanceDef::Virtual(_, idx))) = (i, def) {
+ if let Pair(..) = op.val {
+ // In the case of Rc<Self>, we need to explicitly pass a
+ // *mut RcBox<Self> with a Scalar (not ScalarPair) ABI. This is a hack
+ // that is understood elsewhere in the compiler as a method on
+ // `dyn Trait`.
+ // To get a `*mut RcBox<Self>`, we just keep unwrapping newtypes until
+ // we get a value of a built-in pointer type
+ 'descend_newtypes: while !op.layout.ty.is_unsafe_ptr()
+ && !op.layout.ty.is_region_ptr()
+ {
+ for i in 0..op.layout.fields.count() {
+ let field = op.extract_field(&mut bx, i);
+ if !field.layout.is_zst() {
+ // we found the one non-zero-sized field that is allowed
+ // now find *its* non-zero-sized field, or stop if it's a
+ // pointer
+ op = field;
+ continue 'descend_newtypes;
+ }
+ }
+
+ span_bug!(span, "receiver has no non-zero-sized fields {:?}", op);
+ }
+
+ // now that we have `*dyn Trait` or `&dyn Trait`, split it up into its
+ // data pointer and vtable. Look up the method in the vtable, and pass
+ // the data pointer as the first argument
+ match op.val {
+ Pair(data_ptr, meta) => {
+ llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+ &mut bx,
+ meta,
+ op.layout.ty,
+ &fn_abi,
+ ));
+ llargs.push(data_ptr);
+ continue 'make_args;
+ }
+ other => bug!("expected a Pair, got {:?}", other),
+ }
+ } else if let Ref(data_ptr, Some(meta), _) = op.val {
+ // by-value dynamic dispatch
+ llfn = Some(meth::VirtualIndex::from_index(idx).get_fn(
+ &mut bx,
+ meta,
+ op.layout.ty,
+ &fn_abi,
+ ));
+ llargs.push(data_ptr);
+ continue;
+ } else {
+ span_bug!(span, "can't codegen a virtual call on {:?}", op);
+ }
+ }
+
+ // The callee needs to own the argument memory if we pass it
+ // by-ref, so make a local copy of non-immediate constants.
+ match (arg, op.val) {
+ (&mir::Operand::Copy(_), Ref(_, None, _))
+ | (&mir::Operand::Constant(_), Ref(_, None, _)) => {
+ let tmp = PlaceRef::alloca(&mut bx, op.layout);
+ bx.lifetime_start(tmp.llval, tmp.layout.size);
+ op.val.store(&mut bx, tmp);
+ op.val = Ref(tmp.llval, None, tmp.align);
+ copied_constant_arguments.push(tmp);
+ }
+ _ => {}
+ }
+
+ self.codegen_argument(&mut bx, op, &mut llargs, &fn_abi.args[i]);
+ }
+ let num_untupled = untuple.map(|tup| {
+ self.codegen_arguments_untupled(
+ &mut bx,
+ tup,
+ &mut llargs,
+ &fn_abi.args[first_args.len()..],
+ )
+ });
+
+ let needs_location =
+ instance.map_or(false, |i| i.def.requires_caller_location(self.cx.tcx()));
+ if needs_location {
+ let mir_args = if let Some(num_untupled) = num_untupled {
+ first_args.len() + num_untupled
+ } else {
+ args.len()
+ };
+ assert_eq!(
+ fn_abi.args.len(),
+ mir_args + 1,
+ "#[track_caller] fn's must have 1 more argument in their ABI than in their MIR: {:?} {:?} {:?}",
+ instance,
+ fn_span,
+ fn_abi,
+ );
+ let location =
+ self.get_caller_location(&mut bx, mir::SourceInfo { span: fn_span, ..source_info });
+ debug!(
+ "codegen_call_terminator({:?}): location={:?} (fn_span {:?})",
+ terminator, location, fn_span
+ );
+
+ let last_arg = fn_abi.args.last().unwrap();
+ self.codegen_argument(&mut bx, location, &mut llargs, last_arg);
+ }
+
+ let (is_indirect_call, fn_ptr) = match (llfn, instance) {
+ (Some(llfn), _) => (true, llfn),
+ (None, Some(instance)) => (false, bx.get_fn_addr(instance)),
+ _ => span_bug!(span, "no llfn for call"),
+ };
+
+ // For backends that support CFI using type membership (i.e., testing whether a given
+ // pointer is associated with a type identifier).
+ if bx.tcx().sess.is_sanitizer_cfi_enabled() && is_indirect_call {
+ // Emit type metadata and checks.
+ // FIXME(rcvalle): Add support for generalized identifiers.
+ // FIXME(rcvalle): Create distinct unnamed MDNodes for internal identifiers.
+ let typeid = typeid_for_fnabi(bx.tcx(), fn_abi);
+ let typeid_metadata = self.cx.typeid_metadata(typeid);
+
+ // Test whether the function pointer is associated with the type identifier.
+ let cond = bx.type_test(fn_ptr, typeid_metadata);
+ let bb_pass = bx.append_sibling_block("type_test.pass");
+ let bb_fail = bx.append_sibling_block("type_test.fail");
+ bx.cond_br(cond, bb_pass, bb_fail);
+
+ bx.switch_to_block(bb_pass);
+ helper.do_call(
+ self,
+ &mut bx,
+ fn_abi,
+ fn_ptr,
+ &llargs,
+ target.as_ref().map(|&target| (ret_dest, target)),
+ cleanup,
+ &copied_constant_arguments,
+ );
+
+ bx.switch_to_block(bb_fail);
+ bx.abort();
+ bx.unreachable();
+
+ return;
+ }
+
+ helper.do_call(
+ self,
+ &mut bx,
+ fn_abi,
+ fn_ptr,
+ &llargs,
+ target.as_ref().map(|&target| (ret_dest, target)),
+ cleanup,
+ &copied_constant_arguments,
+ );
+ }
+
+ fn codegen_asm_terminator(
+ &mut self,
+ helper: TerminatorCodegenHelper<'tcx>,
+ mut bx: Bx,
+ terminator: &mir::Terminator<'tcx>,
+ template: &[ast::InlineAsmTemplatePiece],
+ operands: &[mir::InlineAsmOperand<'tcx>],
+ options: ast::InlineAsmOptions,
+ line_spans: &[Span],
+ destination: Option<mir::BasicBlock>,
+ cleanup: Option<mir::BasicBlock>,
+ instance: Instance<'_>,
+ ) {
+ let span = terminator.source_info.span;
+
+ let operands: Vec<_> = operands
+ .iter()
+ .map(|op| match *op {
+ mir::InlineAsmOperand::In { reg, ref value } => {
+ let value = self.codegen_operand(&mut bx, value);
+ InlineAsmOperandRef::In { reg, value }
+ }
+ mir::InlineAsmOperand::Out { reg, late, ref place } => {
+ let place = place.map(|place| self.codegen_place(&mut bx, place.as_ref()));
+ InlineAsmOperandRef::Out { reg, late, place }
+ }
+ mir::InlineAsmOperand::InOut { reg, late, ref in_value, ref out_place } => {
+ let in_value = self.codegen_operand(&mut bx, in_value);
+ let out_place =
+ out_place.map(|out_place| self.codegen_place(&mut bx, out_place.as_ref()));
+ InlineAsmOperandRef::InOut { reg, late, in_value, out_place }
+ }
+ mir::InlineAsmOperand::Const { ref value } => {
+ let const_value = self
+ .eval_mir_constant(value)
+ .unwrap_or_else(|_| span_bug!(span, "asm const cannot be resolved"));
+ let string = common::asm_const_to_str(
+ bx.tcx(),
+ span,
+ const_value,
+ bx.layout_of(value.ty()),
+ );
+ InlineAsmOperandRef::Const { string }
+ }
+ mir::InlineAsmOperand::SymFn { ref value } => {
+ let literal = self.monomorphize(value.literal);
+ if let ty::FnDef(def_id, substs) = *literal.ty().kind() {
+ let instance = ty::Instance::resolve_for_fn_ptr(
+ bx.tcx(),
+ ty::ParamEnv::reveal_all(),
+ def_id,
+ substs,
+ )
+ .unwrap();
+ InlineAsmOperandRef::SymFn { instance }
+ } else {
+ span_bug!(span, "invalid type for asm sym (fn)");
+ }
+ }
+ mir::InlineAsmOperand::SymStatic { def_id } => {
+ InlineAsmOperandRef::SymStatic { def_id }
+ }
+ })
+ .collect();
+
+ helper.do_inlineasm(
+ self,
+ &mut bx,
+ template,
+ &operands,
+ options,
+ line_spans,
+ destination,
+ cleanup,
+ instance,
+ );
+ }
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn codegen_block(&mut self, bb: mir::BasicBlock) {
+ let llbb = self.llbb(bb);
+ let mut bx = Bx::build(self.cx, llbb);
+ let mir = self.mir;
+ let data = &mir[bb];
+
+ debug!("codegen_block({:?}={:?})", bb, data);
+
+ for statement in &data.statements {
+ bx = self.codegen_statement(bx, statement);
+ }
+
+ self.codegen_terminator(bx, bb, data.terminator());
+ }
+
+ fn codegen_terminator(
+ &mut self,
+ mut bx: Bx,
+ bb: mir::BasicBlock,
+ terminator: &'tcx mir::Terminator<'tcx>,
+ ) {
+ debug!("codegen_terminator: {:?}", terminator);
+
+ // Create the cleanup bundle, if needed.
+ let funclet_bb = self.cleanup_kinds[bb].funclet_bb(bb);
+ let helper = TerminatorCodegenHelper { bb, terminator, funclet_bb };
+
+ self.set_debug_loc(&mut bx, terminator.source_info);
+ match terminator.kind {
+ mir::TerminatorKind::Resume => self.codegen_resume_terminator(helper, bx),
+
+ mir::TerminatorKind::Abort => {
+ self.codegen_abort_terminator(helper, bx, terminator);
+ }
+
+ mir::TerminatorKind::Goto { target } => {
+ helper.funclet_br(self, &mut bx, target);
+ }
+
+ mir::TerminatorKind::SwitchInt { ref discr, switch_ty, ref targets } => {
+ self.codegen_switchint_terminator(helper, bx, discr, switch_ty, targets);
+ }
+
+ mir::TerminatorKind::Return => {
+ self.codegen_return_terminator(bx);
+ }
+
+ mir::TerminatorKind::Unreachable => {
+ bx.unreachable();
+ }
+
+ mir::TerminatorKind::Drop { place, target, unwind } => {
+ self.codegen_drop_terminator(helper, bx, place, target, unwind);
+ }
+
+ mir::TerminatorKind::Assert { ref cond, expected, ref msg, target, cleanup } => {
+ self.codegen_assert_terminator(
+ helper, bx, terminator, cond, expected, msg, target, cleanup,
+ );
+ }
+
+ mir::TerminatorKind::DropAndReplace { .. } => {
+ bug!("undesugared DropAndReplace in codegen: {:?}", terminator);
+ }
+
+ mir::TerminatorKind::Call {
+ ref func,
+ ref args,
+ destination,
+ target,
+ cleanup,
+ from_hir_call: _,
+ fn_span,
+ } => {
+ self.codegen_call_terminator(
+ helper,
+ bx,
+ terminator,
+ func,
+ args,
+ destination,
+ target,
+ cleanup,
+ fn_span,
+ );
+ }
+ mir::TerminatorKind::GeneratorDrop | mir::TerminatorKind::Yield { .. } => {
+ bug!("generator ops in codegen")
+ }
+ mir::TerminatorKind::FalseEdge { .. } | mir::TerminatorKind::FalseUnwind { .. } => {
+ bug!("borrowck false edges in codegen")
+ }
+
+ mir::TerminatorKind::InlineAsm {
+ template,
+ ref operands,
+ options,
+ line_spans,
+ destination,
+ cleanup,
+ } => {
+ self.codegen_asm_terminator(
+ helper,
+ bx,
+ terminator,
+ template,
+ operands,
+ options,
+ line_spans,
+ destination,
+ cleanup,
+ self.instance,
+ );
+ }
+ }
+ }
+
+ fn codegen_argument(
+ &mut self,
+ bx: &mut Bx,
+ op: OperandRef<'tcx, Bx::Value>,
+ llargs: &mut Vec<Bx::Value>,
+ arg: &ArgAbi<'tcx, Ty<'tcx>>,
+ ) {
+ // Fill padding with undef value, where applicable.
+ if let Some(ty) = arg.pad {
+ llargs.push(bx.const_undef(bx.reg_backend_type(&ty)))
+ }
+
+ if arg.is_ignore() {
+ return;
+ }
+
+ if let PassMode::Pair(..) = arg.mode {
+ match op.val {
+ Pair(a, b) => {
+ llargs.push(a);
+ llargs.push(b);
+ return;
+ }
+ _ => bug!("codegen_argument: {:?} invalid for pair argument", op),
+ }
+ } else if arg.is_unsized_indirect() {
+ match op.val {
+ Ref(a, Some(b), _) => {
+ llargs.push(a);
+ llargs.push(b);
+ return;
+ }
+ _ => bug!("codegen_argument: {:?} invalid for unsized indirect argument", op),
+ }
+ }
+
+ // Force by-ref if we have to load through a cast pointer.
+ let (mut llval, align, by_ref) = match op.val {
+ Immediate(_) | Pair(..) => match arg.mode {
+ PassMode::Indirect { .. } | PassMode::Cast(_) => {
+ let scratch = PlaceRef::alloca(bx, arg.layout);
+ op.val.store(bx, scratch);
+ (scratch.llval, scratch.align, true)
+ }
+ _ => (op.immediate_or_packed_pair(bx), arg.layout.align.abi, false),
+ },
+ Ref(llval, _, align) => {
+ if arg.is_indirect() && align < arg.layout.align.abi {
+ // `foo(packed.large_field)`. We can't pass the (unaligned) field directly. I
+ // think that ATM (Rust 1.16) we only pass temporaries, but we shouldn't
+ // have scary latent bugs around.
+
+ let scratch = PlaceRef::alloca(bx, arg.layout);
+ base::memcpy_ty(
+ bx,
+ scratch.llval,
+ scratch.align,
+ llval,
+ align,
+ op.layout,
+ MemFlags::empty(),
+ );
+ (scratch.llval, scratch.align, true)
+ } else {
+ (llval, align, true)
+ }
+ }
+ };
+
+ if by_ref && !arg.is_indirect() {
+ // Have to load the argument, maybe while casting it.
+ if let PassMode::Cast(ty) = arg.mode {
+ let llty = bx.cast_backend_type(&ty);
+ let addr = bx.pointercast(llval, bx.type_ptr_to(llty));
+ llval = bx.load(llty, addr, align.min(arg.layout.align.abi));
+ } else {
+ // We can't use `PlaceRef::load` here because the argument
+ // may have a type we don't treat as immediate, but the ABI
+ // used for this call is passing it by-value. In that case,
+ // the load would just produce `OperandValue::Ref` instead
+ // of the `OperandValue::Immediate` we need for the call.
+ llval = bx.load(bx.backend_type(arg.layout), llval, align);
+ if let abi::Abi::Scalar(scalar) = arg.layout.abi {
+ if scalar.is_bool() {
+ bx.range_metadata(llval, WrappingRange { start: 0, end: 1 });
+ }
+ }
+ // We store bools as `i8` so we need to truncate to `i1`.
+ llval = bx.to_immediate(llval, arg.layout);
+ }
+ }
+
+ llargs.push(llval);
+ }
+
+ fn codegen_arguments_untupled(
+ &mut self,
+ bx: &mut Bx,
+ operand: &mir::Operand<'tcx>,
+ llargs: &mut Vec<Bx::Value>,
+ args: &[ArgAbi<'tcx, Ty<'tcx>>],
+ ) -> usize {
+ let tuple = self.codegen_operand(bx, operand);
+
+ // Handle both by-ref and immediate tuples.
+ if let Ref(llval, None, align) = tuple.val {
+ let tuple_ptr = PlaceRef::new_sized_aligned(llval, tuple.layout, align);
+ for i in 0..tuple.layout.fields.count() {
+ let field_ptr = tuple_ptr.project_field(bx, i);
+ let field = bx.load_operand(field_ptr);
+ self.codegen_argument(bx, field, llargs, &args[i]);
+ }
+ } else if let Ref(_, Some(_), _) = tuple.val {
+ bug!("closure arguments must be sized")
+ } else {
+ // If the tuple is immediate, the elements are as well.
+ for i in 0..tuple.layout.fields.count() {
+ let op = tuple.extract_field(bx, i);
+ self.codegen_argument(bx, op, llargs, &args[i]);
+ }
+ }
+ tuple.layout.fields.count()
+ }
+
+ fn get_caller_location(
+ &mut self,
+ bx: &mut Bx,
+ mut source_info: mir::SourceInfo,
+ ) -> OperandRef<'tcx, Bx::Value> {
+ let tcx = bx.tcx();
+
+ let mut span_to_caller_location = |span: Span| {
+ let topmost = span.ctxt().outer_expn().expansion_cause().unwrap_or(span);
+ let caller = tcx.sess.source_map().lookup_char_pos(topmost.lo());
+ let const_loc = tcx.const_caller_location((
+ Symbol::intern(&caller.file.name.prefer_remapped().to_string_lossy()),
+ caller.line as u32,
+ caller.col_display as u32 + 1,
+ ));
+ OperandRef::from_const(bx, const_loc, bx.tcx().caller_location_ty())
+ };
+
+ // Walk up the `SourceScope`s, in case some of them are from MIR inlining.
+ // If so, the starting `source_info.span` is in the innermost inlined
+ // function, and will be replaced with outer callsite spans as long
+ // as the inlined functions were `#[track_caller]`.
+ loop {
+ let scope_data = &self.mir.source_scopes[source_info.scope];
+
+ if let Some((callee, callsite_span)) = scope_data.inlined {
+ // Stop inside the most nested non-`#[track_caller]` function,
+ // before ever reaching its caller (which is irrelevant).
+ if !callee.def.requires_caller_location(tcx) {
+ return span_to_caller_location(source_info.span);
+ }
+ source_info.span = callsite_span;
+ }
+
+ // Skip past all of the parents with `inlined: None`.
+ match scope_data.inlined_parent_scope {
+ Some(parent) => source_info.scope = parent,
+ None => break,
+ }
+ }
+
+ // No inlined `SourceScope`s, or all of them were `#[track_caller]`.
+ self.caller_location.unwrap_or_else(|| span_to_caller_location(source_info.span))
+ }
+
+ fn get_personality_slot(&mut self, bx: &mut Bx) -> PlaceRef<'tcx, Bx::Value> {
+ let cx = bx.cx();
+ if let Some(slot) = self.personality_slot {
+ slot
+ } else {
+ let layout = cx.layout_of(
+ cx.tcx().intern_tup(&[cx.tcx().mk_mut_ptr(cx.tcx().types.u8), cx.tcx().types.i32]),
+ );
+ let slot = PlaceRef::alloca(bx, layout);
+ self.personality_slot = Some(slot);
+ slot
+ }
+ }
+
+ /// Returns the landing/cleanup pad wrapper around the given basic block.
+ // FIXME(eddyb) rename this to `eh_pad_for`.
+ fn landing_pad_for(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock {
+ if let Some(landing_pad) = self.landing_pads[bb] {
+ return landing_pad;
+ }
+
+ let landing_pad = self.landing_pad_for_uncached(bb);
+ self.landing_pads[bb] = Some(landing_pad);
+ landing_pad
+ }
+
+ // FIXME(eddyb) rename this to `eh_pad_for_uncached`.
+ fn landing_pad_for_uncached(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock {
+ let llbb = self.llbb(bb);
+ if base::wants_msvc_seh(self.cx.sess()) {
+ let funclet;
+ let ret_llbb;
+ match self.mir[bb].terminator.as_ref().map(|t| &t.kind) {
+ // This is a basic block that we're aborting the program for,
+ // notably in an `extern` function. These basic blocks are inserted
+ // so that we assert that `extern` functions do indeed not panic,
+ // and if they do we abort the process.
+ //
+ // On MSVC these are tricky though (where we're doing funclets). If
+ // we were to do a cleanuppad (like below) the normal functions like
+ // `longjmp` would trigger the abort logic, terminating the
+ // program. Instead we insert the equivalent of `catch(...)` for C++
+ // which magically doesn't trigger when `longjmp` files over this
+ // frame.
+ //
+ // Lots more discussion can be found on #48251 but this codegen is
+ // modeled after clang's for:
+ //
+ // try {
+ // foo();
+ // } catch (...) {
+ // bar();
+ // }
+ Some(&mir::TerminatorKind::Abort) => {
+ let cs_bb =
+ Bx::append_block(self.cx, self.llfn, &format!("cs_funclet{:?}", bb));
+ let cp_bb =
+ Bx::append_block(self.cx, self.llfn, &format!("cp_funclet{:?}", bb));
+ ret_llbb = cs_bb;
+
+ let mut cs_bx = Bx::build(self.cx, cs_bb);
+ let cs = cs_bx.catch_switch(None, None, &[cp_bb]);
+
+ // The "null" here is actually a RTTI type descriptor for the
+ // C++ personality function, but `catch (...)` has no type so
+ // it's null. The 64 here is actually a bitfield which
+ // represents that this is a catch-all block.
+ let mut cp_bx = Bx::build(self.cx, cp_bb);
+ let null = cp_bx.const_null(
+ cp_bx.type_i8p_ext(cp_bx.cx().data_layout().instruction_address_space),
+ );
+ let sixty_four = cp_bx.const_i32(64);
+ funclet = cp_bx.catch_pad(cs, &[null, sixty_four, null]);
+ cp_bx.br(llbb);
+ }
+ _ => {
+ let cleanup_bb =
+ Bx::append_block(self.cx, self.llfn, &format!("funclet_{:?}", bb));
+ ret_llbb = cleanup_bb;
+ let mut cleanup_bx = Bx::build(self.cx, cleanup_bb);
+ funclet = cleanup_bx.cleanup_pad(None, &[]);
+ cleanup_bx.br(llbb);
+ }
+ }
+ self.funclets[bb] = Some(funclet);
+ ret_llbb
+ } else {
+ let bb = Bx::append_block(self.cx, self.llfn, "cleanup");
+ let mut bx = Bx::build(self.cx, bb);
+
+ let llpersonality = self.cx.eh_personality();
+ let llretty = self.landing_pad_type();
+ let lp = bx.cleanup_landing_pad(llretty, llpersonality);
+
+ let slot = self.get_personality_slot(&mut bx);
+ slot.storage_live(&mut bx);
+ Pair(bx.extract_value(lp, 0), bx.extract_value(lp, 1)).store(&mut bx, slot);
+
+ bx.br(llbb);
+ bx.llbb()
+ }
+ }
+
+ fn landing_pad_type(&self) -> Bx::Type {
+ let cx = self.cx;
+ cx.type_struct(&[cx.type_i8p(), cx.type_i32()], false)
+ }
+
+ fn unreachable_block(&mut self) -> Bx::BasicBlock {
+ self.unreachable_block.unwrap_or_else(|| {
+ let llbb = Bx::append_block(self.cx, self.llfn, "unreachable");
+ let mut bx = Bx::build(self.cx, llbb);
+ bx.unreachable();
+ self.unreachable_block = Some(llbb);
+ llbb
+ })
+ }
+
+ fn double_unwind_guard(&mut self) -> Bx::BasicBlock {
+ self.double_unwind_guard.unwrap_or_else(|| {
+ assert!(!base::wants_msvc_seh(self.cx.sess()));
+
+ let llbb = Bx::append_block(self.cx, self.llfn, "abort");
+ let mut bx = Bx::build(self.cx, llbb);
+ self.set_debug_loc(&mut bx, mir::SourceInfo::outermost(self.mir.span));
+
+ let llpersonality = self.cx.eh_personality();
+ let llretty = self.landing_pad_type();
+ bx.cleanup_landing_pad(llretty, llpersonality);
+
+ let (fn_abi, fn_ptr) = common::build_langcall(&bx, None, LangItem::PanicNoUnwind);
+ let fn_ty = bx.fn_decl_backend_type(&fn_abi);
+
+ let llret = bx.call(fn_ty, fn_ptr, &[], None);
+ bx.apply_attrs_callsite(&fn_abi, llret);
+ bx.do_not_inline(llret);
+
+ bx.unreachable();
+
+ self.double_unwind_guard = Some(llbb);
+ llbb
+ })
+ }
+
+ /// Get the backend `BasicBlock` for a MIR `BasicBlock`, either already
+ /// cached in `self.cached_llbbs`, or created on demand (and cached).
+ // FIXME(eddyb) rename `llbb` and other `ll`-prefixed things to use a
+ // more backend-agnostic prefix such as `cg` (i.e. this would be `cgbb`).
+ pub fn llbb(&mut self, bb: mir::BasicBlock) -> Bx::BasicBlock {
+ self.cached_llbbs[bb].unwrap_or_else(|| {
+ // FIXME(eddyb) only name the block if `fewer_names` is `false`.
+ let llbb = Bx::append_block(self.cx, self.llfn, &format!("{:?}", bb));
+ self.cached_llbbs[bb] = Some(llbb);
+ llbb
+ })
+ }
+
+ fn make_return_dest(
+ &mut self,
+ bx: &mut Bx,
+ dest: mir::Place<'tcx>,
+ fn_ret: &ArgAbi<'tcx, Ty<'tcx>>,
+ llargs: &mut Vec<Bx::Value>,
+ is_intrinsic: bool,
+ ) -> ReturnDest<'tcx, Bx::Value> {
+ // If the return is ignored, we can just return a do-nothing `ReturnDest`.
+ if fn_ret.is_ignore() {
+ return ReturnDest::Nothing;
+ }
+ let dest = if let Some(index) = dest.as_local() {
+ match self.locals[index] {
+ LocalRef::Place(dest) => dest,
+ LocalRef::UnsizedPlace(_) => bug!("return type must be sized"),
+ LocalRef::Operand(None) => {
+ // Handle temporary places, specifically `Operand` ones, as
+ // they don't have `alloca`s.
+ return if fn_ret.is_indirect() {
+ // Odd, but possible, case, we have an operand temporary,
+ // but the calling convention has an indirect return.
+ let tmp = PlaceRef::alloca(bx, fn_ret.layout);
+ tmp.storage_live(bx);
+ llargs.push(tmp.llval);
+ ReturnDest::IndirectOperand(tmp, index)
+ } else if is_intrinsic {
+ // Currently, intrinsics always need a location to store
+ // the result, so we create a temporary `alloca` for the
+ // result.
+ let tmp = PlaceRef::alloca(bx, fn_ret.layout);
+ tmp.storage_live(bx);
+ ReturnDest::IndirectOperand(tmp, index)
+ } else {
+ ReturnDest::DirectOperand(index)
+ };
+ }
+ LocalRef::Operand(Some(_)) => {
+ bug!("place local already assigned to");
+ }
+ }
+ } else {
+ self.codegen_place(
+ bx,
+ mir::PlaceRef { local: dest.local, projection: &dest.projection },
+ )
+ };
+ if fn_ret.is_indirect() {
+ if dest.align < dest.layout.align.abi {
+ // Currently, MIR code generation does not create calls
+ // that store directly to fields of packed structs (in
+ // fact, the calls it creates write only to temps).
+ //
+ // If someone changes that, please update this code path
+ // to create a temporary.
+ span_bug!(self.mir.span, "can't directly store to unaligned value");
+ }
+ llargs.push(dest.llval);
+ ReturnDest::Nothing
+ } else {
+ ReturnDest::Store(dest)
+ }
+ }
+
+ fn codegen_transmute(&mut self, bx: &mut Bx, src: &mir::Operand<'tcx>, dst: mir::Place<'tcx>) {
+ if let Some(index) = dst.as_local() {
+ match self.locals[index] {
+ LocalRef::Place(place) => self.codegen_transmute_into(bx, src, place),
+ LocalRef::UnsizedPlace(_) => bug!("transmute must not involve unsized locals"),
+ LocalRef::Operand(None) => {
+ let dst_layout = bx.layout_of(self.monomorphized_place_ty(dst.as_ref()));
+ assert!(!dst_layout.ty.has_erasable_regions());
+ let place = PlaceRef::alloca(bx, dst_layout);
+ place.storage_live(bx);
+ self.codegen_transmute_into(bx, src, place);
+ let op = bx.load_operand(place);
+ place.storage_dead(bx);
+ self.locals[index] = LocalRef::Operand(Some(op));
+ self.debug_introduce_local(bx, index);
+ }
+ LocalRef::Operand(Some(op)) => {
+ assert!(op.layout.is_zst(), "assigning to initialized SSAtemp");
+ }
+ }
+ } else {
+ let dst = self.codegen_place(bx, dst.as_ref());
+ self.codegen_transmute_into(bx, src, dst);
+ }
+ }
+
+ fn codegen_transmute_into(
+ &mut self,
+ bx: &mut Bx,
+ src: &mir::Operand<'tcx>,
+ dst: PlaceRef<'tcx, Bx::Value>,
+ ) {
+ let src = self.codegen_operand(bx, src);
+
+ // Special-case transmutes between scalars as simple bitcasts.
+ match (src.layout.abi, dst.layout.abi) {
+ (abi::Abi::Scalar(src_scalar), abi::Abi::Scalar(dst_scalar)) => {
+ // HACK(eddyb) LLVM doesn't like `bitcast`s between pointers and non-pointers.
+ if (src_scalar.primitive() == abi::Pointer)
+ == (dst_scalar.primitive() == abi::Pointer)
+ {
+ assert_eq!(src.layout.size, dst.layout.size);
+
+ // NOTE(eddyb) the `from_immediate` and `to_immediate_scalar`
+ // conversions allow handling `bool`s the same as `u8`s.
+ let src = bx.from_immediate(src.immediate());
+ let src_as_dst = bx.bitcast(src, bx.backend_type(dst.layout));
+ Immediate(bx.to_immediate_scalar(src_as_dst, dst_scalar)).store(bx, dst);
+ return;
+ }
+ }
+ _ => {}
+ }
+
+ let llty = bx.backend_type(src.layout);
+ let cast_ptr = bx.pointercast(dst.llval, bx.type_ptr_to(llty));
+ let align = src.layout.align.abi.min(dst.align);
+ src.val.store(bx, PlaceRef::new_sized_aligned(cast_ptr, src.layout, align));
+ }
+
+ // Stores the return value of a function call into it's final location.
+ fn store_return(
+ &mut self,
+ bx: &mut Bx,
+ dest: ReturnDest<'tcx, Bx::Value>,
+ ret_abi: &ArgAbi<'tcx, Ty<'tcx>>,
+ llval: Bx::Value,
+ ) {
+ use self::ReturnDest::*;
+
+ match dest {
+ Nothing => (),
+ Store(dst) => bx.store_arg(&ret_abi, llval, dst),
+ IndirectOperand(tmp, index) => {
+ let op = bx.load_operand(tmp);
+ tmp.storage_dead(bx);
+ self.locals[index] = LocalRef::Operand(Some(op));
+ self.debug_introduce_local(bx, index);
+ }
+ DirectOperand(index) => {
+ // If there is a cast, we have to store and reload.
+ let op = if let PassMode::Cast(_) = ret_abi.mode {
+ let tmp = PlaceRef::alloca(bx, ret_abi.layout);
+ tmp.storage_live(bx);
+ bx.store_arg(&ret_abi, llval, tmp);
+ let op = bx.load_operand(tmp);
+ tmp.storage_dead(bx);
+ op
+ } else {
+ OperandRef::from_immediate_or_packed_pair(bx, llval, ret_abi.layout)
+ };
+ self.locals[index] = LocalRef::Operand(Some(op));
+ self.debug_introduce_local(bx, index);
+ }
+ }
+ }
+}
+
+enum ReturnDest<'tcx, V> {
+ // Do nothing; the return value is indirect or ignored.
+ Nothing,
+ // Store the return value to the pointer.
+ Store(PlaceRef<'tcx, V>),
+ // Store an indirect return value to an operand local place.
+ IndirectOperand(PlaceRef<'tcx, V>, mir::Local),
+ // Store a direct return value to an operand local place.
+ DirectOperand(mir::Local),
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/constant.rs b/compiler/rustc_codegen_ssa/src/mir/constant.rs
new file mode 100644
index 000000000..9a995fbf6
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/constant.rs
@@ -0,0 +1,90 @@
+use crate::mir::operand::OperandRef;
+use crate::traits::*;
+use rustc_middle::mir;
+use rustc_middle::mir::interpret::{ConstValue, ErrorHandled};
+use rustc_middle::ty::layout::HasTyCtxt;
+use rustc_middle::ty::{self, Ty};
+use rustc_span::source_map::Span;
+use rustc_target::abi::Abi;
+
+use super::FunctionCx;
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn eval_mir_constant_to_operand(
+ &self,
+ bx: &mut Bx,
+ constant: &mir::Constant<'tcx>,
+ ) -> Result<OperandRef<'tcx, Bx::Value>, ErrorHandled> {
+ let val = self.eval_mir_constant(constant)?;
+ let ty = self.monomorphize(constant.ty());
+ Ok(OperandRef::from_const(bx, val, ty))
+ }
+
+ pub fn eval_mir_constant(
+ &self,
+ constant: &mir::Constant<'tcx>,
+ ) -> Result<ConstValue<'tcx>, ErrorHandled> {
+ let ct = self.monomorphize(constant.literal);
+ let ct = match ct {
+ mir::ConstantKind::Ty(ct) => ct,
+ mir::ConstantKind::Val(val, _) => return Ok(val),
+ };
+ match ct.kind() {
+ ty::ConstKind::Unevaluated(ct) => self
+ .cx
+ .tcx()
+ .const_eval_resolve(ty::ParamEnv::reveal_all(), ct, None)
+ .map_err(|err| {
+ self.cx.tcx().sess.span_err(constant.span, "erroneous constant encountered");
+ err
+ }),
+ ty::ConstKind::Value(val) => Ok(self.cx.tcx().valtree_to_const_val((ct.ty(), val))),
+ err => span_bug!(
+ constant.span,
+ "encountered bad ConstKind after monomorphizing: {:?}",
+ err
+ ),
+ }
+ }
+
+ /// process constant containing SIMD shuffle indices
+ pub fn simd_shuffle_indices(
+ &mut self,
+ bx: &Bx,
+ span: Span,
+ ty: Ty<'tcx>,
+ constant: Result<ConstValue<'tcx>, ErrorHandled>,
+ ) -> (Bx::Value, Ty<'tcx>) {
+ constant
+ .map(|val| {
+ let field_ty = ty.builtin_index().unwrap();
+ let c = mir::ConstantKind::from_value(val, ty);
+ let values: Vec<_> = bx
+ .tcx()
+ .destructure_mir_constant(ty::ParamEnv::reveal_all(), c)
+ .fields
+ .iter()
+ .map(|field| {
+ if let Some(prim) = field.try_to_scalar() {
+ let layout = bx.layout_of(field_ty);
+ let Abi::Scalar(scalar) = layout.abi else {
+ bug!("from_const: invalid ByVal layout: {:#?}", layout);
+ };
+ bx.scalar_to_backend(prim, scalar, bx.immediate_backend_type(layout))
+ } else {
+ bug!("simd shuffle field {:?}", field)
+ }
+ })
+ .collect();
+ let llval = bx.const_struct(&values, false);
+ (llval, c.ty())
+ })
+ .unwrap_or_else(|_| {
+ bx.tcx().sess.span_err(span, "could not evaluate shuffle_indices at compile time");
+ // We've errored, so we don't have to produce working code.
+ let ty = self.monomorphize(ty);
+ let llty = bx.backend_type(bx.layout_of(ty));
+ (bx.const_undef(llty), ty)
+ })
+ }
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs
new file mode 100644
index 000000000..f1fe49528
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/coverageinfo.rs
@@ -0,0 +1,55 @@
+use crate::traits::*;
+
+use rustc_middle::mir::coverage::*;
+use rustc_middle::mir::Coverage;
+use rustc_middle::mir::SourceScope;
+
+use super::FunctionCx;
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn codegen_coverage(&self, bx: &mut Bx, coverage: Coverage, scope: SourceScope) {
+ // Determine the instance that coverage data was originally generated for.
+ let instance = if let Some(inlined) = scope.inlined_instance(&self.mir.source_scopes) {
+ self.monomorphize(inlined)
+ } else {
+ self.instance
+ };
+
+ let Coverage { kind, code_region } = coverage;
+ match kind {
+ CoverageKind::Counter { function_source_hash, id } => {
+ if bx.set_function_source_hash(instance, function_source_hash) {
+ // If `set_function_source_hash()` returned true, the coverage map is enabled,
+ // so continue adding the counter.
+ if let Some(code_region) = code_region {
+ // Note: Some counters do not have code regions, but may still be referenced
+ // from expressions. In that case, don't add the counter to the coverage map,
+ // but do inject the counter intrinsic.
+ bx.add_coverage_counter(instance, id, code_region);
+ }
+
+ let coverageinfo = bx.tcx().coverageinfo(instance.def);
+
+ let fn_name = bx.get_pgo_func_name_var(instance);
+ let hash = bx.const_u64(function_source_hash);
+ let num_counters = bx.const_u32(coverageinfo.num_counters);
+ let index = bx.const_u32(id.zero_based_index());
+ debug!(
+ "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, index={:?})",
+ fn_name, hash, num_counters, index,
+ );
+ bx.instrprof_increment(fn_name, hash, num_counters, index);
+ }
+ }
+ CoverageKind::Expression { id, lhs, op, rhs } => {
+ bx.add_coverage_counter_expression(instance, id, lhs, op, rhs, code_region);
+ }
+ CoverageKind::Unreachable => {
+ bx.add_coverage_unreachable(
+ instance,
+ code_region.expect("unreachable regions always have code regions"),
+ );
+ }
+ }
+ }
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
new file mode 100644
index 000000000..8c3186efc
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/debuginfo.rs
@@ -0,0 +1,418 @@
+use crate::traits::*;
+use rustc_index::vec::IndexVec;
+use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags;
+use rustc_middle::mir;
+use rustc_middle::ty;
+use rustc_middle::ty::layout::LayoutOf;
+use rustc_session::config::DebugInfo;
+use rustc_span::symbol::{kw, Symbol};
+use rustc_span::{BytePos, Span};
+use rustc_target::abi::Abi;
+use rustc_target::abi::Size;
+
+use super::operand::{OperandRef, OperandValue};
+use super::place::PlaceRef;
+use super::{FunctionCx, LocalRef};
+
+pub struct FunctionDebugContext<S, L> {
+ pub scopes: IndexVec<mir::SourceScope, DebugScope<S, L>>,
+}
+
+#[derive(Copy, Clone)]
+pub enum VariableKind {
+ ArgumentVariable(usize /*index*/),
+ LocalVariable,
+}
+
+/// Like `mir::VarDebugInfo`, but within a `mir::Local`.
+#[derive(Copy, Clone)]
+pub struct PerLocalVarDebugInfo<'tcx, D> {
+ pub name: Symbol,
+ pub source_info: mir::SourceInfo,
+
+ /// `DIVariable` returned by `create_dbg_var`.
+ pub dbg_var: Option<D>,
+
+ /// `.place.projection` from `mir::VarDebugInfo`.
+ pub projection: &'tcx ty::List<mir::PlaceElem<'tcx>>,
+}
+
+#[derive(Clone, Copy, Debug)]
+pub struct DebugScope<S, L> {
+ pub dbg_scope: S,
+
+ /// Call site location, if this scope was inlined from another function.
+ pub inlined_at: Option<L>,
+
+ // Start and end offsets of the file to which this DIScope belongs.
+ // These are used to quickly determine whether some span refers to the same file.
+ pub file_start_pos: BytePos,
+ pub file_end_pos: BytePos,
+}
+
+impl<'tcx, S: Copy, L: Copy> DebugScope<S, L> {
+ /// DILocations inherit source file name from the parent DIScope. Due to macro expansions
+ /// it may so happen that the current span belongs to a different file than the DIScope
+ /// corresponding to span's containing source scope. If so, we need to create a DIScope
+ /// "extension" into that file.
+ pub fn adjust_dbg_scope_for_span<Cx: CodegenMethods<'tcx, DIScope = S, DILocation = L>>(
+ &self,
+ cx: &Cx,
+ span: Span,
+ ) -> S {
+ let pos = span.lo();
+ if pos < self.file_start_pos || pos >= self.file_end_pos {
+ let sm = cx.sess().source_map();
+ cx.extend_scope_to_file(self.dbg_scope, &sm.lookup_char_pos(pos).file)
+ } else {
+ self.dbg_scope
+ }
+ }
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn set_debug_loc(&self, bx: &mut Bx, source_info: mir::SourceInfo) {
+ bx.set_span(source_info.span);
+ if let Some(dbg_loc) = self.dbg_loc(source_info) {
+ bx.set_dbg_loc(dbg_loc);
+ }
+ }
+
+ fn dbg_loc(&self, source_info: mir::SourceInfo) -> Option<Bx::DILocation> {
+ let (dbg_scope, inlined_at, span) = self.adjusted_span_and_dbg_scope(source_info)?;
+ Some(self.cx.dbg_loc(dbg_scope, inlined_at, span))
+ }
+
+ fn adjusted_span_and_dbg_scope(
+ &self,
+ source_info: mir::SourceInfo,
+ ) -> Option<(Bx::DIScope, Option<Bx::DILocation>, Span)> {
+ let span = self.adjust_span_for_debugging(source_info.span);
+ let scope = &self.debug_context.as_ref()?.scopes[source_info.scope];
+ Some((scope.adjust_dbg_scope_for_span(self.cx, span), scope.inlined_at, span))
+ }
+
+ /// In order to have a good line stepping behavior in debugger, we overwrite debug
+ /// locations of macro expansions with that of the outermost expansion site
+ /// (unless the crate is being compiled with `-Z debug-macros`).
+ fn adjust_span_for_debugging(&self, mut span: Span) -> Span {
+ // Bail out if debug info emission is not enabled.
+ if self.debug_context.is_none() {
+ return span;
+ }
+
+ if span.from_expansion() && !self.cx.sess().opts.unstable_opts.debug_macros {
+ // Walk up the macro expansion chain until we reach a non-expanded span.
+ // We also stop at the function body level because no line stepping can occur
+ // at the level above that.
+ // Use span of the outermost expansion site, while keeping the original lexical scope.
+ span = rustc_span::hygiene::walk_chain(span, self.mir.span.ctxt());
+ }
+
+ span
+ }
+
+ fn spill_operand_to_stack(
+ operand: &OperandRef<'tcx, Bx::Value>,
+ name: Option<String>,
+ bx: &mut Bx,
+ ) -> PlaceRef<'tcx, Bx::Value> {
+ // "Spill" the value onto the stack, for debuginfo,
+ // without forcing non-debuginfo uses of the local
+ // to also load from the stack every single time.
+ // FIXME(#68817) use `llvm.dbg.value` instead,
+ // at least for the cases which LLVM handles correctly.
+ let spill_slot = PlaceRef::alloca(bx, operand.layout);
+ if let Some(name) = name {
+ bx.set_var_name(spill_slot.llval, &(name + ".dbg.spill"));
+ }
+ operand.val.store(bx, spill_slot);
+ spill_slot
+ }
+
+ /// Apply debuginfo and/or name, after creating the `alloca` for a local,
+ /// or initializing the local with an operand (whichever applies).
+ pub fn debug_introduce_local(&self, bx: &mut Bx, local: mir::Local) {
+ let full_debug_info = bx.sess().opts.debuginfo == DebugInfo::Full;
+
+ // FIXME(eddyb) maybe name the return place as `_0` or `return`?
+ if local == mir::RETURN_PLACE && !self.mir.local_decls[mir::RETURN_PLACE].is_user_variable()
+ {
+ return;
+ }
+
+ let vars = match &self.per_local_var_debug_info {
+ Some(per_local) => &per_local[local],
+ None => return,
+ };
+ let whole_local_var = vars.iter().find(|var| var.projection.is_empty()).copied();
+ let has_proj = || vars.iter().any(|var| !var.projection.is_empty());
+
+ let fallback_var = if self.mir.local_kind(local) == mir::LocalKind::Arg {
+ let arg_index = local.index() - 1;
+
+ // Add debuginfo even to unnamed arguments.
+ // FIXME(eddyb) is this really needed?
+ if arg_index == 0 && has_proj() {
+ // Hide closure environments from debuginfo.
+ // FIXME(eddyb) shouldn't `ArgumentVariable` indices
+ // be offset to account for the hidden environment?
+ None
+ } else if whole_local_var.is_some() {
+ // No need to make up anything, there is a `mir::VarDebugInfo`
+ // covering the whole local.
+ // FIXME(eddyb) take `whole_local_var.source_info.scope` into
+ // account, just in case it doesn't use `ArgumentVariable`
+ // (after #67586 gets fixed).
+ None
+ } else {
+ let name = kw::Empty;
+ let decl = &self.mir.local_decls[local];
+ let dbg_var = if full_debug_info {
+ self.adjusted_span_and_dbg_scope(decl.source_info).map(
+ |(dbg_scope, _, span)| {
+ // FIXME(eddyb) is this `+ 1` needed at all?
+ let kind = VariableKind::ArgumentVariable(arg_index + 1);
+
+ let arg_ty = self.monomorphize(decl.ty);
+
+ self.cx.create_dbg_var(name, arg_ty, dbg_scope, kind, span)
+ },
+ )
+ } else {
+ None
+ };
+
+ Some(PerLocalVarDebugInfo {
+ name,
+ source_info: decl.source_info,
+ dbg_var,
+ projection: ty::List::empty(),
+ })
+ }
+ } else {
+ None
+ };
+
+ let local_ref = &self.locals[local];
+
+ let name = if bx.sess().fewer_names() {
+ None
+ } else {
+ Some(match whole_local_var.or(fallback_var) {
+ Some(var) if var.name != kw::Empty => var.name.to_string(),
+ _ => format!("{:?}", local),
+ })
+ };
+
+ if let Some(name) = &name {
+ match local_ref {
+ LocalRef::Place(place) | LocalRef::UnsizedPlace(place) => {
+ bx.set_var_name(place.llval, name);
+ }
+ LocalRef::Operand(Some(operand)) => match operand.val {
+ OperandValue::Ref(x, ..) | OperandValue::Immediate(x) => {
+ bx.set_var_name(x, name);
+ }
+ OperandValue::Pair(a, b) => {
+ // FIXME(eddyb) these are scalar components,
+ // maybe extract the high-level fields?
+ bx.set_var_name(a, &(name.clone() + ".0"));
+ bx.set_var_name(b, &(name.clone() + ".1"));
+ }
+ },
+ LocalRef::Operand(None) => {}
+ }
+ }
+
+ if !full_debug_info || vars.is_empty() && fallback_var.is_none() {
+ return;
+ }
+
+ let base = match local_ref {
+ LocalRef::Operand(None) => return,
+
+ LocalRef::Operand(Some(operand)) => {
+ // Don't spill operands onto the stack in naked functions.
+ // See: https://github.com/rust-lang/rust/issues/42779
+ let attrs = bx.tcx().codegen_fn_attrs(self.instance.def_id());
+ if attrs.flags.contains(CodegenFnAttrFlags::NAKED) {
+ return;
+ }
+
+ Self::spill_operand_to_stack(operand, name, bx)
+ }
+
+ LocalRef::Place(place) => *place,
+
+ // FIXME(eddyb) add debuginfo for unsized places too.
+ LocalRef::UnsizedPlace(_) => return,
+ };
+
+ let vars = vars.iter().copied().chain(fallback_var);
+
+ for var in vars {
+ let Some(dbg_var) = var.dbg_var else { continue };
+ let Some(dbg_loc) = self.dbg_loc(var.source_info) else { continue };
+
+ let mut direct_offset = Size::ZERO;
+ // FIXME(eddyb) use smallvec here.
+ let mut indirect_offsets = vec![];
+ let mut place = base;
+
+ for elem in &var.projection[..] {
+ match *elem {
+ mir::ProjectionElem::Deref => {
+ indirect_offsets.push(Size::ZERO);
+ place = bx.load_operand(place).deref(bx.cx());
+ }
+ mir::ProjectionElem::Field(field, _) => {
+ let i = field.index();
+ let offset = indirect_offsets.last_mut().unwrap_or(&mut direct_offset);
+ *offset += place.layout.fields.offset(i);
+ place = place.project_field(bx, i);
+ }
+ mir::ProjectionElem::Downcast(_, variant) => {
+ place = place.project_downcast(bx, variant);
+ }
+ _ => span_bug!(
+ var.source_info.span,
+ "unsupported var debuginfo place `{:?}`",
+ mir::Place { local, projection: var.projection },
+ ),
+ }
+ }
+
+ // When targeting MSVC, create extra allocas for arguments instead of pointing multiple
+ // dbg_var_addr() calls into the same alloca with offsets. MSVC uses CodeView records
+ // not DWARF and LLVM doesn't support translating the resulting
+ // [DW_OP_deref, DW_OP_plus_uconst, offset, DW_OP_deref] debug info to CodeView.
+ // Creating extra allocas on the stack makes the resulting debug info simple enough
+ // that LLVM can generate correct CodeView records and thus the values appear in the
+ // debugger. (#83709)
+ let should_create_individual_allocas = bx.cx().sess().target.is_like_msvc
+ && self.mir.local_kind(local) == mir::LocalKind::Arg
+ // LLVM can handle simple things but anything more complex than just a direct
+ // offset or one indirect offset of 0 is too complex for it to generate CV records
+ // correctly.
+ && (direct_offset != Size::ZERO
+ || !matches!(&indirect_offsets[..], [Size::ZERO] | []));
+
+ if should_create_individual_allocas {
+ // Create a variable which will be a pointer to the actual value
+ let ptr_ty = bx.tcx().mk_ty(ty::RawPtr(ty::TypeAndMut {
+ mutbl: mir::Mutability::Mut,
+ ty: place.layout.ty,
+ }));
+ let ptr_layout = bx.layout_of(ptr_ty);
+ let alloca = PlaceRef::alloca(bx, ptr_layout);
+ bx.set_var_name(alloca.llval, &(var.name.to_string() + ".dbg.spill"));
+
+ // Write the pointer to the variable
+ bx.store(place.llval, alloca.llval, alloca.align);
+
+ // Point the debug info to `*alloca` for the current variable
+ bx.dbg_var_addr(dbg_var, dbg_loc, alloca.llval, Size::ZERO, &[Size::ZERO]);
+ } else {
+ bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, direct_offset, &indirect_offsets);
+ }
+ }
+ }
+
+ pub fn debug_introduce_locals(&self, bx: &mut Bx) {
+ if bx.sess().opts.debuginfo == DebugInfo::Full || !bx.sess().fewer_names() {
+ for local in self.locals.indices() {
+ self.debug_introduce_local(bx, local);
+ }
+ }
+ }
+
+ /// Partition all `VarDebugInfo` in `self.mir`, by their base `Local`.
+ pub fn compute_per_local_var_debug_info(
+ &self,
+ bx: &mut Bx,
+ ) -> Option<IndexVec<mir::Local, Vec<PerLocalVarDebugInfo<'tcx, Bx::DIVariable>>>> {
+ let full_debug_info = self.cx.sess().opts.debuginfo == DebugInfo::Full;
+
+ let target_is_msvc = self.cx.sess().target.is_like_msvc;
+
+ if !full_debug_info && self.cx.sess().fewer_names() {
+ return None;
+ }
+
+ let mut per_local = IndexVec::from_elem(vec![], &self.mir.local_decls);
+ for var in &self.mir.var_debug_info {
+ let dbg_scope_and_span = if full_debug_info {
+ self.adjusted_span_and_dbg_scope(var.source_info)
+ } else {
+ None
+ };
+
+ let dbg_var = dbg_scope_and_span.map(|(dbg_scope, _, span)| {
+ let (var_ty, var_kind) = match var.value {
+ mir::VarDebugInfoContents::Place(place) => {
+ let var_ty = self.monomorphized_place_ty(place.as_ref());
+ let var_kind = if self.mir.local_kind(place.local) == mir::LocalKind::Arg
+ && place.projection.is_empty()
+ && var.source_info.scope == mir::OUTERMOST_SOURCE_SCOPE
+ {
+ let arg_index = place.local.index() - 1;
+ if target_is_msvc {
+ // ScalarPair parameters are spilled to the stack so they need to
+ // be marked as a `LocalVariable` for MSVC debuggers to visualize
+ // their data correctly. (See #81894 & #88625)
+ let var_ty_layout = self.cx.layout_of(var_ty);
+ if let Abi::ScalarPair(_, _) = var_ty_layout.abi {
+ VariableKind::LocalVariable
+ } else {
+ VariableKind::ArgumentVariable(arg_index + 1)
+ }
+ } else {
+ // FIXME(eddyb) shouldn't `ArgumentVariable` indices be
+ // offset in closures to account for the hidden environment?
+ // Also, is this `+ 1` needed at all?
+ VariableKind::ArgumentVariable(arg_index + 1)
+ }
+ } else {
+ VariableKind::LocalVariable
+ };
+ (var_ty, var_kind)
+ }
+ mir::VarDebugInfoContents::Const(c) => {
+ let ty = self.monomorphize(c.ty());
+ (ty, VariableKind::LocalVariable)
+ }
+ };
+
+ self.cx.create_dbg_var(var.name, var_ty, dbg_scope, var_kind, span)
+ });
+
+ match var.value {
+ mir::VarDebugInfoContents::Place(place) => {
+ per_local[place.local].push(PerLocalVarDebugInfo {
+ name: var.name,
+ source_info: var.source_info,
+ dbg_var,
+ projection: place.projection,
+ });
+ }
+ mir::VarDebugInfoContents::Const(c) => {
+ if let Some(dbg_var) = dbg_var {
+ let Some(dbg_loc) = self.dbg_loc(var.source_info) else { continue };
+
+ if let Ok(operand) = self.eval_mir_constant_to_operand(bx, &c) {
+ let base = Self::spill_operand_to_stack(
+ &operand,
+ Some(var.name.to_string()),
+ bx,
+ );
+
+ bx.dbg_var_addr(dbg_var, dbg_loc, base.llval, Size::ZERO, &[]);
+ }
+ }
+ }
+ }
+ }
+ Some(per_local)
+ }
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
new file mode 100644
index 000000000..94ac71a4d
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/intrinsic.rs
@@ -0,0 +1,636 @@
+use super::operand::{OperandRef, OperandValue};
+use super::place::PlaceRef;
+use super::FunctionCx;
+use crate::common::{span_invalid_monomorphization_error, IntPredicate};
+use crate::glue;
+use crate::meth;
+use crate::traits::*;
+use crate::MemFlags;
+
+use rustc_middle::ty::{self, Ty, TyCtxt};
+use rustc_span::{sym, Span};
+use rustc_target::abi::{
+ call::{FnAbi, PassMode},
+ WrappingRange,
+};
+
+fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ allow_overlap: bool,
+ volatile: bool,
+ ty: Ty<'tcx>,
+ dst: Bx::Value,
+ src: Bx::Value,
+ count: Bx::Value,
+) {
+ let layout = bx.layout_of(ty);
+ let size = layout.size;
+ let align = layout.align.abi;
+ let size = bx.mul(bx.const_usize(size.bytes()), count);
+ let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
+ if allow_overlap {
+ bx.memmove(dst, align, src, align, size, flags);
+ } else {
+ bx.memcpy(dst, align, src, align, size, flags);
+ }
+}
+
+fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ volatile: bool,
+ ty: Ty<'tcx>,
+ dst: Bx::Value,
+ val: Bx::Value,
+ count: Bx::Value,
+) {
+ let layout = bx.layout_of(ty);
+ let size = layout.size;
+ let align = layout.align.abi;
+ let size = bx.mul(bx.const_usize(size.bytes()), count);
+ let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() };
+ bx.memset(dst, val, size, align, flags);
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn codegen_intrinsic_call(
+ bx: &mut Bx,
+ instance: ty::Instance<'tcx>,
+ fn_abi: &FnAbi<'tcx, Ty<'tcx>>,
+ args: &[OperandRef<'tcx, Bx::Value>],
+ llresult: Bx::Value,
+ span: Span,
+ ) {
+ let callee_ty = instance.ty(bx.tcx(), ty::ParamEnv::reveal_all());
+
+ let ty::FnDef(def_id, substs) = *callee_ty.kind() else {
+ bug!("expected fn item type, found {}", callee_ty);
+ };
+
+ let sig = callee_ty.fn_sig(bx.tcx());
+ let sig = bx.tcx().normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig);
+ let arg_tys = sig.inputs();
+ let ret_ty = sig.output();
+ let name = bx.tcx().item_name(def_id);
+ let name_str = name.as_str();
+
+ let llret_ty = bx.backend_type(bx.layout_of(ret_ty));
+ let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout);
+
+ let llval = match name {
+ sym::assume => {
+ bx.assume(args[0].immediate());
+ return;
+ }
+ sym::abort => {
+ bx.abort();
+ return;
+ }
+
+ sym::va_start => bx.va_start(args[0].immediate()),
+ sym::va_end => bx.va_end(args[0].immediate()),
+ sym::size_of_val => {
+ let tp_ty = substs.type_at(0);
+ if let OperandValue::Pair(_, meta) = args[0].val {
+ let (llsize, _) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta));
+ llsize
+ } else {
+ bx.const_usize(bx.layout_of(tp_ty).size.bytes())
+ }
+ }
+ sym::min_align_of_val => {
+ let tp_ty = substs.type_at(0);
+ if let OperandValue::Pair(_, meta) = args[0].val {
+ let (_, llalign) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta));
+ llalign
+ } else {
+ bx.const_usize(bx.layout_of(tp_ty).align.abi.bytes())
+ }
+ }
+ sym::vtable_size | sym::vtable_align => {
+ let vtable = args[0].immediate();
+ let idx = match name {
+ sym::vtable_size => ty::COMMON_VTABLE_ENTRIES_SIZE,
+ sym::vtable_align => ty::COMMON_VTABLE_ENTRIES_ALIGN,
+ _ => bug!(),
+ };
+ let value = meth::VirtualIndex::from_index(idx).get_usize(bx, vtable);
+ if name == sym::vtable_align {
+ // Alignment is always nonzero.
+ bx.range_metadata(value, WrappingRange { start: 1, end: !0 });
+ };
+ value
+ }
+ sym::pref_align_of
+ | sym::needs_drop
+ | sym::type_id
+ | sym::type_name
+ | sym::variant_count => {
+ let value = bx
+ .tcx()
+ .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None)
+ .unwrap();
+ OperandRef::from_const(bx, value, ret_ty).immediate_or_packed_pair(bx)
+ }
+ sym::offset => {
+ let ty = substs.type_at(0);
+ let layout = bx.layout_of(ty);
+ let ptr = args[0].immediate();
+ let offset = args[1].immediate();
+ bx.inbounds_gep(bx.backend_type(layout), ptr, &[offset])
+ }
+ sym::arith_offset => {
+ let ty = substs.type_at(0);
+ let layout = bx.layout_of(ty);
+ let ptr = args[0].immediate();
+ let offset = args[1].immediate();
+ bx.gep(bx.backend_type(layout), ptr, &[offset])
+ }
+ sym::copy => {
+ copy_intrinsic(
+ bx,
+ true,
+ false,
+ substs.type_at(0),
+ args[1].immediate(),
+ args[0].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+ sym::write_bytes => {
+ memset_intrinsic(
+ bx,
+ false,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+
+ sym::volatile_copy_nonoverlapping_memory => {
+ copy_intrinsic(
+ bx,
+ false,
+ true,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+ sym::volatile_copy_memory => {
+ copy_intrinsic(
+ bx,
+ true,
+ true,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+ sym::volatile_set_memory => {
+ memset_intrinsic(
+ bx,
+ true,
+ substs.type_at(0),
+ args[0].immediate(),
+ args[1].immediate(),
+ args[2].immediate(),
+ );
+ return;
+ }
+ sym::volatile_store => {
+ let dst = args[0].deref(bx.cx());
+ args[1].val.volatile_store(bx, dst);
+ return;
+ }
+ sym::unaligned_volatile_store => {
+ let dst = args[0].deref(bx.cx());
+ args[1].val.unaligned_volatile_store(bx, dst);
+ return;
+ }
+ sym::add_with_overflow
+ | sym::sub_with_overflow
+ | sym::mul_with_overflow
+ | sym::unchecked_div
+ | sym::unchecked_rem
+ | sym::unchecked_shl
+ | sym::unchecked_shr
+ | sym::unchecked_add
+ | sym::unchecked_sub
+ | sym::unchecked_mul
+ | sym::exact_div => {
+ let ty = arg_tys[0];
+ match int_type_width_signed(ty, bx.tcx()) {
+ Some((_width, signed)) => match name {
+ sym::add_with_overflow
+ | sym::sub_with_overflow
+ | sym::mul_with_overflow => {
+ let op = match name {
+ sym::add_with_overflow => OverflowOp::Add,
+ sym::sub_with_overflow => OverflowOp::Sub,
+ sym::mul_with_overflow => OverflowOp::Mul,
+ _ => bug!(),
+ };
+ let (val, overflow) =
+ bx.checked_binop(op, ty, args[0].immediate(), args[1].immediate());
+ // Convert `i1` to a `bool`, and write it to the out parameter
+ let val = bx.from_immediate(val);
+ let overflow = bx.from_immediate(overflow);
+
+ let dest = result.project_field(bx, 0);
+ bx.store(val, dest.llval, dest.align);
+ let dest = result.project_field(bx, 1);
+ bx.store(overflow, dest.llval, dest.align);
+
+ return;
+ }
+ sym::exact_div => {
+ if signed {
+ bx.exactsdiv(args[0].immediate(), args[1].immediate())
+ } else {
+ bx.exactudiv(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_div => {
+ if signed {
+ bx.sdiv(args[0].immediate(), args[1].immediate())
+ } else {
+ bx.udiv(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_rem => {
+ if signed {
+ bx.srem(args[0].immediate(), args[1].immediate())
+ } else {
+ bx.urem(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_shl => bx.shl(args[0].immediate(), args[1].immediate()),
+ sym::unchecked_shr => {
+ if signed {
+ bx.ashr(args[0].immediate(), args[1].immediate())
+ } else {
+ bx.lshr(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_add => {
+ if signed {
+ bx.unchecked_sadd(args[0].immediate(), args[1].immediate())
+ } else {
+ bx.unchecked_uadd(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_sub => {
+ if signed {
+ bx.unchecked_ssub(args[0].immediate(), args[1].immediate())
+ } else {
+ bx.unchecked_usub(args[0].immediate(), args[1].immediate())
+ }
+ }
+ sym::unchecked_mul => {
+ if signed {
+ bx.unchecked_smul(args[0].immediate(), args[1].immediate())
+ } else {
+ bx.unchecked_umul(args[0].immediate(), args[1].immediate())
+ }
+ }
+ _ => bug!(),
+ },
+ None => {
+ span_invalid_monomorphization_error(
+ bx.tcx().sess,
+ span,
+ &format!(
+ "invalid monomorphization of `{}` intrinsic: \
+ expected basic integer type, found `{}`",
+ name, ty
+ ),
+ );
+ return;
+ }
+ }
+ }
+ sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => {
+ match float_type_width(arg_tys[0]) {
+ Some(_width) => match name {
+ sym::fadd_fast => bx.fadd_fast(args[0].immediate(), args[1].immediate()),
+ sym::fsub_fast => bx.fsub_fast(args[0].immediate(), args[1].immediate()),
+ sym::fmul_fast => bx.fmul_fast(args[0].immediate(), args[1].immediate()),
+ sym::fdiv_fast => bx.fdiv_fast(args[0].immediate(), args[1].immediate()),
+ sym::frem_fast => bx.frem_fast(args[0].immediate(), args[1].immediate()),
+ _ => bug!(),
+ },
+ None => {
+ span_invalid_monomorphization_error(
+ bx.tcx().sess,
+ span,
+ &format!(
+ "invalid monomorphization of `{}` intrinsic: \
+ expected basic float type, found `{}`",
+ name, arg_tys[0]
+ ),
+ );
+ return;
+ }
+ }
+ }
+
+ sym::float_to_int_unchecked => {
+ if float_type_width(arg_tys[0]).is_none() {
+ span_invalid_monomorphization_error(
+ bx.tcx().sess,
+ span,
+ &format!(
+ "invalid monomorphization of `float_to_int_unchecked` \
+ intrinsic: expected basic float type, \
+ found `{}`",
+ arg_tys[0]
+ ),
+ );
+ return;
+ }
+ let Some((_width, signed)) = int_type_width_signed(ret_ty, bx.tcx()) else {
+ span_invalid_monomorphization_error(
+ bx.tcx().sess,
+ span,
+ &format!(
+ "invalid monomorphization of `float_to_int_unchecked` \
+ intrinsic: expected basic integer type, \
+ found `{}`",
+ ret_ty
+ ),
+ );
+ return;
+ };
+ if signed {
+ bx.fptosi(args[0].immediate(), llret_ty)
+ } else {
+ bx.fptoui(args[0].immediate(), llret_ty)
+ }
+ }
+
+ sym::discriminant_value => {
+ if ret_ty.is_integral() {
+ args[0].deref(bx.cx()).codegen_get_discr(bx, ret_ty)
+ } else {
+ span_bug!(span, "Invalid discriminant type for `{:?}`", arg_tys[0])
+ }
+ }
+
+ sym::const_allocate => {
+ // returns a null pointer at runtime.
+ bx.const_null(bx.type_i8p())
+ }
+
+ sym::const_deallocate => {
+ // nop at runtime.
+ return;
+ }
+
+ // This requires that atomic intrinsics follow a specific naming pattern:
+ // "atomic_<operation>[_<ordering>]"
+ name if let Some(atomic) = name_str.strip_prefix("atomic_") => {
+ use crate::common::AtomicOrdering::*;
+ use crate::common::{AtomicRmwBinOp, SynchronizationScope};
+
+ let Some((instruction, ordering)) = atomic.split_once('_') else {
+ bx.sess().fatal("Atomic intrinsic missing memory ordering");
+ };
+
+ let parse_ordering = |bx: &Bx, s| match s {
+ "unordered" => Unordered,
+ "relaxed" => Relaxed,
+ "acquire" => Acquire,
+ "release" => Release,
+ "acqrel" => AcquireRelease,
+ "seqcst" => SequentiallyConsistent,
+ _ => bx.sess().fatal("unknown ordering in atomic intrinsic"),
+ };
+
+ let invalid_monomorphization = |ty| {
+ span_invalid_monomorphization_error(
+ bx.tcx().sess,
+ span,
+ &format!(
+ "invalid monomorphization of `{}` intrinsic: \
+ expected basic integer type, found `{}`",
+ name, ty
+ ),
+ );
+ };
+
+ match instruction {
+ "cxchg" | "cxchgweak" => {
+ let Some((success, failure)) = ordering.split_once('_') else {
+ bx.sess().fatal("Atomic compare-exchange intrinsic missing failure memory ordering");
+ };
+ let ty = substs.type_at(0);
+ if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
+ let weak = instruction == "cxchgweak";
+ let mut dst = args[0].immediate();
+ let mut cmp = args[1].immediate();
+ let mut src = args[2].immediate();
+ if ty.is_unsafe_ptr() {
+ // Some platforms do not support atomic operations on pointers,
+ // so we cast to integer first.
+ let ptr_llty = bx.type_ptr_to(bx.type_isize());
+ dst = bx.pointercast(dst, ptr_llty);
+ cmp = bx.ptrtoint(cmp, bx.type_isize());
+ src = bx.ptrtoint(src, bx.type_isize());
+ }
+ let pair = bx.atomic_cmpxchg(dst, cmp, src, parse_ordering(bx, success), parse_ordering(bx, failure), weak);
+ let val = bx.extract_value(pair, 0);
+ let success = bx.extract_value(pair, 1);
+ let val = bx.from_immediate(val);
+ let success = bx.from_immediate(success);
+
+ let dest = result.project_field(bx, 0);
+ bx.store(val, dest.llval, dest.align);
+ let dest = result.project_field(bx, 1);
+ bx.store(success, dest.llval, dest.align);
+ return;
+ } else {
+ return invalid_monomorphization(ty);
+ }
+ }
+
+ "load" => {
+ let ty = substs.type_at(0);
+ if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
+ let layout = bx.layout_of(ty);
+ let size = layout.size;
+ let mut source = args[0].immediate();
+ if ty.is_unsafe_ptr() {
+ // Some platforms do not support atomic operations on pointers,
+ // so we cast to integer first...
+ let llty = bx.type_isize();
+ let ptr_llty = bx.type_ptr_to(llty);
+ source = bx.pointercast(source, ptr_llty);
+ let result = bx.atomic_load(llty, source, parse_ordering(bx, ordering), size);
+ // ... and then cast the result back to a pointer
+ bx.inttoptr(result, bx.backend_type(layout))
+ } else {
+ bx.atomic_load(bx.backend_type(layout), source, parse_ordering(bx, ordering), size)
+ }
+ } else {
+ return invalid_monomorphization(ty);
+ }
+ }
+
+ "store" => {
+ let ty = substs.type_at(0);
+ if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
+ let size = bx.layout_of(ty).size;
+ let mut val = args[1].immediate();
+ let mut ptr = args[0].immediate();
+ if ty.is_unsafe_ptr() {
+ // Some platforms do not support atomic operations on pointers,
+ // so we cast to integer first.
+ let ptr_llty = bx.type_ptr_to(bx.type_isize());
+ ptr = bx.pointercast(ptr, ptr_llty);
+ val = bx.ptrtoint(val, bx.type_isize());
+ }
+ bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size);
+ return;
+ } else {
+ return invalid_monomorphization(ty);
+ }
+ }
+
+ "fence" => {
+ bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::CrossThread);
+ return;
+ }
+
+ "singlethreadfence" => {
+ bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::SingleThread);
+ return;
+ }
+
+ // These are all AtomicRMW ops
+ op => {
+ let atom_op = match op {
+ "xchg" => AtomicRmwBinOp::AtomicXchg,
+ "xadd" => AtomicRmwBinOp::AtomicAdd,
+ "xsub" => AtomicRmwBinOp::AtomicSub,
+ "and" => AtomicRmwBinOp::AtomicAnd,
+ "nand" => AtomicRmwBinOp::AtomicNand,
+ "or" => AtomicRmwBinOp::AtomicOr,
+ "xor" => AtomicRmwBinOp::AtomicXor,
+ "max" => AtomicRmwBinOp::AtomicMax,
+ "min" => AtomicRmwBinOp::AtomicMin,
+ "umax" => AtomicRmwBinOp::AtomicUMax,
+ "umin" => AtomicRmwBinOp::AtomicUMin,
+ _ => bx.sess().fatal("unknown atomic operation"),
+ };
+
+ let ty = substs.type_at(0);
+ if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() {
+ let mut ptr = args[0].immediate();
+ let mut val = args[1].immediate();
+ if ty.is_unsafe_ptr() {
+ // Some platforms do not support atomic operations on pointers,
+ // so we cast to integer first.
+ let ptr_llty = bx.type_ptr_to(bx.type_isize());
+ ptr = bx.pointercast(ptr, ptr_llty);
+ val = bx.ptrtoint(val, bx.type_isize());
+ }
+ bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering))
+ } else {
+ return invalid_monomorphization(ty);
+ }
+ }
+ }
+ }
+
+ sym::nontemporal_store => {
+ let dst = args[0].deref(bx.cx());
+ args[1].val.nontemporal_store(bx, dst);
+ return;
+ }
+
+ sym::ptr_guaranteed_eq | sym::ptr_guaranteed_ne => {
+ let a = args[0].immediate();
+ let b = args[1].immediate();
+ if name == sym::ptr_guaranteed_eq {
+ bx.icmp(IntPredicate::IntEQ, a, b)
+ } else {
+ bx.icmp(IntPredicate::IntNE, a, b)
+ }
+ }
+
+ sym::ptr_offset_from | sym::ptr_offset_from_unsigned => {
+ let ty = substs.type_at(0);
+ let pointee_size = bx.layout_of(ty).size;
+
+ let a = args[0].immediate();
+ let b = args[1].immediate();
+ let a = bx.ptrtoint(a, bx.type_isize());
+ let b = bx.ptrtoint(b, bx.type_isize());
+ let pointee_size = bx.const_usize(pointee_size.bytes());
+ if name == sym::ptr_offset_from {
+ // This is the same sequence that Clang emits for pointer subtraction.
+ // It can be neither `nsw` nor `nuw` because the input is treated as
+ // unsigned but then the output is treated as signed, so neither works.
+ let d = bx.sub(a, b);
+ // this is where the signed magic happens (notice the `s` in `exactsdiv`)
+ bx.exactsdiv(d, pointee_size)
+ } else {
+ // The `_unsigned` version knows the relative ordering of the pointers,
+ // so can use `sub nuw` and `udiv exact` instead of dealing in signed.
+ let d = bx.unchecked_usub(a, b);
+ bx.exactudiv(d, pointee_size)
+ }
+ }
+
+ _ => {
+ // Need to use backend-specific things in the implementation.
+ bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span);
+ return;
+ }
+ };
+
+ if !fn_abi.ret.is_ignore() {
+ if let PassMode::Cast(ty) = fn_abi.ret.mode {
+ let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(&ty));
+ let ptr = bx.pointercast(result.llval, ptr_llty);
+ bx.store(llval, ptr, result.align);
+ } else {
+ OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout)
+ .val
+ .store(bx, result);
+ }
+ }
+ }
+}
+
+// Returns the width of an int Ty, and if it's signed or not
+// Returns None if the type is not an integer
+// FIXME: there’s multiple of this functions, investigate using some of the already existing
+// stuffs.
+fn int_type_width_signed(ty: Ty<'_>, tcx: TyCtxt<'_>) -> Option<(u64, bool)> {
+ match ty.kind() {
+ ty::Int(t) => {
+ Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), true))
+ }
+ ty::Uint(t) => {
+ Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), false))
+ }
+ _ => None,
+ }
+}
+
+// Returns the width of a float Ty
+// Returns None if the type is not a float
+fn float_type_width(ty: Ty<'_>) -> Option<u64> {
+ match ty.kind() {
+ ty::Float(t) => Some(t.bit_width()),
+ _ => None,
+ }
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/mod.rs b/compiler/rustc_codegen_ssa/src/mir/mod.rs
new file mode 100644
index 000000000..8ee375fa9
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/mod.rs
@@ -0,0 +1,410 @@
+use crate::traits::*;
+use rustc_middle::mir;
+use rustc_middle::mir::interpret::ErrorHandled;
+use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, TyAndLayout};
+use rustc_middle::ty::{self, Instance, Ty, TypeFoldable, TypeVisitable};
+use rustc_target::abi::call::{FnAbi, PassMode};
+
+use std::iter;
+
+use rustc_index::bit_set::BitSet;
+use rustc_index::vec::IndexVec;
+
+use self::debuginfo::{FunctionDebugContext, PerLocalVarDebugInfo};
+use self::place::PlaceRef;
+use rustc_middle::mir::traversal;
+
+use self::operand::{OperandRef, OperandValue};
+
+/// Master context for codegenning from MIR.
+pub struct FunctionCx<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> {
+ instance: Instance<'tcx>,
+
+ mir: &'tcx mir::Body<'tcx>,
+
+ debug_context: Option<FunctionDebugContext<Bx::DIScope, Bx::DILocation>>,
+
+ llfn: Bx::Function,
+
+ cx: &'a Bx::CodegenCx,
+
+ fn_abi: &'tcx FnAbi<'tcx, Ty<'tcx>>,
+
+ /// When unwinding is initiated, we have to store this personality
+ /// value somewhere so that we can load it and re-use it in the
+ /// resume instruction. The personality is (afaik) some kind of
+ /// value used for C++ unwinding, which must filter by type: we
+ /// don't really care about it very much. Anyway, this value
+ /// contains an alloca into which the personality is stored and
+ /// then later loaded when generating the DIVERGE_BLOCK.
+ personality_slot: Option<PlaceRef<'tcx, Bx::Value>>,
+
+ /// A backend `BasicBlock` for each MIR `BasicBlock`, created lazily
+ /// as-needed (e.g. RPO reaching it or another block branching to it).
+ // FIXME(eddyb) rename `llbbs` and other `ll`-prefixed things to use a
+ // more backend-agnostic prefix such as `cg` (i.e. this would be `cgbbs`).
+ cached_llbbs: IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>>,
+
+ /// The funclet status of each basic block
+ cleanup_kinds: IndexVec<mir::BasicBlock, analyze::CleanupKind>,
+
+ /// When targeting MSVC, this stores the cleanup info for each funclet BB.
+ /// This is initialized at the same time as the `landing_pads` entry for the
+ /// funclets' head block, i.e. when needed by an unwind / `cleanup_ret` edge.
+ funclets: IndexVec<mir::BasicBlock, Option<Bx::Funclet>>,
+
+ /// This stores the cached landing/cleanup pad block for a given BB.
+ // FIXME(eddyb) rename this to `eh_pads`.
+ landing_pads: IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>>,
+
+ /// Cached unreachable block
+ unreachable_block: Option<Bx::BasicBlock>,
+
+ /// Cached double unwind guarding block
+ double_unwind_guard: Option<Bx::BasicBlock>,
+
+ /// The location where each MIR arg/var/tmp/ret is stored. This is
+ /// usually an `PlaceRef` representing an alloca, but not always:
+ /// sometimes we can skip the alloca and just store the value
+ /// directly using an `OperandRef`, which makes for tighter LLVM
+ /// IR. The conditions for using an `OperandRef` are as follows:
+ ///
+ /// - the type of the local must be judged "immediate" by `is_llvm_immediate`
+ /// - the operand must never be referenced indirectly
+ /// - we should not take its address using the `&` operator
+ /// - nor should it appear in a place path like `tmp.a`
+ /// - the operand must be defined by an rvalue that can generate immediate
+ /// values
+ ///
+ /// Avoiding allocs can also be important for certain intrinsics,
+ /// notably `expect`.
+ locals: IndexVec<mir::Local, LocalRef<'tcx, Bx::Value>>,
+
+ /// All `VarDebugInfo` from the MIR body, partitioned by `Local`.
+ /// This is `None` if no var`#[non_exhaustive]`iable debuginfo/names are needed.
+ per_local_var_debug_info:
+ Option<IndexVec<mir::Local, Vec<PerLocalVarDebugInfo<'tcx, Bx::DIVariable>>>>,
+
+ /// Caller location propagated if this function has `#[track_caller]`.
+ caller_location: Option<OperandRef<'tcx, Bx::Value>>,
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn monomorphize<T>(&self, value: T) -> T
+ where
+ T: Copy + TypeFoldable<'tcx>,
+ {
+ debug!("monomorphize: self.instance={:?}", self.instance);
+ self.instance.subst_mir_and_normalize_erasing_regions(
+ self.cx.tcx(),
+ ty::ParamEnv::reveal_all(),
+ value,
+ )
+ }
+}
+
+enum LocalRef<'tcx, V> {
+ Place(PlaceRef<'tcx, V>),
+ /// `UnsizedPlace(p)`: `p` itself is a thin pointer (indirect place).
+ /// `*p` is the fat pointer that references the actual unsized place.
+ /// Every time it is initialized, we have to reallocate the place
+ /// and update the fat pointer. That's the reason why it is indirect.
+ UnsizedPlace(PlaceRef<'tcx, V>),
+ Operand(Option<OperandRef<'tcx, V>>),
+}
+
+impl<'a, 'tcx, V: CodegenObject> LocalRef<'tcx, V> {
+ fn new_operand<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ bx: &mut Bx,
+ layout: TyAndLayout<'tcx>,
+ ) -> LocalRef<'tcx, V> {
+ if layout.is_zst() {
+ // Zero-size temporaries aren't always initialized, which
+ // doesn't matter because they don't contain data, but
+ // we need something in the operand.
+ LocalRef::Operand(Some(OperandRef::new_zst(bx, layout)))
+ } else {
+ LocalRef::Operand(None)
+ }
+ }
+}
+
+///////////////////////////////////////////////////////////////////////////
+
+#[instrument(level = "debug", skip(cx))]
+pub fn codegen_mir<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ cx: &'a Bx::CodegenCx,
+ instance: Instance<'tcx>,
+) {
+ assert!(!instance.substs.needs_infer());
+
+ let llfn = cx.get_fn(instance);
+
+ let mir = cx.tcx().instance_mir(instance.def);
+
+ let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty());
+ debug!("fn_abi: {:?}", fn_abi);
+
+ let debug_context = cx.create_function_debug_context(instance, &fn_abi, llfn, &mir);
+
+ let start_llbb = Bx::append_block(cx, llfn, "start");
+ let mut bx = Bx::build(cx, start_llbb);
+
+ if mir.basic_blocks().iter().any(|bb| bb.is_cleanup) {
+ bx.set_personality_fn(cx.eh_personality());
+ }
+
+ let cleanup_kinds = analyze::cleanup_kinds(&mir);
+ let cached_llbbs: IndexVec<mir::BasicBlock, Option<Bx::BasicBlock>> = mir
+ .basic_blocks()
+ .indices()
+ .map(|bb| if bb == mir::START_BLOCK { Some(start_llbb) } else { None })
+ .collect();
+
+ let mut fx = FunctionCx {
+ instance,
+ mir,
+ llfn,
+ fn_abi,
+ cx,
+ personality_slot: None,
+ cached_llbbs,
+ unreachable_block: None,
+ double_unwind_guard: None,
+ cleanup_kinds,
+ landing_pads: IndexVec::from_elem(None, mir.basic_blocks()),
+ funclets: IndexVec::from_fn_n(|_| None, mir.basic_blocks().len()),
+ locals: IndexVec::new(),
+ debug_context,
+ per_local_var_debug_info: None,
+ caller_location: None,
+ };
+
+ fx.per_local_var_debug_info = fx.compute_per_local_var_debug_info(&mut bx);
+
+ // Evaluate all required consts; codegen later assumes that CTFE will never fail.
+ let mut all_consts_ok = true;
+ for const_ in &mir.required_consts {
+ if let Err(err) = fx.eval_mir_constant(const_) {
+ all_consts_ok = false;
+ match err {
+ // errored or at least linted
+ ErrorHandled::Reported(_) | ErrorHandled::Linted => {}
+ ErrorHandled::TooGeneric => {
+ span_bug!(const_.span, "codgen encountered polymorphic constant: {:?}", err)
+ }
+ }
+ }
+ }
+ if !all_consts_ok {
+ // We leave the IR in some half-built state here, and rely on this code not even being
+ // submitted to LLVM once an error was raised.
+ return;
+ }
+
+ let memory_locals = analyze::non_ssa_locals(&fx);
+
+ // Allocate variable and temp allocas
+ fx.locals = {
+ let args = arg_local_refs(&mut bx, &mut fx, &memory_locals);
+
+ let mut allocate_local = |local| {
+ let decl = &mir.local_decls[local];
+ let layout = bx.layout_of(fx.monomorphize(decl.ty));
+ assert!(!layout.ty.has_erasable_regions());
+
+ if local == mir::RETURN_PLACE && fx.fn_abi.ret.is_indirect() {
+ debug!("alloc: {:?} (return place) -> place", local);
+ let llretptr = bx.get_param(0);
+ return LocalRef::Place(PlaceRef::new_sized(llretptr, layout));
+ }
+
+ if memory_locals.contains(local) {
+ debug!("alloc: {:?} -> place", local);
+ if layout.is_unsized() {
+ LocalRef::UnsizedPlace(PlaceRef::alloca_unsized_indirect(&mut bx, layout))
+ } else {
+ LocalRef::Place(PlaceRef::alloca(&mut bx, layout))
+ }
+ } else {
+ debug!("alloc: {:?} -> operand", local);
+ LocalRef::new_operand(&mut bx, layout)
+ }
+ };
+
+ let retptr = allocate_local(mir::RETURN_PLACE);
+ iter::once(retptr)
+ .chain(args.into_iter())
+ .chain(mir.vars_and_temps_iter().map(allocate_local))
+ .collect()
+ };
+
+ // Apply debuginfo to the newly allocated locals.
+ fx.debug_introduce_locals(&mut bx);
+
+ // Codegen the body of each block using reverse postorder
+ for (bb, _) in traversal::reverse_postorder(&mir) {
+ fx.codegen_block(bb);
+ }
+}
+
+/// Produces, for each argument, a `Value` pointing at the
+/// argument's value. As arguments are places, these are always
+/// indirect.
+fn arg_local_refs<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ fx: &mut FunctionCx<'a, 'tcx, Bx>,
+ memory_locals: &BitSet<mir::Local>,
+) -> Vec<LocalRef<'tcx, Bx::Value>> {
+ let mir = fx.mir;
+ let mut idx = 0;
+ let mut llarg_idx = fx.fn_abi.ret.is_indirect() as usize;
+
+ let mut num_untupled = None;
+
+ let args = mir
+ .args_iter()
+ .enumerate()
+ .map(|(arg_index, local)| {
+ let arg_decl = &mir.local_decls[local];
+
+ if Some(local) == mir.spread_arg {
+ // This argument (e.g., the last argument in the "rust-call" ABI)
+ // is a tuple that was spread at the ABI level and now we have
+ // to reconstruct it into a tuple local variable, from multiple
+ // individual LLVM function arguments.
+
+ let arg_ty = fx.monomorphize(arg_decl.ty);
+ let ty::Tuple(tupled_arg_tys) = arg_ty.kind() else {
+ bug!("spread argument isn't a tuple?!");
+ };
+
+ let place = PlaceRef::alloca(bx, bx.layout_of(arg_ty));
+ for i in 0..tupled_arg_tys.len() {
+ let arg = &fx.fn_abi.args[idx];
+ idx += 1;
+ if arg.pad.is_some() {
+ llarg_idx += 1;
+ }
+ let pr_field = place.project_field(bx, i);
+ bx.store_fn_arg(arg, &mut llarg_idx, pr_field);
+ }
+ assert_eq!(
+ None,
+ num_untupled.replace(tupled_arg_tys.len()),
+ "Replaced existing num_tupled"
+ );
+
+ return LocalRef::Place(place);
+ }
+
+ if fx.fn_abi.c_variadic && arg_index == fx.fn_abi.args.len() {
+ let arg_ty = fx.monomorphize(arg_decl.ty);
+
+ let va_list = PlaceRef::alloca(bx, bx.layout_of(arg_ty));
+ bx.va_start(va_list.llval);
+
+ return LocalRef::Place(va_list);
+ }
+
+ let arg = &fx.fn_abi.args[idx];
+ idx += 1;
+ if arg.pad.is_some() {
+ llarg_idx += 1;
+ }
+
+ if !memory_locals.contains(local) {
+ // We don't have to cast or keep the argument in the alloca.
+ // FIXME(eddyb): We should figure out how to use llvm.dbg.value instead
+ // of putting everything in allocas just so we can use llvm.dbg.declare.
+ let local = |op| LocalRef::Operand(Some(op));
+ match arg.mode {
+ PassMode::Ignore => {
+ return local(OperandRef::new_zst(bx, arg.layout));
+ }
+ PassMode::Direct(_) => {
+ let llarg = bx.get_param(llarg_idx);
+ llarg_idx += 1;
+ return local(OperandRef::from_immediate_or_packed_pair(
+ bx, llarg, arg.layout,
+ ));
+ }
+ PassMode::Pair(..) => {
+ let (a, b) = (bx.get_param(llarg_idx), bx.get_param(llarg_idx + 1));
+ llarg_idx += 2;
+
+ return local(OperandRef {
+ val: OperandValue::Pair(a, b),
+ layout: arg.layout,
+ });
+ }
+ _ => {}
+ }
+ }
+
+ if arg.is_sized_indirect() {
+ // Don't copy an indirect argument to an alloca, the caller
+ // already put it in a temporary alloca and gave it up.
+ // FIXME: lifetimes
+ let llarg = bx.get_param(llarg_idx);
+ llarg_idx += 1;
+ LocalRef::Place(PlaceRef::new_sized(llarg, arg.layout))
+ } else if arg.is_unsized_indirect() {
+ // As the storage for the indirect argument lives during
+ // the whole function call, we just copy the fat pointer.
+ let llarg = bx.get_param(llarg_idx);
+ llarg_idx += 1;
+ let llextra = bx.get_param(llarg_idx);
+ llarg_idx += 1;
+ let indirect_operand = OperandValue::Pair(llarg, llextra);
+
+ let tmp = PlaceRef::alloca_unsized_indirect(bx, arg.layout);
+ indirect_operand.store(bx, tmp);
+ LocalRef::UnsizedPlace(tmp)
+ } else {
+ let tmp = PlaceRef::alloca(bx, arg.layout);
+ bx.store_fn_arg(arg, &mut llarg_idx, tmp);
+ LocalRef::Place(tmp)
+ }
+ })
+ .collect::<Vec<_>>();
+
+ if fx.instance.def.requires_caller_location(bx.tcx()) {
+ let mir_args = if let Some(num_untupled) = num_untupled {
+ // Subtract off the tupled argument that gets 'expanded'
+ args.len() - 1 + num_untupled
+ } else {
+ args.len()
+ };
+ assert_eq!(
+ fx.fn_abi.args.len(),
+ mir_args + 1,
+ "#[track_caller] instance {:?} must have 1 more argument in their ABI than in their MIR",
+ fx.instance
+ );
+
+ let arg = fx.fn_abi.args.last().unwrap();
+ match arg.mode {
+ PassMode::Direct(_) => (),
+ _ => bug!("caller location must be PassMode::Direct, found {:?}", arg.mode),
+ }
+
+ fx.caller_location = Some(OperandRef {
+ val: OperandValue::Immediate(bx.get_param(llarg_idx)),
+ layout: arg.layout,
+ });
+ }
+
+ args
+}
+
+mod analyze;
+mod block;
+pub mod constant;
+pub mod coverageinfo;
+pub mod debuginfo;
+mod intrinsic;
+pub mod operand;
+pub mod place;
+mod rvalue;
+mod statement;
diff --git a/compiler/rustc_codegen_ssa/src/mir/operand.rs b/compiler/rustc_codegen_ssa/src/mir/operand.rs
new file mode 100644
index 000000000..c612634fc
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/operand.rs
@@ -0,0 +1,461 @@
+use super::place::PlaceRef;
+use super::{FunctionCx, LocalRef};
+
+use crate::base;
+use crate::glue;
+use crate::traits::*;
+use crate::MemFlags;
+
+use rustc_middle::mir;
+use rustc_middle::mir::interpret::{ConstValue, Pointer, Scalar};
+use rustc_middle::ty::layout::{LayoutOf, TyAndLayout};
+use rustc_middle::ty::Ty;
+use rustc_target::abi::{Abi, Align, Size};
+
+use std::fmt;
+
+/// The representation of a Rust value. The enum variant is in fact
+/// uniquely determined by the value's type, but is kept as a
+/// safety check.
+#[derive(Copy, Clone, Debug)]
+pub enum OperandValue<V> {
+ /// A reference to the actual operand. The data is guaranteed
+ /// to be valid for the operand's lifetime.
+ /// The second value, if any, is the extra data (vtable or length)
+ /// which indicates that it refers to an unsized rvalue.
+ Ref(V, Option<V>, Align),
+ /// A single LLVM value.
+ Immediate(V),
+ /// A pair of immediate LLVM values. Used by fat pointers too.
+ Pair(V, V),
+}
+
+/// An `OperandRef` is an "SSA" reference to a Rust value, along with
+/// its type.
+///
+/// NOTE: unless you know a value's type exactly, you should not
+/// generate LLVM opcodes acting on it and instead act via methods,
+/// to avoid nasty edge cases. In particular, using `Builder::store`
+/// directly is sure to cause problems -- use `OperandRef::store`
+/// instead.
+#[derive(Copy, Clone)]
+pub struct OperandRef<'tcx, V> {
+ // The value.
+ pub val: OperandValue<V>,
+
+ // The layout of value, based on its Rust type.
+ pub layout: TyAndLayout<'tcx>,
+}
+
+impl<V: CodegenObject> fmt::Debug for OperandRef<'_, V> {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ write!(f, "OperandRef({:?} @ {:?})", self.val, self.layout)
+ }
+}
+
+impl<'a, 'tcx, V: CodegenObject> OperandRef<'tcx, V> {
+ pub fn new_zst<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ bx: &mut Bx,
+ layout: TyAndLayout<'tcx>,
+ ) -> OperandRef<'tcx, V> {
+ assert!(layout.is_zst());
+ OperandRef {
+ val: OperandValue::Immediate(bx.const_undef(bx.immediate_backend_type(layout))),
+ layout,
+ }
+ }
+
+ pub fn from_const<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ bx: &mut Bx,
+ val: ConstValue<'tcx>,
+ ty: Ty<'tcx>,
+ ) -> Self {
+ let layout = bx.layout_of(ty);
+
+ if layout.is_zst() {
+ return OperandRef::new_zst(bx, layout);
+ }
+
+ let val = match val {
+ ConstValue::Scalar(x) => {
+ let Abi::Scalar(scalar) = layout.abi else {
+ bug!("from_const: invalid ByVal layout: {:#?}", layout);
+ };
+ let llval = bx.scalar_to_backend(x, scalar, bx.immediate_backend_type(layout));
+ OperandValue::Immediate(llval)
+ }
+ ConstValue::ZeroSized => {
+ let llval = bx.zst_to_backend(bx.immediate_backend_type(layout));
+ OperandValue::Immediate(llval)
+ }
+ ConstValue::Slice { data, start, end } => {
+ let Abi::ScalarPair(a_scalar, _) = layout.abi else {
+ bug!("from_const: invalid ScalarPair layout: {:#?}", layout);
+ };
+ let a = Scalar::from_pointer(
+ Pointer::new(bx.tcx().create_memory_alloc(data), Size::from_bytes(start)),
+ &bx.tcx(),
+ );
+ let a_llval = bx.scalar_to_backend(
+ a,
+ a_scalar,
+ bx.scalar_pair_element_backend_type(layout, 0, true),
+ );
+ let b_llval = bx.const_usize((end - start) as u64);
+ OperandValue::Pair(a_llval, b_llval)
+ }
+ ConstValue::ByRef { alloc, offset } => {
+ return bx.load_operand(bx.from_const_alloc(layout, alloc, offset));
+ }
+ };
+
+ OperandRef { val, layout }
+ }
+
+ /// Asserts that this operand refers to a scalar and returns
+ /// a reference to its value.
+ pub fn immediate(self) -> V {
+ match self.val {
+ OperandValue::Immediate(s) => s,
+ _ => bug!("not immediate: {:?}", self),
+ }
+ }
+
+ pub fn deref<Cx: LayoutTypeMethods<'tcx>>(self, cx: &Cx) -> PlaceRef<'tcx, V> {
+ if self.layout.ty.is_box() {
+ bug!("dereferencing {:?} in codegen", self.layout.ty);
+ }
+
+ let projected_ty = self
+ .layout
+ .ty
+ .builtin_deref(true)
+ .unwrap_or_else(|| bug!("deref of non-pointer {:?}", self))
+ .ty;
+
+ let (llptr, llextra) = match self.val {
+ OperandValue::Immediate(llptr) => (llptr, None),
+ OperandValue::Pair(llptr, llextra) => (llptr, Some(llextra)),
+ OperandValue::Ref(..) => bug!("Deref of by-Ref operand {:?}", self),
+ };
+ let layout = cx.layout_of(projected_ty);
+ PlaceRef { llval: llptr, llextra, layout, align: layout.align.abi }
+ }
+
+ /// If this operand is a `Pair`, we return an aggregate with the two values.
+ /// For other cases, see `immediate`.
+ pub fn immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ ) -> V {
+ if let OperandValue::Pair(a, b) = self.val {
+ let llty = bx.cx().backend_type(self.layout);
+ debug!("Operand::immediate_or_packed_pair: packing {:?} into {:?}", self, llty);
+ // Reconstruct the immediate aggregate.
+ let mut llpair = bx.cx().const_undef(llty);
+ let imm_a = bx.from_immediate(a);
+ let imm_b = bx.from_immediate(b);
+ llpair = bx.insert_value(llpair, imm_a, 0);
+ llpair = bx.insert_value(llpair, imm_b, 1);
+ llpair
+ } else {
+ self.immediate()
+ }
+ }
+
+ /// If the type is a pair, we return a `Pair`, otherwise, an `Immediate`.
+ pub fn from_immediate_or_packed_pair<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ bx: &mut Bx,
+ llval: V,
+ layout: TyAndLayout<'tcx>,
+ ) -> Self {
+ let val = if let Abi::ScalarPair(a, b) = layout.abi {
+ debug!("Operand::from_immediate_or_packed_pair: unpacking {:?} @ {:?}", llval, layout);
+
+ // Deconstruct the immediate aggregate.
+ let a_llval = bx.extract_value(llval, 0);
+ let a_llval = bx.to_immediate_scalar(a_llval, a);
+ let b_llval = bx.extract_value(llval, 1);
+ let b_llval = bx.to_immediate_scalar(b_llval, b);
+ OperandValue::Pair(a_llval, b_llval)
+ } else {
+ OperandValue::Immediate(llval)
+ };
+ OperandRef { val, layout }
+ }
+
+ pub fn extract_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ &self,
+ bx: &mut Bx,
+ i: usize,
+ ) -> Self {
+ let field = self.layout.field(bx.cx(), i);
+ let offset = self.layout.fields.offset(i);
+
+ let mut val = match (self.val, self.layout.abi) {
+ // If the field is ZST, it has no data.
+ _ if field.is_zst() => {
+ return OperandRef::new_zst(bx, field);
+ }
+
+ // Newtype of a scalar, scalar pair or vector.
+ (OperandValue::Immediate(_) | OperandValue::Pair(..), _)
+ if field.size == self.layout.size =>
+ {
+ assert_eq!(offset.bytes(), 0);
+ self.val
+ }
+
+ // Extract a scalar component from a pair.
+ (OperandValue::Pair(a_llval, b_llval), Abi::ScalarPair(a, b)) => {
+ if offset.bytes() == 0 {
+ assert_eq!(field.size, a.size(bx.cx()));
+ OperandValue::Immediate(a_llval)
+ } else {
+ assert_eq!(offset, a.size(bx.cx()).align_to(b.align(bx.cx()).abi));
+ assert_eq!(field.size, b.size(bx.cx()));
+ OperandValue::Immediate(b_llval)
+ }
+ }
+
+ // `#[repr(simd)]` types are also immediate.
+ (OperandValue::Immediate(llval), Abi::Vector { .. }) => {
+ OperandValue::Immediate(bx.extract_element(llval, bx.cx().const_usize(i as u64)))
+ }
+
+ _ => bug!("OperandRef::extract_field({:?}): not applicable", self),
+ };
+
+ match (&mut val, field.abi) {
+ (OperandValue::Immediate(llval), _) => {
+ // Bools in union fields needs to be truncated.
+ *llval = bx.to_immediate(*llval, field);
+ // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
+ *llval = bx.bitcast(*llval, bx.cx().immediate_backend_type(field));
+ }
+ (OperandValue::Pair(a, b), Abi::ScalarPair(a_abi, b_abi)) => {
+ // Bools in union fields needs to be truncated.
+ *a = bx.to_immediate_scalar(*a, a_abi);
+ *b = bx.to_immediate_scalar(*b, b_abi);
+ // HACK(eddyb) have to bitcast pointers until LLVM removes pointee types.
+ *a = bx.bitcast(*a, bx.cx().scalar_pair_element_backend_type(field, 0, true));
+ *b = bx.bitcast(*b, bx.cx().scalar_pair_element_backend_type(field, 1, true));
+ }
+ (OperandValue::Pair(..), _) => bug!(),
+ (OperandValue::Ref(..), _) => bug!(),
+ }
+
+ OperandRef { val, layout: field }
+ }
+}
+
+impl<'a, 'tcx, V: CodegenObject> OperandValue<V> {
+ pub fn store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ ) {
+ self.store_with_flags(bx, dest, MemFlags::empty());
+ }
+
+ pub fn volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ ) {
+ self.store_with_flags(bx, dest, MemFlags::VOLATILE);
+ }
+
+ pub fn unaligned_volatile_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ ) {
+ self.store_with_flags(bx, dest, MemFlags::VOLATILE | MemFlags::UNALIGNED);
+ }
+
+ pub fn nontemporal_store<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ ) {
+ self.store_with_flags(bx, dest, MemFlags::NONTEMPORAL);
+ }
+
+ fn store_with_flags<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ dest: PlaceRef<'tcx, V>,
+ flags: MemFlags,
+ ) {
+ debug!("OperandRef::store: operand={:?}, dest={:?}", self, dest);
+ // Avoid generating stores of zero-sized values, because the only way to have a zero-sized
+ // value is through `undef`, and store itself is useless.
+ if dest.layout.is_zst() {
+ return;
+ }
+ match self {
+ OperandValue::Ref(r, None, source_align) => {
+ if flags.contains(MemFlags::NONTEMPORAL) {
+ // HACK(nox): This is inefficient but there is no nontemporal memcpy.
+ let ty = bx.backend_type(dest.layout);
+ let ptr = bx.pointercast(r, bx.type_ptr_to(ty));
+ let val = bx.load(ty, ptr, source_align);
+ bx.store_with_flags(val, dest.llval, dest.align, flags);
+ return;
+ }
+ base::memcpy_ty(bx, dest.llval, dest.align, r, source_align, dest.layout, flags)
+ }
+ OperandValue::Ref(_, Some(_), _) => {
+ bug!("cannot directly store unsized values");
+ }
+ OperandValue::Immediate(s) => {
+ let val = bx.from_immediate(s);
+ bx.store_with_flags(val, dest.llval, dest.align, flags);
+ }
+ OperandValue::Pair(a, b) => {
+ let Abi::ScalarPair(a_scalar, b_scalar) = dest.layout.abi else {
+ bug!("store_with_flags: invalid ScalarPair layout: {:#?}", dest.layout);
+ };
+ let ty = bx.backend_type(dest.layout);
+ let b_offset = a_scalar.size(bx).align_to(b_scalar.align(bx).abi);
+
+ let llptr = bx.struct_gep(ty, dest.llval, 0);
+ let val = bx.from_immediate(a);
+ let align = dest.align;
+ bx.store_with_flags(val, llptr, align, flags);
+
+ let llptr = bx.struct_gep(ty, dest.llval, 1);
+ let val = bx.from_immediate(b);
+ let align = dest.align.restrict_for_offset(b_offset);
+ bx.store_with_flags(val, llptr, align, flags);
+ }
+ }
+ }
+
+ pub fn store_unsized<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ indirect_dest: PlaceRef<'tcx, V>,
+ ) {
+ debug!("OperandRef::store_unsized: operand={:?}, indirect_dest={:?}", self, indirect_dest);
+ let flags = MemFlags::empty();
+
+ // `indirect_dest` must have `*mut T` type. We extract `T` out of it.
+ let unsized_ty = indirect_dest
+ .layout
+ .ty
+ .builtin_deref(true)
+ .unwrap_or_else(|| bug!("indirect_dest has non-pointer type: {:?}", indirect_dest))
+ .ty;
+
+ let OperandValue::Ref(llptr, Some(llextra), _) = self else {
+ bug!("store_unsized called with a sized value")
+ };
+
+ // FIXME: choose an appropriate alignment, or use dynamic align somehow
+ let max_align = Align::from_bits(128).unwrap();
+ let min_align = Align::from_bits(8).unwrap();
+
+ // Allocate an appropriate region on the stack, and copy the value into it
+ let (llsize, _) = glue::size_and_align_of_dst(bx, unsized_ty, Some(llextra));
+ let lldst = bx.array_alloca(bx.cx().type_i8(), llsize, max_align);
+ bx.memcpy(lldst, max_align, llptr, min_align, llsize, flags);
+
+ // Store the allocated region and the extra to the indirect place.
+ let indirect_operand = OperandValue::Pair(lldst, llextra);
+ indirect_operand.store(bx, indirect_dest);
+ }
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ fn maybe_codegen_consume_direct(
+ &mut self,
+ bx: &mut Bx,
+ place_ref: mir::PlaceRef<'tcx>,
+ ) -> Option<OperandRef<'tcx, Bx::Value>> {
+ debug!("maybe_codegen_consume_direct(place_ref={:?})", place_ref);
+
+ match self.locals[place_ref.local] {
+ LocalRef::Operand(Some(mut o)) => {
+ // Moves out of scalar and scalar pair fields are trivial.
+ for elem in place_ref.projection.iter() {
+ match elem {
+ mir::ProjectionElem::Field(ref f, _) => {
+ o = o.extract_field(bx, f.index());
+ }
+ mir::ProjectionElem::Index(_)
+ | mir::ProjectionElem::ConstantIndex { .. } => {
+ // ZSTs don't require any actual memory access.
+ // FIXME(eddyb) deduplicate this with the identical
+ // checks in `codegen_consume` and `extract_field`.
+ let elem = o.layout.field(bx.cx(), 0);
+ if elem.is_zst() {
+ o = OperandRef::new_zst(bx, elem);
+ } else {
+ return None;
+ }
+ }
+ _ => return None,
+ }
+ }
+
+ Some(o)
+ }
+ LocalRef::Operand(None) => {
+ bug!("use of {:?} before def", place_ref);
+ }
+ LocalRef::Place(..) | LocalRef::UnsizedPlace(..) => {
+ // watch out for locals that do not have an
+ // alloca; they are handled somewhat differently
+ None
+ }
+ }
+ }
+
+ pub fn codegen_consume(
+ &mut self,
+ bx: &mut Bx,
+ place_ref: mir::PlaceRef<'tcx>,
+ ) -> OperandRef<'tcx, Bx::Value> {
+ debug!("codegen_consume(place_ref={:?})", place_ref);
+
+ let ty = self.monomorphized_place_ty(place_ref);
+ let layout = bx.cx().layout_of(ty);
+
+ // ZSTs don't require any actual memory access.
+ if layout.is_zst() {
+ return OperandRef::new_zst(bx, layout);
+ }
+
+ if let Some(o) = self.maybe_codegen_consume_direct(bx, place_ref) {
+ return o;
+ }
+
+ // for most places, to consume them we just load them
+ // out from their home
+ let place = self.codegen_place(bx, place_ref);
+ bx.load_operand(place)
+ }
+
+ pub fn codegen_operand(
+ &mut self,
+ bx: &mut Bx,
+ operand: &mir::Operand<'tcx>,
+ ) -> OperandRef<'tcx, Bx::Value> {
+ debug!("codegen_operand(operand={:?})", operand);
+
+ match *operand {
+ mir::Operand::Copy(ref place) | mir::Operand::Move(ref place) => {
+ self.codegen_consume(bx, place.as_ref())
+ }
+
+ mir::Operand::Constant(ref constant) => {
+ // This cannot fail because we checked all required_consts in advance.
+ self.eval_mir_constant_to_operand(bx, constant).unwrap_or_else(|_err| {
+ span_bug!(constant.span, "erroneous constant not captured by required_consts")
+ })
+ }
+ }
+ }
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/place.rs b/compiler/rustc_codegen_ssa/src/mir/place.rs
new file mode 100644
index 000000000..268c4d765
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/place.rs
@@ -0,0 +1,549 @@
+use super::operand::OperandValue;
+use super::{FunctionCx, LocalRef};
+
+use crate::common::IntPredicate;
+use crate::glue;
+use crate::traits::*;
+use crate::MemFlags;
+
+use rustc_middle::mir;
+use rustc_middle::mir::tcx::PlaceTy;
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf, TyAndLayout};
+use rustc_middle::ty::{self, Ty};
+use rustc_target::abi::{Abi, Align, FieldsShape, Int, TagEncoding};
+use rustc_target::abi::{VariantIdx, Variants};
+
+#[derive(Copy, Clone, Debug)]
+pub struct PlaceRef<'tcx, V> {
+ /// A pointer to the contents of the place.
+ pub llval: V,
+
+ /// This place's extra data if it is unsized, or `None` if null.
+ pub llextra: Option<V>,
+
+ /// The monomorphized type of this place, including variant information.
+ pub layout: TyAndLayout<'tcx>,
+
+ /// The alignment we know for this place.
+ pub align: Align,
+}
+
+impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
+ pub fn new_sized(llval: V, layout: TyAndLayout<'tcx>) -> PlaceRef<'tcx, V> {
+ assert!(!layout.is_unsized());
+ PlaceRef { llval, llextra: None, layout, align: layout.align.abi }
+ }
+
+ pub fn new_sized_aligned(
+ llval: V,
+ layout: TyAndLayout<'tcx>,
+ align: Align,
+ ) -> PlaceRef<'tcx, V> {
+ assert!(!layout.is_unsized());
+ PlaceRef { llval, llextra: None, layout, align }
+ }
+
+ // FIXME(eddyb) pass something else for the name so no work is done
+ // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`).
+ pub fn alloca<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ bx: &mut Bx,
+ layout: TyAndLayout<'tcx>,
+ ) -> Self {
+ assert!(!layout.is_unsized(), "tried to statically allocate unsized place");
+ let tmp = bx.alloca(bx.cx().backend_type(layout), layout.align.abi);
+ Self::new_sized(tmp, layout)
+ }
+
+ /// Returns a place for an indirect reference to an unsized place.
+ // FIXME(eddyb) pass something else for the name so no work is done
+ // unless LLVM IR names are turned on (e.g. for `--emit=llvm-ir`).
+ pub fn alloca_unsized_indirect<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ bx: &mut Bx,
+ layout: TyAndLayout<'tcx>,
+ ) -> Self {
+ assert!(layout.is_unsized(), "tried to allocate indirect place for sized values");
+ let ptr_ty = bx.cx().tcx().mk_mut_ptr(layout.ty);
+ let ptr_layout = bx.cx().layout_of(ptr_ty);
+ Self::alloca(bx, ptr_layout)
+ }
+
+ pub fn len<Cx: ConstMethods<'tcx, Value = V>>(&self, cx: &Cx) -> V {
+ if let FieldsShape::Array { count, .. } = self.layout.fields {
+ if self.layout.is_unsized() {
+ assert_eq!(count, 0);
+ self.llextra.unwrap()
+ } else {
+ cx.const_usize(count)
+ }
+ } else {
+ bug!("unexpected layout `{:#?}` in PlaceRef::len", self.layout)
+ }
+ }
+}
+
+impl<'a, 'tcx, V: CodegenObject> PlaceRef<'tcx, V> {
+ /// Access a field, at a point when the value's case is known.
+ pub fn project_field<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ ix: usize,
+ ) -> Self {
+ let field = self.layout.field(bx.cx(), ix);
+ let offset = self.layout.fields.offset(ix);
+ let effective_field_align = self.align.restrict_for_offset(offset);
+
+ let mut simple = || {
+ let llval = match self.layout.abi {
+ _ if offset.bytes() == 0 => {
+ // Unions and newtypes only use an offset of 0.
+ // Also handles the first field of Scalar, ScalarPair, and Vector layouts.
+ self.llval
+ }
+ Abi::ScalarPair(a, b)
+ if offset == a.size(bx.cx()).align_to(b.align(bx.cx()).abi) =>
+ {
+ // Offset matches second field.
+ let ty = bx.backend_type(self.layout);
+ bx.struct_gep(ty, self.llval, 1)
+ }
+ Abi::Scalar(_) | Abi::ScalarPair(..) | Abi::Vector { .. } if field.is_zst() => {
+ // ZST fields are not included in Scalar, ScalarPair, and Vector layouts, so manually offset the pointer.
+ let byte_ptr = bx.pointercast(self.llval, bx.cx().type_i8p());
+ bx.gep(bx.cx().type_i8(), byte_ptr, &[bx.const_usize(offset.bytes())])
+ }
+ Abi::Scalar(_) | Abi::ScalarPair(..) => {
+ // All fields of Scalar and ScalarPair layouts must have been handled by this point.
+ // Vector layouts have additional fields for each element of the vector, so don't panic in that case.
+ bug!(
+ "offset of non-ZST field `{:?}` does not match layout `{:#?}`",
+ field,
+ self.layout
+ );
+ }
+ _ => {
+ let ty = bx.backend_type(self.layout);
+ bx.struct_gep(ty, self.llval, bx.cx().backend_field_index(self.layout, ix))
+ }
+ };
+ PlaceRef {
+ // HACK(eddyb): have to bitcast pointers until LLVM removes pointee types.
+ llval: bx.pointercast(llval, bx.cx().type_ptr_to(bx.cx().backend_type(field))),
+ llextra: if bx.cx().type_has_metadata(field.ty) { self.llextra } else { None },
+ layout: field,
+ align: effective_field_align,
+ }
+ };
+
+ // Simple cases, which don't need DST adjustment:
+ // * no metadata available - just log the case
+ // * known alignment - sized types, `[T]`, `str` or a foreign type
+ // * packed struct - there is no alignment padding
+ match field.ty.kind() {
+ _ if self.llextra.is_none() => {
+ debug!(
+ "unsized field `{}`, of `{:?}` has no metadata for adjustment",
+ ix, self.llval
+ );
+ return simple();
+ }
+ _ if !field.is_unsized() => return simple(),
+ ty::Slice(..) | ty::Str | ty::Foreign(..) => return simple(),
+ ty::Adt(def, _) => {
+ if def.repr().packed() {
+ // FIXME(eddyb) generalize the adjustment when we
+ // start supporting packing to larger alignments.
+ assert_eq!(self.layout.align.abi.bytes(), 1);
+ return simple();
+ }
+ }
+ _ => {}
+ }
+
+ // We need to get the pointer manually now.
+ // We do this by casting to a `*i8`, then offsetting it by the appropriate amount.
+ // We do this instead of, say, simply adjusting the pointer from the result of a GEP
+ // because the field may have an arbitrary alignment in the LLVM representation
+ // anyway.
+ //
+ // To demonstrate:
+ //
+ // struct Foo<T: ?Sized> {
+ // x: u16,
+ // y: T
+ // }
+ //
+ // The type `Foo<Foo<Trait>>` is represented in LLVM as `{ u16, { u16, u8 }}`, meaning that
+ // the `y` field has 16-bit alignment.
+
+ let meta = self.llextra;
+
+ let unaligned_offset = bx.cx().const_usize(offset.bytes());
+
+ // Get the alignment of the field
+ let (_, unsized_align) = glue::size_and_align_of_dst(bx, field.ty, meta);
+
+ // Bump the unaligned offset up to the appropriate alignment
+ let offset = round_up_const_value_to_alignment(bx, unaligned_offset, unsized_align);
+
+ debug!("struct_field_ptr: DST field offset: {:?}", offset);
+
+ // Cast and adjust pointer.
+ let byte_ptr = bx.pointercast(self.llval, bx.cx().type_i8p());
+ let byte_ptr = bx.gep(bx.cx().type_i8(), byte_ptr, &[offset]);
+
+ // Finally, cast back to the type expected.
+ let ll_fty = bx.cx().backend_type(field);
+ debug!("struct_field_ptr: Field type is {:?}", ll_fty);
+
+ PlaceRef {
+ llval: bx.pointercast(byte_ptr, bx.cx().type_ptr_to(ll_fty)),
+ llextra: self.llextra,
+ layout: field,
+ align: effective_field_align,
+ }
+ }
+
+ /// Obtain the actual discriminant of a value.
+ #[instrument(level = "trace", skip(bx))]
+ pub fn codegen_get_discr<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ self,
+ bx: &mut Bx,
+ cast_to: Ty<'tcx>,
+ ) -> V {
+ let cast_to = bx.cx().immediate_backend_type(bx.cx().layout_of(cast_to));
+ if self.layout.abi.is_uninhabited() {
+ return bx.cx().const_undef(cast_to);
+ }
+ let (tag_scalar, tag_encoding, tag_field) = match self.layout.variants {
+ Variants::Single { index } => {
+ let discr_val = self
+ .layout
+ .ty
+ .discriminant_for_variant(bx.cx().tcx(), index)
+ .map_or(index.as_u32() as u128, |discr| discr.val);
+ return bx.cx().const_uint_big(cast_to, discr_val);
+ }
+ Variants::Multiple { tag, ref tag_encoding, tag_field, .. } => {
+ (tag, tag_encoding, tag_field)
+ }
+ };
+
+ // Read the tag/niche-encoded discriminant from memory.
+ let tag = self.project_field(bx, tag_field);
+ let tag = bx.load_operand(tag);
+
+ // Decode the discriminant (specifically if it's niche-encoded).
+ match *tag_encoding {
+ TagEncoding::Direct => {
+ let signed = match tag_scalar.primitive() {
+ // We use `i1` for bytes that are always `0` or `1`,
+ // e.g., `#[repr(i8)] enum E { A, B }`, but we can't
+ // let LLVM interpret the `i1` as signed, because
+ // then `i1 1` (i.e., `E::B`) is effectively `i8 -1`.
+ Int(_, signed) => !tag_scalar.is_bool() && signed,
+ _ => false,
+ };
+ bx.intcast(tag.immediate(), cast_to, signed)
+ }
+ TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
+ // Rebase from niche values to discriminants, and check
+ // whether the result is in range for the niche variants.
+ let niche_llty = bx.cx().immediate_backend_type(tag.layout);
+ let tag = tag.immediate();
+
+ // We first compute the "relative discriminant" (wrt `niche_variants`),
+ // that is, if `n = niche_variants.end() - niche_variants.start()`,
+ // we remap `niche_start..=niche_start + n` (which may wrap around)
+ // to (non-wrap-around) `0..=n`, to be able to check whether the
+ // discriminant corresponds to a niche variant with one comparison.
+ // We also can't go directly to the (variant index) discriminant
+ // and check that it is in the range `niche_variants`, because
+ // that might not fit in the same type, on top of needing an extra
+ // comparison (see also the comment on `let niche_discr`).
+ let relative_discr = if niche_start == 0 {
+ // Avoid subtracting `0`, which wouldn't work for pointers.
+ // FIXME(eddyb) check the actual primitive type here.
+ tag
+ } else {
+ bx.sub(tag, bx.cx().const_uint_big(niche_llty, niche_start))
+ };
+ let relative_max = niche_variants.end().as_u32() - niche_variants.start().as_u32();
+ let is_niche = if relative_max == 0 {
+ // Avoid calling `const_uint`, which wouldn't work for pointers.
+ // Also use canonical == 0 instead of non-canonical u<= 0.
+ // FIXME(eddyb) check the actual primitive type here.
+ bx.icmp(IntPredicate::IntEQ, relative_discr, bx.cx().const_null(niche_llty))
+ } else {
+ let relative_max = bx.cx().const_uint(niche_llty, relative_max as u64);
+ bx.icmp(IntPredicate::IntULE, relative_discr, relative_max)
+ };
+
+ // NOTE(eddyb) this addition needs to be performed on the final
+ // type, in case the niche itself can't represent all variant
+ // indices (e.g. `u8` niche with more than `256` variants,
+ // but enough uninhabited variants so that the remaining variants
+ // fit in the niche).
+ // In other words, `niche_variants.end - niche_variants.start`
+ // is representable in the niche, but `niche_variants.end`
+ // might not be, in extreme cases.
+ let niche_discr = {
+ let relative_discr = if relative_max == 0 {
+ // HACK(eddyb) since we have only one niche, we know which
+ // one it is, and we can avoid having a dynamic value here.
+ bx.cx().const_uint(cast_to, 0)
+ } else {
+ bx.intcast(relative_discr, cast_to, false)
+ };
+ bx.add(
+ relative_discr,
+ bx.cx().const_uint(cast_to, niche_variants.start().as_u32() as u64),
+ )
+ };
+
+ bx.select(
+ is_niche,
+ niche_discr,
+ bx.cx().const_uint(cast_to, dataful_variant.as_u32() as u64),
+ )
+ }
+ }
+ }
+
+ /// Sets the discriminant for a new value of the given case of the given
+ /// representation.
+ pub fn codegen_set_discr<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ &self,
+ bx: &mut Bx,
+ variant_index: VariantIdx,
+ ) {
+ if self.layout.for_variant(bx.cx(), variant_index).abi.is_uninhabited() {
+ // We play it safe by using a well-defined `abort`, but we could go for immediate UB
+ // if that turns out to be helpful.
+ bx.abort();
+ return;
+ }
+ match self.layout.variants {
+ Variants::Single { index } => {
+ assert_eq!(index, variant_index);
+ }
+ Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } => {
+ let ptr = self.project_field(bx, tag_field);
+ let to =
+ self.layout.ty.discriminant_for_variant(bx.tcx(), variant_index).unwrap().val;
+ bx.store(
+ bx.cx().const_uint_big(bx.cx().backend_type(ptr.layout), to),
+ ptr.llval,
+ ptr.align,
+ );
+ }
+ Variants::Multiple {
+ tag_encoding:
+ TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
+ tag_field,
+ ..
+ } => {
+ if variant_index != dataful_variant {
+ if bx.cx().sess().target.arch == "arm"
+ || bx.cx().sess().target.arch == "aarch64"
+ {
+ // FIXME(#34427): as workaround for LLVM bug on ARM,
+ // use memset of 0 before assigning niche value.
+ let fill_byte = bx.cx().const_u8(0);
+ let size = bx.cx().const_usize(self.layout.size.bytes());
+ bx.memset(self.llval, fill_byte, size, self.align, MemFlags::empty());
+ }
+
+ let niche = self.project_field(bx, tag_field);
+ let niche_llty = bx.cx().immediate_backend_type(niche.layout);
+ let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
+ let niche_value = (niche_value as u128).wrapping_add(niche_start);
+ // FIXME(eddyb): check the actual primitive type here.
+ let niche_llval = if niche_value == 0 {
+ // HACK(eddyb): using `c_null` as it works on all types.
+ bx.cx().const_null(niche_llty)
+ } else {
+ bx.cx().const_uint_big(niche_llty, niche_value)
+ };
+ OperandValue::Immediate(niche_llval).store(bx, niche);
+ }
+ }
+ }
+ }
+
+ pub fn project_index<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ &self,
+ bx: &mut Bx,
+ llindex: V,
+ ) -> Self {
+ // Statically compute the offset if we can, otherwise just use the element size,
+ // as this will yield the lowest alignment.
+ let layout = self.layout.field(bx, 0);
+ let offset = if let Some(llindex) = bx.const_to_opt_uint(llindex) {
+ layout.size.checked_mul(llindex, bx).unwrap_or(layout.size)
+ } else {
+ layout.size
+ };
+
+ PlaceRef {
+ llval: bx.inbounds_gep(
+ bx.cx().backend_type(self.layout),
+ self.llval,
+ &[bx.cx().const_usize(0), llindex],
+ ),
+ llextra: None,
+ layout,
+ align: self.align.restrict_for_offset(offset),
+ }
+ }
+
+ pub fn project_downcast<Bx: BuilderMethods<'a, 'tcx, Value = V>>(
+ &self,
+ bx: &mut Bx,
+ variant_index: VariantIdx,
+ ) -> Self {
+ let mut downcast = *self;
+ downcast.layout = self.layout.for_variant(bx.cx(), variant_index);
+
+ // Cast to the appropriate variant struct type.
+ let variant_ty = bx.cx().backend_type(downcast.layout);
+ downcast.llval = bx.pointercast(downcast.llval, bx.cx().type_ptr_to(variant_ty));
+
+ downcast
+ }
+
+ pub fn storage_live<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) {
+ bx.lifetime_start(self.llval, self.layout.size);
+ }
+
+ pub fn storage_dead<Bx: BuilderMethods<'a, 'tcx, Value = V>>(&self, bx: &mut Bx) {
+ bx.lifetime_end(self.llval, self.layout.size);
+ }
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ #[instrument(level = "trace", skip(self, bx))]
+ pub fn codegen_place(
+ &mut self,
+ bx: &mut Bx,
+ place_ref: mir::PlaceRef<'tcx>,
+ ) -> PlaceRef<'tcx, Bx::Value> {
+ let cx = self.cx;
+ let tcx = self.cx.tcx();
+
+ let mut base = 0;
+ let mut cg_base = match self.locals[place_ref.local] {
+ LocalRef::Place(place) => place,
+ LocalRef::UnsizedPlace(place) => bx.load_operand(place).deref(cx),
+ LocalRef::Operand(..) => {
+ if place_ref.has_deref() {
+ base = 1;
+ let cg_base = self.codegen_consume(
+ bx,
+ mir::PlaceRef { projection: &place_ref.projection[..0], ..place_ref },
+ );
+ cg_base.deref(bx.cx())
+ } else {
+ bug!("using operand local {:?} as place", place_ref);
+ }
+ }
+ };
+ for elem in place_ref.projection[base..].iter() {
+ cg_base = match *elem {
+ mir::ProjectionElem::Deref => bx.load_operand(cg_base).deref(bx.cx()),
+ mir::ProjectionElem::Field(ref field, _) => {
+ cg_base.project_field(bx, field.index())
+ }
+ mir::ProjectionElem::Index(index) => {
+ let index = &mir::Operand::Copy(mir::Place::from(index));
+ let index = self.codegen_operand(bx, index);
+ let llindex = index.immediate();
+ cg_base.project_index(bx, llindex)
+ }
+ mir::ProjectionElem::ConstantIndex { offset, from_end: false, min_length: _ } => {
+ let lloffset = bx.cx().const_usize(offset as u64);
+ cg_base.project_index(bx, lloffset)
+ }
+ mir::ProjectionElem::ConstantIndex { offset, from_end: true, min_length: _ } => {
+ let lloffset = bx.cx().const_usize(offset as u64);
+ let lllen = cg_base.len(bx.cx());
+ let llindex = bx.sub(lllen, lloffset);
+ cg_base.project_index(bx, llindex)
+ }
+ mir::ProjectionElem::Subslice { from, to, from_end } => {
+ let mut subslice = cg_base.project_index(bx, bx.cx().const_usize(from as u64));
+ let projected_ty =
+ PlaceTy::from_ty(cg_base.layout.ty).projection_ty(tcx, *elem).ty;
+ subslice.layout = bx.cx().layout_of(self.monomorphize(projected_ty));
+
+ if subslice.layout.is_unsized() {
+ assert!(from_end, "slice subslices should be `from_end`");
+ subslice.llextra = Some(bx.sub(
+ cg_base.llextra.unwrap(),
+ bx.cx().const_usize((from as u64) + (to as u64)),
+ ));
+ }
+
+ // Cast the place pointer type to the new
+ // array or slice type (`*[%_; new_len]`).
+ subslice.llval = bx.pointercast(
+ subslice.llval,
+ bx.cx().type_ptr_to(bx.cx().backend_type(subslice.layout)),
+ );
+
+ subslice
+ }
+ mir::ProjectionElem::Downcast(_, v) => cg_base.project_downcast(bx, v),
+ };
+ }
+ debug!("codegen_place(place={:?}) => {:?}", place_ref, cg_base);
+ cg_base
+ }
+
+ pub fn monomorphized_place_ty(&self, place_ref: mir::PlaceRef<'tcx>) -> Ty<'tcx> {
+ let tcx = self.cx.tcx();
+ let place_ty = place_ref.ty(self.mir, tcx);
+ self.monomorphize(place_ty.ty)
+ }
+}
+
+fn round_up_const_value_to_alignment<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
+ bx: &mut Bx,
+ value: Bx::Value,
+ align: Bx::Value,
+) -> Bx::Value {
+ // In pseudo code:
+ //
+ // if value & (align - 1) == 0 {
+ // value
+ // } else {
+ // (value & !(align - 1)) + align
+ // }
+ //
+ // Usually this is written without branches as
+ //
+ // (value + align - 1) & !(align - 1)
+ //
+ // But this formula cannot take advantage of constant `value`. E.g. if `value` is known
+ // at compile time to be `1`, this expression should be optimized to `align`. However,
+ // optimization only holds if `align` is a power of two. Since the optimizer doesn't know
+ // that `align` is a power of two, it cannot perform this optimization.
+ //
+ // Instead we use
+ //
+ // value + (-value & (align - 1))
+ //
+ // Since `align` is used only once, the expression can be optimized. For `value = 0`
+ // its optimized to `0` even in debug mode.
+ //
+ // NB: The previous version of this code used
+ //
+ // (value + align - 1) & -align
+ //
+ // Even though `-align == !(align - 1)`, LLVM failed to optimize this even for
+ // `value = 0`. Bug report: https://bugs.llvm.org/show_bug.cgi?id=48559
+ let one = bx.const_usize(1);
+ let align_minus_1 = bx.sub(align, one);
+ let neg_value = bx.neg(value);
+ let offset = bx.and(neg_value, align_minus_1);
+ bx.add(value, offset)
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/rvalue.rs b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
new file mode 100644
index 000000000..26b9fbf44
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/rvalue.rs
@@ -0,0 +1,729 @@
+use super::operand::{OperandRef, OperandValue};
+use super::place::PlaceRef;
+use super::{FunctionCx, LocalRef};
+
+use crate::base;
+use crate::common::{self, IntPredicate};
+use crate::traits::*;
+use crate::MemFlags;
+
+use rustc_middle::mir;
+use rustc_middle::mir::Operand;
+use rustc_middle::ty::cast::{CastTy, IntTy};
+use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf};
+use rustc_middle::ty::{self, adjustment::PointerCast, Instance, Ty, TyCtxt};
+use rustc_span::source_map::{Span, DUMMY_SP};
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ #[instrument(level = "trace", skip(self, bx))]
+ pub fn codegen_rvalue(
+ &mut self,
+ mut bx: Bx,
+ dest: PlaceRef<'tcx, Bx::Value>,
+ rvalue: &mir::Rvalue<'tcx>,
+ ) -> Bx {
+ match *rvalue {
+ mir::Rvalue::Use(ref operand) => {
+ let cg_operand = self.codegen_operand(&mut bx, operand);
+ // FIXME: consider not copying constants through stack. (Fixable by codegen'ing
+ // constants into `OperandValue::Ref`; why don’t we do that yet if we don’t?)
+ cg_operand.val.store(&mut bx, dest);
+ bx
+ }
+
+ mir::Rvalue::Cast(mir::CastKind::Pointer(PointerCast::Unsize), ref source, _) => {
+ // The destination necessarily contains a fat pointer, so if
+ // it's a scalar pair, it's a fat pointer or newtype thereof.
+ if bx.cx().is_backend_scalar_pair(dest.layout) {
+ // Into-coerce of a thin pointer to a fat pointer -- just
+ // use the operand path.
+ let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue);
+ temp.val.store(&mut bx, dest);
+ return bx;
+ }
+
+ // Unsize of a nontrivial struct. I would prefer for
+ // this to be eliminated by MIR building, but
+ // `CoerceUnsized` can be passed by a where-clause,
+ // so the (generic) MIR may not be able to expand it.
+ let operand = self.codegen_operand(&mut bx, source);
+ match operand.val {
+ OperandValue::Pair(..) | OperandValue::Immediate(_) => {
+ // Unsize from an immediate structure. We don't
+ // really need a temporary alloca here, but
+ // avoiding it would require us to have
+ // `coerce_unsized_into` use `extractvalue` to
+ // index into the struct, and this case isn't
+ // important enough for it.
+ debug!("codegen_rvalue: creating ugly alloca");
+ let scratch = PlaceRef::alloca(&mut bx, operand.layout);
+ scratch.storage_live(&mut bx);
+ operand.val.store(&mut bx, scratch);
+ base::coerce_unsized_into(&mut bx, scratch, dest);
+ scratch.storage_dead(&mut bx);
+ }
+ OperandValue::Ref(llref, None, align) => {
+ let source = PlaceRef::new_sized_aligned(llref, operand.layout, align);
+ base::coerce_unsized_into(&mut bx, source, dest);
+ }
+ OperandValue::Ref(_, Some(_), _) => {
+ bug!("unsized coercion on an unsized rvalue");
+ }
+ }
+ bx
+ }
+
+ mir::Rvalue::Repeat(ref elem, count) => {
+ let cg_elem = self.codegen_operand(&mut bx, elem);
+
+ // Do not generate the loop for zero-sized elements or empty arrays.
+ if dest.layout.is_zst() {
+ return bx;
+ }
+
+ if let OperandValue::Immediate(v) = cg_elem.val {
+ let zero = bx.const_usize(0);
+ let start = dest.project_index(&mut bx, zero).llval;
+ let size = bx.const_usize(dest.layout.size.bytes());
+
+ // Use llvm.memset.p0i8.* to initialize all zero arrays
+ if bx.cx().const_to_opt_uint(v) == Some(0) {
+ let fill = bx.cx().const_u8(0);
+ bx.memset(start, fill, size, dest.align, MemFlags::empty());
+ return bx;
+ }
+
+ // Use llvm.memset.p0i8.* to initialize byte arrays
+ let v = bx.from_immediate(v);
+ if bx.cx().val_ty(v) == bx.cx().type_i8() {
+ bx.memset(start, v, size, dest.align, MemFlags::empty());
+ return bx;
+ }
+ }
+
+ let count =
+ self.monomorphize(count).eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
+
+ bx.write_operand_repeatedly(cg_elem, count, dest)
+ }
+
+ mir::Rvalue::Aggregate(ref kind, ref operands) => {
+ let (dest, active_field_index) = match **kind {
+ mir::AggregateKind::Adt(adt_did, variant_index, _, _, active_field_index) => {
+ dest.codegen_set_discr(&mut bx, variant_index);
+ if bx.tcx().adt_def(adt_did).is_enum() {
+ (dest.project_downcast(&mut bx, variant_index), active_field_index)
+ } else {
+ (dest, active_field_index)
+ }
+ }
+ _ => (dest, None),
+ };
+ for (i, operand) in operands.iter().enumerate() {
+ let op = self.codegen_operand(&mut bx, operand);
+ // Do not generate stores and GEPis for zero-sized fields.
+ if !op.layout.is_zst() {
+ let field_index = active_field_index.unwrap_or(i);
+ let field = if let mir::AggregateKind::Array(_) = **kind {
+ let llindex = bx.cx().const_usize(field_index as u64);
+ dest.project_index(&mut bx, llindex)
+ } else {
+ dest.project_field(&mut bx, field_index)
+ };
+ op.val.store(&mut bx, field);
+ }
+ }
+ bx
+ }
+
+ _ => {
+ assert!(self.rvalue_creates_operand(rvalue, DUMMY_SP));
+ let (mut bx, temp) = self.codegen_rvalue_operand(bx, rvalue);
+ temp.val.store(&mut bx, dest);
+ bx
+ }
+ }
+ }
+
+ pub fn codegen_rvalue_unsized(
+ &mut self,
+ mut bx: Bx,
+ indirect_dest: PlaceRef<'tcx, Bx::Value>,
+ rvalue: &mir::Rvalue<'tcx>,
+ ) -> Bx {
+ debug!(
+ "codegen_rvalue_unsized(indirect_dest.llval={:?}, rvalue={:?})",
+ indirect_dest.llval, rvalue
+ );
+
+ match *rvalue {
+ mir::Rvalue::Use(ref operand) => {
+ let cg_operand = self.codegen_operand(&mut bx, operand);
+ cg_operand.val.store_unsized(&mut bx, indirect_dest);
+ bx
+ }
+
+ _ => bug!("unsized assignment other than `Rvalue::Use`"),
+ }
+ }
+
+ pub fn codegen_rvalue_operand(
+ &mut self,
+ mut bx: Bx,
+ rvalue: &mir::Rvalue<'tcx>,
+ ) -> (Bx, OperandRef<'tcx, Bx::Value>) {
+ assert!(
+ self.rvalue_creates_operand(rvalue, DUMMY_SP),
+ "cannot codegen {:?} to operand",
+ rvalue,
+ );
+
+ match *rvalue {
+ mir::Rvalue::Cast(ref kind, ref source, mir_cast_ty) => {
+ let operand = self.codegen_operand(&mut bx, source);
+ debug!("cast operand is {:?}", operand);
+ let cast = bx.cx().layout_of(self.monomorphize(mir_cast_ty));
+
+ let val = match *kind {
+ mir::CastKind::PointerExposeAddress => {
+ assert!(bx.cx().is_backend_immediate(cast));
+ let llptr = operand.immediate();
+ let llcast_ty = bx.cx().immediate_backend_type(cast);
+ let lladdr = bx.ptrtoint(llptr, llcast_ty);
+ OperandValue::Immediate(lladdr)
+ }
+ mir::CastKind::Pointer(PointerCast::ReifyFnPointer) => {
+ match *operand.layout.ty.kind() {
+ ty::FnDef(def_id, substs) => {
+ let instance = ty::Instance::resolve_for_fn_ptr(
+ bx.tcx(),
+ ty::ParamEnv::reveal_all(),
+ def_id,
+ substs,
+ )
+ .unwrap()
+ .polymorphize(bx.cx().tcx());
+ OperandValue::Immediate(bx.get_fn_addr(instance))
+ }
+ _ => bug!("{} cannot be reified to a fn ptr", operand.layout.ty),
+ }
+ }
+ mir::CastKind::Pointer(PointerCast::ClosureFnPointer(_)) => {
+ match *operand.layout.ty.kind() {
+ ty::Closure(def_id, substs) => {
+ let instance = Instance::resolve_closure(
+ bx.cx().tcx(),
+ def_id,
+ substs,
+ ty::ClosureKind::FnOnce,
+ )
+ .expect("failed to normalize and resolve closure during codegen")
+ .polymorphize(bx.cx().tcx());
+ OperandValue::Immediate(bx.cx().get_fn_addr(instance))
+ }
+ _ => bug!("{} cannot be cast to a fn ptr", operand.layout.ty),
+ }
+ }
+ mir::CastKind::Pointer(PointerCast::UnsafeFnPointer) => {
+ // This is a no-op at the LLVM level.
+ operand.val
+ }
+ mir::CastKind::Pointer(PointerCast::Unsize) => {
+ assert!(bx.cx().is_backend_scalar_pair(cast));
+ let (lldata, llextra) = match operand.val {
+ OperandValue::Pair(lldata, llextra) => {
+ // unsize from a fat pointer -- this is a
+ // "trait-object-to-supertrait" coercion.
+ (lldata, Some(llextra))
+ }
+ OperandValue::Immediate(lldata) => {
+ // "standard" unsize
+ (lldata, None)
+ }
+ OperandValue::Ref(..) => {
+ bug!("by-ref operand {:?} in `codegen_rvalue_operand`", operand);
+ }
+ };
+ let (lldata, llextra) =
+ base::unsize_ptr(&mut bx, lldata, operand.layout.ty, cast.ty, llextra);
+ OperandValue::Pair(lldata, llextra)
+ }
+ mir::CastKind::Pointer(PointerCast::MutToConstPointer)
+ | mir::CastKind::Misc
+ if bx.cx().is_backend_scalar_pair(operand.layout) =>
+ {
+ if let OperandValue::Pair(data_ptr, meta) = operand.val {
+ if bx.cx().is_backend_scalar_pair(cast) {
+ let data_cast = bx.pointercast(
+ data_ptr,
+ bx.cx().scalar_pair_element_backend_type(cast, 0, true),
+ );
+ OperandValue::Pair(data_cast, meta)
+ } else {
+ // cast to thin-ptr
+ // Cast of fat-ptr to thin-ptr is an extraction of data-ptr and
+ // pointer-cast of that pointer to desired pointer type.
+ let llcast_ty = bx.cx().immediate_backend_type(cast);
+ let llval = bx.pointercast(data_ptr, llcast_ty);
+ OperandValue::Immediate(llval)
+ }
+ } else {
+ bug!("unexpected non-pair operand");
+ }
+ }
+ mir::CastKind::Pointer(
+ PointerCast::MutToConstPointer | PointerCast::ArrayToPointer,
+ )
+ | mir::CastKind::Misc
+ // Since int2ptr can have arbitrary integer types as input (so we have to do
+ // sign extension and all that), it is currently best handled in the same code
+ // path as the other integer-to-X casts.
+ | mir::CastKind::PointerFromExposedAddress => {
+ assert!(bx.cx().is_backend_immediate(cast));
+ let ll_t_out = bx.cx().immediate_backend_type(cast);
+ if operand.layout.abi.is_uninhabited() {
+ let val = OperandValue::Immediate(bx.cx().const_undef(ll_t_out));
+ return (bx, OperandRef { val, layout: cast });
+ }
+ let r_t_in =
+ CastTy::from_ty(operand.layout.ty).expect("bad input type for cast");
+ let r_t_out = CastTy::from_ty(cast.ty).expect("bad output type for cast");
+ let ll_t_in = bx.cx().immediate_backend_type(operand.layout);
+ let llval = operand.immediate();
+
+ let newval = match (r_t_in, r_t_out) {
+ (CastTy::Int(i), CastTy::Int(_)) => {
+ bx.intcast(llval, ll_t_out, i.is_signed())
+ }
+ (CastTy::Float, CastTy::Float) => {
+ let srcsz = bx.cx().float_width(ll_t_in);
+ let dstsz = bx.cx().float_width(ll_t_out);
+ if dstsz > srcsz {
+ bx.fpext(llval, ll_t_out)
+ } else if srcsz > dstsz {
+ bx.fptrunc(llval, ll_t_out)
+ } else {
+ llval
+ }
+ }
+ (CastTy::Int(i), CastTy::Float) => {
+ if i.is_signed() {
+ bx.sitofp(llval, ll_t_out)
+ } else {
+ bx.uitofp(llval, ll_t_out)
+ }
+ }
+ (CastTy::Ptr(_) | CastTy::FnPtr, CastTy::Ptr(_)) => {
+ bx.pointercast(llval, ll_t_out)
+ }
+ (CastTy::Int(i), CastTy::Ptr(_)) => {
+ let usize_llval =
+ bx.intcast(llval, bx.cx().type_isize(), i.is_signed());
+ bx.inttoptr(usize_llval, ll_t_out)
+ }
+ (CastTy::Float, CastTy::Int(IntTy::I)) => {
+ bx.cast_float_to_int(true, llval, ll_t_out)
+ }
+ (CastTy::Float, CastTy::Int(_)) => {
+ bx.cast_float_to_int(false, llval, ll_t_out)
+ }
+ _ => bug!("unsupported cast: {:?} to {:?}", operand.layout.ty, cast.ty),
+ };
+ OperandValue::Immediate(newval)
+ }
+ };
+ (bx, OperandRef { val, layout: cast })
+ }
+
+ mir::Rvalue::Ref(_, bk, place) => {
+ let mk_ref = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| {
+ tcx.mk_ref(
+ tcx.lifetimes.re_erased,
+ ty::TypeAndMut { ty, mutbl: bk.to_mutbl_lossy() },
+ )
+ };
+ self.codegen_place_to_pointer(bx, place, mk_ref)
+ }
+
+ mir::Rvalue::CopyForDeref(place) => {
+ let operand = self.codegen_operand(&mut bx, &Operand::Copy(place));
+ (bx, operand)
+ }
+ mir::Rvalue::AddressOf(mutability, place) => {
+ let mk_ptr = move |tcx: TyCtxt<'tcx>, ty: Ty<'tcx>| {
+ tcx.mk_ptr(ty::TypeAndMut { ty, mutbl: mutability })
+ };
+ self.codegen_place_to_pointer(bx, place, mk_ptr)
+ }
+
+ mir::Rvalue::Len(place) => {
+ let size = self.evaluate_array_len(&mut bx, place);
+ let operand = OperandRef {
+ val: OperandValue::Immediate(size),
+ layout: bx.cx().layout_of(bx.tcx().types.usize),
+ };
+ (bx, operand)
+ }
+
+ mir::Rvalue::BinaryOp(op, box (ref lhs, ref rhs)) => {
+ let lhs = self.codegen_operand(&mut bx, lhs);
+ let rhs = self.codegen_operand(&mut bx, rhs);
+ let llresult = match (lhs.val, rhs.val) {
+ (
+ OperandValue::Pair(lhs_addr, lhs_extra),
+ OperandValue::Pair(rhs_addr, rhs_extra),
+ ) => self.codegen_fat_ptr_binop(
+ &mut bx,
+ op,
+ lhs_addr,
+ lhs_extra,
+ rhs_addr,
+ rhs_extra,
+ lhs.layout.ty,
+ ),
+
+ (OperandValue::Immediate(lhs_val), OperandValue::Immediate(rhs_val)) => {
+ self.codegen_scalar_binop(&mut bx, op, lhs_val, rhs_val, lhs.layout.ty)
+ }
+
+ _ => bug!(),
+ };
+ let operand = OperandRef {
+ val: OperandValue::Immediate(llresult),
+ layout: bx.cx().layout_of(op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty)),
+ };
+ (bx, operand)
+ }
+ mir::Rvalue::CheckedBinaryOp(op, box (ref lhs, ref rhs)) => {
+ let lhs = self.codegen_operand(&mut bx, lhs);
+ let rhs = self.codegen_operand(&mut bx, rhs);
+ let result = self.codegen_scalar_checked_binop(
+ &mut bx,
+ op,
+ lhs.immediate(),
+ rhs.immediate(),
+ lhs.layout.ty,
+ );
+ let val_ty = op.ty(bx.tcx(), lhs.layout.ty, rhs.layout.ty);
+ let operand_ty = bx.tcx().intern_tup(&[val_ty, bx.tcx().types.bool]);
+ let operand = OperandRef { val: result, layout: bx.cx().layout_of(operand_ty) };
+
+ (bx, operand)
+ }
+
+ mir::Rvalue::UnaryOp(op, ref operand) => {
+ let operand = self.codegen_operand(&mut bx, operand);
+ let lloperand = operand.immediate();
+ let is_float = operand.layout.ty.is_floating_point();
+ let llval = match op {
+ mir::UnOp::Not => bx.not(lloperand),
+ mir::UnOp::Neg => {
+ if is_float {
+ bx.fneg(lloperand)
+ } else {
+ bx.neg(lloperand)
+ }
+ }
+ };
+ (bx, OperandRef { val: OperandValue::Immediate(llval), layout: operand.layout })
+ }
+
+ mir::Rvalue::Discriminant(ref place) => {
+ let discr_ty = rvalue.ty(self.mir, bx.tcx());
+ let discr_ty = self.monomorphize(discr_ty);
+ let discr = self
+ .codegen_place(&mut bx, place.as_ref())
+ .codegen_get_discr(&mut bx, discr_ty);
+ (
+ bx,
+ OperandRef {
+ val: OperandValue::Immediate(discr),
+ layout: self.cx.layout_of(discr_ty),
+ },
+ )
+ }
+
+ mir::Rvalue::NullaryOp(null_op, ty) => {
+ let ty = self.monomorphize(ty);
+ assert!(bx.cx().type_is_sized(ty));
+ let layout = bx.cx().layout_of(ty);
+ let val = match null_op {
+ mir::NullOp::SizeOf => layout.size.bytes(),
+ mir::NullOp::AlignOf => layout.align.abi.bytes(),
+ };
+ let val = bx.cx().const_usize(val);
+ let tcx = self.cx.tcx();
+ (
+ bx,
+ OperandRef {
+ val: OperandValue::Immediate(val),
+ layout: self.cx.layout_of(tcx.types.usize),
+ },
+ )
+ }
+
+ mir::Rvalue::ThreadLocalRef(def_id) => {
+ assert!(bx.cx().tcx().is_static(def_id));
+ let static_ = bx.get_static(def_id);
+ let layout = bx.layout_of(bx.cx().tcx().static_ptr_ty(def_id));
+ let operand = OperandRef::from_immediate_or_packed_pair(&mut bx, static_, layout);
+ (bx, operand)
+ }
+ mir::Rvalue::Use(ref operand) => {
+ let operand = self.codegen_operand(&mut bx, operand);
+ (bx, operand)
+ }
+ mir::Rvalue::Repeat(..) | mir::Rvalue::Aggregate(..) => {
+ // According to `rvalue_creates_operand`, only ZST
+ // aggregate rvalues are allowed to be operands.
+ let ty = rvalue.ty(self.mir, self.cx.tcx());
+ let operand =
+ OperandRef::new_zst(&mut bx, self.cx.layout_of(self.monomorphize(ty)));
+ (bx, operand)
+ }
+ mir::Rvalue::ShallowInitBox(ref operand, content_ty) => {
+ let operand = self.codegen_operand(&mut bx, operand);
+ let lloperand = operand.immediate();
+
+ let content_ty = self.monomorphize(content_ty);
+ let box_layout = bx.cx().layout_of(bx.tcx().mk_box(content_ty));
+ let llty_ptr = bx.cx().backend_type(box_layout);
+
+ let val = bx.pointercast(lloperand, llty_ptr);
+ let operand = OperandRef { val: OperandValue::Immediate(val), layout: box_layout };
+ (bx, operand)
+ }
+ }
+ }
+
+ fn evaluate_array_len(&mut self, bx: &mut Bx, place: mir::Place<'tcx>) -> Bx::Value {
+ // ZST are passed as operands and require special handling
+ // because codegen_place() panics if Local is operand.
+ if let Some(index) = place.as_local() {
+ if let LocalRef::Operand(Some(op)) = self.locals[index] {
+ if let ty::Array(_, n) = op.layout.ty.kind() {
+ let n = n.eval_usize(bx.cx().tcx(), ty::ParamEnv::reveal_all());
+ return bx.cx().const_usize(n);
+ }
+ }
+ }
+ // use common size calculation for non zero-sized types
+ let cg_value = self.codegen_place(bx, place.as_ref());
+ cg_value.len(bx.cx())
+ }
+
+ /// Codegen an `Rvalue::AddressOf` or `Rvalue::Ref`
+ fn codegen_place_to_pointer(
+ &mut self,
+ mut bx: Bx,
+ place: mir::Place<'tcx>,
+ mk_ptr_ty: impl FnOnce(TyCtxt<'tcx>, Ty<'tcx>) -> Ty<'tcx>,
+ ) -> (Bx, OperandRef<'tcx, Bx::Value>) {
+ let cg_place = self.codegen_place(&mut bx, place.as_ref());
+
+ let ty = cg_place.layout.ty;
+
+ // Note: places are indirect, so storing the `llval` into the
+ // destination effectively creates a reference.
+ let val = if !bx.cx().type_has_metadata(ty) {
+ OperandValue::Immediate(cg_place.llval)
+ } else {
+ OperandValue::Pair(cg_place.llval, cg_place.llextra.unwrap())
+ };
+ (bx, OperandRef { val, layout: self.cx.layout_of(mk_ptr_ty(self.cx.tcx(), ty)) })
+ }
+
+ pub fn codegen_scalar_binop(
+ &mut self,
+ bx: &mut Bx,
+ op: mir::BinOp,
+ lhs: Bx::Value,
+ rhs: Bx::Value,
+ input_ty: Ty<'tcx>,
+ ) -> Bx::Value {
+ let is_float = input_ty.is_floating_point();
+ let is_signed = input_ty.is_signed();
+ match op {
+ mir::BinOp::Add => {
+ if is_float {
+ bx.fadd(lhs, rhs)
+ } else {
+ bx.add(lhs, rhs)
+ }
+ }
+ mir::BinOp::Sub => {
+ if is_float {
+ bx.fsub(lhs, rhs)
+ } else {
+ bx.sub(lhs, rhs)
+ }
+ }
+ mir::BinOp::Mul => {
+ if is_float {
+ bx.fmul(lhs, rhs)
+ } else {
+ bx.mul(lhs, rhs)
+ }
+ }
+ mir::BinOp::Div => {
+ if is_float {
+ bx.fdiv(lhs, rhs)
+ } else if is_signed {
+ bx.sdiv(lhs, rhs)
+ } else {
+ bx.udiv(lhs, rhs)
+ }
+ }
+ mir::BinOp::Rem => {
+ if is_float {
+ bx.frem(lhs, rhs)
+ } else if is_signed {
+ bx.srem(lhs, rhs)
+ } else {
+ bx.urem(lhs, rhs)
+ }
+ }
+ mir::BinOp::BitOr => bx.or(lhs, rhs),
+ mir::BinOp::BitAnd => bx.and(lhs, rhs),
+ mir::BinOp::BitXor => bx.xor(lhs, rhs),
+ mir::BinOp::Offset => {
+ let pointee_type = input_ty
+ .builtin_deref(true)
+ .unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty))
+ .ty;
+ let llty = bx.cx().backend_type(bx.cx().layout_of(pointee_type));
+ bx.inbounds_gep(llty, lhs, &[rhs])
+ }
+ mir::BinOp::Shl => common::build_unchecked_lshift(bx, lhs, rhs),
+ mir::BinOp::Shr => common::build_unchecked_rshift(bx, input_ty, lhs, rhs),
+ mir::BinOp::Ne
+ | mir::BinOp::Lt
+ | mir::BinOp::Gt
+ | mir::BinOp::Eq
+ | mir::BinOp::Le
+ | mir::BinOp::Ge => {
+ if is_float {
+ bx.fcmp(base::bin_op_to_fcmp_predicate(op.to_hir_binop()), lhs, rhs)
+ } else {
+ bx.icmp(base::bin_op_to_icmp_predicate(op.to_hir_binop(), is_signed), lhs, rhs)
+ }
+ }
+ }
+ }
+
+ pub fn codegen_fat_ptr_binop(
+ &mut self,
+ bx: &mut Bx,
+ op: mir::BinOp,
+ lhs_addr: Bx::Value,
+ lhs_extra: Bx::Value,
+ rhs_addr: Bx::Value,
+ rhs_extra: Bx::Value,
+ _input_ty: Ty<'tcx>,
+ ) -> Bx::Value {
+ match op {
+ mir::BinOp::Eq => {
+ let lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr);
+ let rhs = bx.icmp(IntPredicate::IntEQ, lhs_extra, rhs_extra);
+ bx.and(lhs, rhs)
+ }
+ mir::BinOp::Ne => {
+ let lhs = bx.icmp(IntPredicate::IntNE, lhs_addr, rhs_addr);
+ let rhs = bx.icmp(IntPredicate::IntNE, lhs_extra, rhs_extra);
+ bx.or(lhs, rhs)
+ }
+ mir::BinOp::Le | mir::BinOp::Lt | mir::BinOp::Ge | mir::BinOp::Gt => {
+ // a OP b ~ a.0 STRICT(OP) b.0 | (a.0 == b.0 && a.1 OP a.1)
+ let (op, strict_op) = match op {
+ mir::BinOp::Lt => (IntPredicate::IntULT, IntPredicate::IntULT),
+ mir::BinOp::Le => (IntPredicate::IntULE, IntPredicate::IntULT),
+ mir::BinOp::Gt => (IntPredicate::IntUGT, IntPredicate::IntUGT),
+ mir::BinOp::Ge => (IntPredicate::IntUGE, IntPredicate::IntUGT),
+ _ => bug!(),
+ };
+ let lhs = bx.icmp(strict_op, lhs_addr, rhs_addr);
+ let and_lhs = bx.icmp(IntPredicate::IntEQ, lhs_addr, rhs_addr);
+ let and_rhs = bx.icmp(op, lhs_extra, rhs_extra);
+ let rhs = bx.and(and_lhs, and_rhs);
+ bx.or(lhs, rhs)
+ }
+ _ => {
+ bug!("unexpected fat ptr binop");
+ }
+ }
+ }
+
+ pub fn codegen_scalar_checked_binop(
+ &mut self,
+ bx: &mut Bx,
+ op: mir::BinOp,
+ lhs: Bx::Value,
+ rhs: Bx::Value,
+ input_ty: Ty<'tcx>,
+ ) -> OperandValue<Bx::Value> {
+ // This case can currently arise only from functions marked
+ // with #[rustc_inherit_overflow_checks] and inlined from
+ // another crate (mostly core::num generic/#[inline] fns),
+ // while the current crate doesn't use overflow checks.
+ if !bx.cx().check_overflow() {
+ let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty);
+ return OperandValue::Pair(val, bx.cx().const_bool(false));
+ }
+
+ let (val, of) = match op {
+ // These are checked using intrinsics
+ mir::BinOp::Add | mir::BinOp::Sub | mir::BinOp::Mul => {
+ let oop = match op {
+ mir::BinOp::Add => OverflowOp::Add,
+ mir::BinOp::Sub => OverflowOp::Sub,
+ mir::BinOp::Mul => OverflowOp::Mul,
+ _ => unreachable!(),
+ };
+ bx.checked_binop(oop, input_ty, lhs, rhs)
+ }
+ mir::BinOp::Shl | mir::BinOp::Shr => {
+ let lhs_llty = bx.cx().val_ty(lhs);
+ let rhs_llty = bx.cx().val_ty(rhs);
+ let invert_mask = common::shift_mask_val(bx, lhs_llty, rhs_llty, true);
+ let outer_bits = bx.and(rhs, invert_mask);
+
+ let of = bx.icmp(IntPredicate::IntNE, outer_bits, bx.cx().const_null(rhs_llty));
+ let val = self.codegen_scalar_binop(bx, op, lhs, rhs, input_ty);
+
+ (val, of)
+ }
+ _ => bug!("Operator `{:?}` is not a checkable operator", op),
+ };
+
+ OperandValue::Pair(val, of)
+ }
+}
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ pub fn rvalue_creates_operand(&self, rvalue: &mir::Rvalue<'tcx>, span: Span) -> bool {
+ match *rvalue {
+ mir::Rvalue::Ref(..) |
+ mir::Rvalue::CopyForDeref(..) |
+ mir::Rvalue::AddressOf(..) |
+ mir::Rvalue::Len(..) |
+ mir::Rvalue::Cast(..) | // (*)
+ mir::Rvalue::ShallowInitBox(..) | // (*)
+ mir::Rvalue::BinaryOp(..) |
+ mir::Rvalue::CheckedBinaryOp(..) |
+ mir::Rvalue::UnaryOp(..) |
+ mir::Rvalue::Discriminant(..) |
+ mir::Rvalue::NullaryOp(..) |
+ mir::Rvalue::ThreadLocalRef(_) |
+ mir::Rvalue::Use(..) => // (*)
+ true,
+ mir::Rvalue::Repeat(..) |
+ mir::Rvalue::Aggregate(..) => {
+ let ty = rvalue.ty(self.mir, self.cx.tcx());
+ let ty = self.monomorphize(ty);
+ self.cx.spanned_layout_of(ty, span).is_zst()
+ }
+ }
+
+ // (*) this is only true if the type is suitable
+ }
+}
diff --git a/compiler/rustc_codegen_ssa/src/mir/statement.rs b/compiler/rustc_codegen_ssa/src/mir/statement.rs
new file mode 100644
index 000000000..f452f2988
--- /dev/null
+++ b/compiler/rustc_codegen_ssa/src/mir/statement.rs
@@ -0,0 +1,102 @@
+use rustc_middle::mir;
+
+use super::FunctionCx;
+use super::LocalRef;
+use crate::traits::BuilderMethods;
+use crate::traits::*;
+
+impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
+ #[instrument(level = "debug", skip(self, bx))]
+ pub fn codegen_statement(&mut self, mut bx: Bx, statement: &mir::Statement<'tcx>) -> Bx {
+ self.set_debug_loc(&mut bx, statement.source_info);
+ match statement.kind {
+ mir::StatementKind::Assign(box (ref place, ref rvalue)) => {
+ if let Some(index) = place.as_local() {
+ match self.locals[index] {
+ LocalRef::Place(cg_dest) => self.codegen_rvalue(bx, cg_dest, rvalue),
+ LocalRef::UnsizedPlace(cg_indirect_dest) => {
+ self.codegen_rvalue_unsized(bx, cg_indirect_dest, rvalue)
+ }
+ LocalRef::Operand(None) => {
+ let (mut bx, operand) = self.codegen_rvalue_operand(bx, rvalue);
+ self.locals[index] = LocalRef::Operand(Some(operand));
+ self.debug_introduce_local(&mut bx, index);
+ bx
+ }
+ LocalRef::Operand(Some(op)) => {
+ if !op.layout.is_zst() {
+ span_bug!(
+ statement.source_info.span,
+ "operand {:?} already assigned",
+ rvalue
+ );
+ }
+
+ // If the type is zero-sized, it's already been set here,
+ // but we still need to make sure we codegen the operand
+ self.codegen_rvalue_operand(bx, rvalue).0
+ }
+ }
+ } else {
+ let cg_dest = self.codegen_place(&mut bx, place.as_ref());
+ self.codegen_rvalue(bx, cg_dest, rvalue)
+ }
+ }
+ mir::StatementKind::SetDiscriminant { box ref place, variant_index } => {
+ self.codegen_place(&mut bx, place.as_ref())
+ .codegen_set_discr(&mut bx, variant_index);
+ bx
+ }
+ mir::StatementKind::Deinit(..) => {
+ // For now, don't codegen this to anything. In the future it may be worth
+ // experimenting with what kind of information we can emit to LLVM without hurting
+ // perf here
+ bx
+ }
+ mir::StatementKind::StorageLive(local) => {
+ if let LocalRef::Place(cg_place) = self.locals[local] {
+ cg_place.storage_live(&mut bx);
+ } else if let LocalRef::UnsizedPlace(cg_indirect_place) = self.locals[local] {
+ cg_indirect_place.storage_live(&mut bx);
+ }
+ bx
+ }
+ mir::StatementKind::StorageDead(local) => {
+ if let LocalRef::Place(cg_place) = self.locals[local] {
+ cg_place.storage_dead(&mut bx);
+ } else if let LocalRef::UnsizedPlace(cg_indirect_place) = self.locals[local] {
+ cg_indirect_place.storage_dead(&mut bx);
+ }
+ bx
+ }
+ mir::StatementKind::Coverage(box ref coverage) => {
+ self.codegen_coverage(&mut bx, coverage.clone(), statement.source_info.scope);
+ bx
+ }
+ mir::StatementKind::CopyNonOverlapping(box mir::CopyNonOverlapping {
+ ref src,
+ ref dst,
+ ref count,
+ }) => {
+ let dst_val = self.codegen_operand(&mut bx, dst);
+ let src_val = self.codegen_operand(&mut bx, src);
+ let count = self.codegen_operand(&mut bx, count).immediate();
+ let pointee_layout = dst_val
+ .layout
+ .pointee_info_at(&bx, rustc_target::abi::Size::ZERO)
+ .expect("Expected pointer");
+ let bytes = bx.mul(count, bx.const_usize(pointee_layout.size.bytes()));
+
+ let align = pointee_layout.align;
+ let dst = dst_val.immediate();
+ let src = src_val.immediate();
+ bx.memcpy(dst, align, src, align, bytes, crate::MemFlags::empty());
+ bx
+ }
+ mir::StatementKind::FakeRead(..)
+ | mir::StatementKind::Retag { .. }
+ | mir::StatementKind::AscribeUserType(..)
+ | mir::StatementKind::Nop => bx,
+ }
+ }
+}