From 698f8c2f01ea549d77d7dc3338a12e04c11057b9 Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Wed, 17 Apr 2024 14:02:58 +0200 Subject: Adding upstream version 1.64.0+dfsg1. Signed-off-by: Daniel Baumann --- compiler/rustc_codegen_llvm/src/abi.rs | 599 +++++ compiler/rustc_codegen_llvm/src/allocator.rs | 157 ++ compiler/rustc_codegen_llvm/src/asm.rs | 1037 ++++++++ compiler/rustc_codegen_llvm/src/attributes.rs | 449 ++++ compiler/rustc_codegen_llvm/src/back/archive.rs | 383 +++ compiler/rustc_codegen_llvm/src/back/lto.rs | 936 +++++++ compiler/rustc_codegen_llvm/src/back/profiling.rs | 58 + compiler/rustc_codegen_llvm/src/back/write.rs | 1212 ++++++++++ compiler/rustc_codegen_llvm/src/base.rs | 173 ++ compiler/rustc_codegen_llvm/src/builder.rs | 1508 ++++++++++++ compiler/rustc_codegen_llvm/src/callee.rs | 194 ++ compiler/rustc_codegen_llvm/src/common.rs | 359 +++ compiler/rustc_codegen_llvm/src/consts.rs | 577 +++++ compiler/rustc_codegen_llvm/src/context.rs | 1014 ++++++++ .../rustc_codegen_llvm/src/coverageinfo/mapgen.rs | 334 +++ .../rustc_codegen_llvm/src/coverageinfo/mod.rs | 385 +++ .../src/debuginfo/create_scope_map.rs | 126 + compiler/rustc_codegen_llvm/src/debuginfo/doc.md | 131 + compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs | 120 + .../rustc_codegen_llvm/src/debuginfo/metadata.rs | 1618 +++++++++++++ .../src/debuginfo/metadata/enums/cpp_like.rs | 514 ++++ .../src/debuginfo/metadata/enums/mod.rs | 437 ++++ .../src/debuginfo/metadata/enums/native.rs | 441 ++++ .../src/debuginfo/metadata/type_map.rs | 267 ++ compiler/rustc_codegen_llvm/src/debuginfo/mod.rs | 614 +++++ .../rustc_codegen_llvm/src/debuginfo/namespace.rs | 48 + compiler/rustc_codegen_llvm/src/debuginfo/utils.rs | 99 + compiler/rustc_codegen_llvm/src/declare.rs | 146 ++ compiler/rustc_codegen_llvm/src/intrinsic.rs | 1924 +++++++++++++++ compiler/rustc_codegen_llvm/src/lib.rs | 442 ++++ compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs | 105 + compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs | 213 ++ compiler/rustc_codegen_llvm/src/llvm/ffi.rs | 2547 ++++++++++++++++++++ compiler/rustc_codegen_llvm/src/llvm/mod.rs | 318 +++ compiler/rustc_codegen_llvm/src/llvm_util.rs | 562 +++++ compiler/rustc_codegen_llvm/src/mono_item.rs | 150 ++ compiler/rustc_codegen_llvm/src/type_.rs | 319 +++ compiler/rustc_codegen_llvm/src/type_of.rs | 418 ++++ compiler/rustc_codegen_llvm/src/va_arg.rs | 214 ++ compiler/rustc_codegen_llvm/src/value.rs | 32 + 40 files changed, 21180 insertions(+) create mode 100644 compiler/rustc_codegen_llvm/src/abi.rs create mode 100644 compiler/rustc_codegen_llvm/src/allocator.rs create mode 100644 compiler/rustc_codegen_llvm/src/asm.rs create mode 100644 compiler/rustc_codegen_llvm/src/attributes.rs create mode 100644 compiler/rustc_codegen_llvm/src/back/archive.rs create mode 100644 compiler/rustc_codegen_llvm/src/back/lto.rs create mode 100644 compiler/rustc_codegen_llvm/src/back/profiling.rs create mode 100644 compiler/rustc_codegen_llvm/src/back/write.rs create mode 100644 compiler/rustc_codegen_llvm/src/base.rs create mode 100644 compiler/rustc_codegen_llvm/src/builder.rs create mode 100644 compiler/rustc_codegen_llvm/src/callee.rs create mode 100644 compiler/rustc_codegen_llvm/src/common.rs create mode 100644 compiler/rustc_codegen_llvm/src/consts.rs create mode 100644 compiler/rustc_codegen_llvm/src/context.rs create mode 100644 compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs create mode 100644 compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/doc.md create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/mod.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs create mode 100644 compiler/rustc_codegen_llvm/src/debuginfo/utils.rs create mode 100644 compiler/rustc_codegen_llvm/src/declare.rs create mode 100644 compiler/rustc_codegen_llvm/src/intrinsic.rs create mode 100644 compiler/rustc_codegen_llvm/src/lib.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm/ffi.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm/mod.rs create mode 100644 compiler/rustc_codegen_llvm/src/llvm_util.rs create mode 100644 compiler/rustc_codegen_llvm/src/mono_item.rs create mode 100644 compiler/rustc_codegen_llvm/src/type_.rs create mode 100644 compiler/rustc_codegen_llvm/src/type_of.rs create mode 100644 compiler/rustc_codegen_llvm/src/va_arg.rs create mode 100644 compiler/rustc_codegen_llvm/src/value.rs (limited to 'compiler/rustc_codegen_llvm/src') diff --git a/compiler/rustc_codegen_llvm/src/abi.rs b/compiler/rustc_codegen_llvm/src/abi.rs new file mode 100644 index 000000000..9eb3574e7 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/abi.rs @@ -0,0 +1,599 @@ +use crate::attributes; +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::llvm::{self, Attribute, AttributePlace}; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; + +use rustc_codegen_ssa::mir::operand::OperandValue; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::MemFlags; +use rustc_middle::bug; +use rustc_middle::ty::layout::LayoutOf; +pub use rustc_middle::ty::layout::{FAT_PTR_ADDR, FAT_PTR_EXTRA}; +use rustc_middle::ty::Ty; +use rustc_session::config; +use rustc_target::abi::call::ArgAbi; +pub use rustc_target::abi::call::*; +use rustc_target::abi::{self, HasDataLayout, Int}; +pub use rustc_target::spec::abi::Abi; + +use libc::c_uint; +use smallvec::SmallVec; + +pub trait ArgAttributesExt { + fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value); + fn apply_attrs_to_callsite( + &self, + idx: AttributePlace, + cx: &CodegenCx<'_, '_>, + callsite: &Value, + ); +} + +fn should_use_mutable_noalias(cx: &CodegenCx<'_, '_>) -> bool { + // LLVM prior to version 12 had known miscompiles in the presence of + // noalias attributes (see #54878), but we don't support earlier + // versions at all anymore. We now enable mutable noalias by default. + cx.tcx.sess.opts.unstable_opts.mutable_noalias.unwrap_or(true) +} + +const ABI_AFFECTING_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 1] = + [(ArgAttribute::InReg, llvm::AttributeKind::InReg)]; + +const OPTIMIZATION_ATTRIBUTES: [(ArgAttribute, llvm::AttributeKind); 5] = [ + (ArgAttribute::NoAlias, llvm::AttributeKind::NoAlias), + (ArgAttribute::NoCapture, llvm::AttributeKind::NoCapture), + (ArgAttribute::NonNull, llvm::AttributeKind::NonNull), + (ArgAttribute::ReadOnly, llvm::AttributeKind::ReadOnly), + (ArgAttribute::NoUndef, llvm::AttributeKind::NoUndef), +]; + +fn get_attrs<'ll>(this: &ArgAttributes, cx: &CodegenCx<'ll, '_>) -> SmallVec<[&'ll Attribute; 8]> { + let mut regular = this.regular; + + let mut attrs = SmallVec::new(); + + // ABI-affecting attributes must always be applied + for (attr, llattr) in ABI_AFFECTING_ATTRIBUTES { + if regular.contains(attr) { + attrs.push(llattr.create_attr(cx.llcx)); + } + } + if let Some(align) = this.pointee_align { + attrs.push(llvm::CreateAlignmentAttr(cx.llcx, align.bytes())); + } + match this.arg_ext { + ArgExtension::None => {} + ArgExtension::Zext => attrs.push(llvm::AttributeKind::ZExt.create_attr(cx.llcx)), + ArgExtension::Sext => attrs.push(llvm::AttributeKind::SExt.create_attr(cx.llcx)), + } + + // Only apply remaining attributes when optimizing + if cx.sess().opts.optimize != config::OptLevel::No { + let deref = this.pointee_size.bytes(); + if deref != 0 { + if regular.contains(ArgAttribute::NonNull) { + attrs.push(llvm::CreateDereferenceableAttr(cx.llcx, deref)); + } else { + attrs.push(llvm::CreateDereferenceableOrNullAttr(cx.llcx, deref)); + } + regular -= ArgAttribute::NonNull; + } + for (attr, llattr) in OPTIMIZATION_ATTRIBUTES { + if regular.contains(attr) { + attrs.push(llattr.create_attr(cx.llcx)); + } + } + if regular.contains(ArgAttribute::NoAliasMutRef) && should_use_mutable_noalias(cx) { + attrs.push(llvm::AttributeKind::NoAlias.create_attr(cx.llcx)); + } + } + + attrs +} + +impl ArgAttributesExt for ArgAttributes { + fn apply_attrs_to_llfn(&self, idx: AttributePlace, cx: &CodegenCx<'_, '_>, llfn: &Value) { + let attrs = get_attrs(self, cx); + attributes::apply_to_llfn(llfn, idx, &attrs); + } + + fn apply_attrs_to_callsite( + &self, + idx: AttributePlace, + cx: &CodegenCx<'_, '_>, + callsite: &Value, + ) { + let attrs = get_attrs(self, cx); + attributes::apply_to_callsite(callsite, idx, &attrs); + } +} + +pub trait LlvmType { + fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type; +} + +impl LlvmType for Reg { + fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { + match self.kind { + RegKind::Integer => cx.type_ix(self.size.bits()), + RegKind::Float => match self.size.bits() { + 32 => cx.type_f32(), + 64 => cx.type_f64(), + _ => bug!("unsupported float: {:?}", self), + }, + RegKind::Vector => cx.type_vector(cx.type_i8(), self.size.bytes()), + } + } +} + +impl LlvmType for CastTarget { + fn llvm_type<'ll>(&self, cx: &CodegenCx<'ll, '_>) -> &'ll Type { + let rest_ll_unit = self.rest.unit.llvm_type(cx); + let (rest_count, rem_bytes) = if self.rest.unit.size.bytes() == 0 { + (0, 0) + } else { + ( + self.rest.total.bytes() / self.rest.unit.size.bytes(), + self.rest.total.bytes() % self.rest.unit.size.bytes(), + ) + }; + + if self.prefix.iter().all(|x| x.is_none()) { + // Simplify to a single unit when there is no prefix and size <= unit size + if self.rest.total <= self.rest.unit.size { + return rest_ll_unit; + } + + // Simplify to array when all chunks are the same size and type + if rem_bytes == 0 { + return cx.type_array(rest_ll_unit, rest_count); + } + } + + // Create list of fields in the main structure + let mut args: Vec<_> = self + .prefix + .iter() + .flat_map(|option_reg| option_reg.map(|reg| reg.llvm_type(cx))) + .chain((0..rest_count).map(|_| rest_ll_unit)) + .collect(); + + // Append final integer + if rem_bytes != 0 { + // Only integers can be really split further. + assert_eq!(self.rest.unit.kind, RegKind::Integer); + args.push(cx.type_ix(rem_bytes * 8)); + } + + cx.type_struct(&args, false) + } +} + +pub trait ArgAbiExt<'ll, 'tcx> { + fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn store( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + val: &'ll Value, + dst: PlaceRef<'tcx, &'ll Value>, + ); + fn store_fn_arg( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + idx: &mut usize, + dst: PlaceRef<'tcx, &'ll Value>, + ); +} + +impl<'ll, 'tcx> ArgAbiExt<'ll, 'tcx> for ArgAbi<'tcx, Ty<'tcx>> { + /// Gets the LLVM type for a place of the original Rust type of + /// this argument/return, i.e., the result of `type_of::type_of`. + fn memory_ty(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + self.layout.llvm_type(cx) + } + + /// Stores a direct/indirect value described by this ArgAbi into a + /// place for the original Rust type of this argument/return. + /// Can be used for both storing formal arguments into Rust variables + /// or results of call/invoke instructions into their destinations. + fn store( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + val: &'ll Value, + dst: PlaceRef<'tcx, &'ll Value>, + ) { + if self.is_ignore() { + return; + } + if self.is_sized_indirect() { + OperandValue::Ref(val, None, self.layout.align.abi).store(bx, dst) + } else if self.is_unsized_indirect() { + bug!("unsized `ArgAbi` must be handled through `store_fn_arg`"); + } else if let PassMode::Cast(cast) = self.mode { + // FIXME(eddyb): Figure out when the simpler Store is safe, clang + // uses it for i16 -> {i8, i8}, but not for i24 -> {i8, i8, i8}. + let can_store_through_cast_ptr = false; + if can_store_through_cast_ptr { + let cast_ptr_llty = bx.type_ptr_to(cast.llvm_type(bx)); + let cast_dst = bx.pointercast(dst.llval, cast_ptr_llty); + bx.store(val, cast_dst, self.layout.align.abi); + } else { + // The actual return type is a struct, but the ABI + // adaptation code has cast it into some scalar type. The + // code that follows is the only reliable way I have + // found to do a transform like i64 -> {i32,i32}. + // Basically we dump the data onto the stack then memcpy it. + // + // Other approaches I tried: + // - Casting rust ret pointer to the foreign type and using Store + // is (a) unsafe if size of foreign type > size of rust type and + // (b) runs afoul of strict aliasing rules, yielding invalid + // assembly under -O (specifically, the store gets removed). + // - Truncating foreign type to correct integral type and then + // bitcasting to the struct type yields invalid cast errors. + + // We instead thus allocate some scratch space... + let scratch_size = cast.size(bx); + let scratch_align = cast.align(bx); + let llscratch = bx.alloca(cast.llvm_type(bx), scratch_align); + bx.lifetime_start(llscratch, scratch_size); + + // ... where we first store the value... + bx.store(val, llscratch, scratch_align); + + // ... and then memcpy it to the intended destination. + bx.memcpy( + dst.llval, + self.layout.align.abi, + llscratch, + scratch_align, + bx.const_usize(self.layout.size.bytes()), + MemFlags::empty(), + ); + + bx.lifetime_end(llscratch, scratch_size); + } + } else { + OperandValue::Immediate(val).store(bx, dst); + } + } + + fn store_fn_arg( + &self, + bx: &mut Builder<'_, 'll, 'tcx>, + idx: &mut usize, + dst: PlaceRef<'tcx, &'ll Value>, + ) { + let mut next = || { + let val = llvm::get_param(bx.llfn(), *idx as c_uint); + *idx += 1; + val + }; + match self.mode { + PassMode::Ignore => {} + PassMode::Pair(..) => { + OperandValue::Pair(next(), next()).store(bx, dst); + } + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + OperandValue::Ref(next(), Some(next()), self.layout.align.abi).store(bx, dst); + } + PassMode::Direct(_) + | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } + | PassMode::Cast(_) => { + let next_arg = next(); + self.store(bx, next_arg, dst); + } + } + } +} + +impl<'ll, 'tcx> ArgAbiMethods<'tcx> for Builder<'_, 'll, 'tcx> { + fn store_fn_arg( + &mut self, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + idx: &mut usize, + dst: PlaceRef<'tcx, Self::Value>, + ) { + arg_abi.store_fn_arg(self, idx, dst) + } + fn store_arg( + &mut self, + arg_abi: &ArgAbi<'tcx, Ty<'tcx>>, + val: &'ll Value, + dst: PlaceRef<'tcx, &'ll Value>, + ) { + arg_abi.store(self, val, dst) + } + fn arg_memory_ty(&self, arg_abi: &ArgAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + arg_abi.memory_ty(self) + } +} + +pub trait FnAbiLlvmExt<'ll, 'tcx> { + fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type; + fn llvm_cconv(&self) -> llvm::CallConv; + fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value); + fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value); +} + +impl<'ll, 'tcx> FnAbiLlvmExt<'ll, 'tcx> for FnAbi<'tcx, Ty<'tcx>> { + fn llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + // Ignore "extra" args from the call site for C variadic functions. + // Only the "fixed" args are part of the LLVM function signature. + let args = if self.c_variadic { &self.args[..self.fixed_count] } else { &self.args }; + + let args_capacity: usize = args.iter().map(|arg| + if arg.pad.is_some() { 1 } else { 0 } + + if let PassMode::Pair(_, _) = arg.mode { 2 } else { 1 } + ).sum(); + let mut llargument_tys = Vec::with_capacity( + if let PassMode::Indirect { .. } = self.ret.mode { 1 } else { 0 } + args_capacity, + ); + + let llreturn_ty = match self.ret.mode { + PassMode::Ignore => cx.type_void(), + PassMode::Direct(_) | PassMode::Pair(..) => self.ret.layout.immediate_llvm_type(cx), + PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Indirect { .. } => { + llargument_tys.push(cx.type_ptr_to(self.ret.memory_ty(cx))); + cx.type_void() + } + }; + + for arg in args { + // add padding + if let Some(ty) = arg.pad { + llargument_tys.push(ty.llvm_type(cx)); + } + + let llarg_ty = match arg.mode { + PassMode::Ignore => continue, + PassMode::Direct(_) => arg.layout.immediate_llvm_type(cx), + PassMode::Pair(..) => { + llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 0, true)); + llargument_tys.push(arg.layout.scalar_pair_element_llvm_type(cx, 1, true)); + continue; + } + PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + let ptr_ty = cx.tcx.mk_mut_ptr(arg.layout.ty); + let ptr_layout = cx.layout_of(ptr_ty); + llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 0, true)); + llargument_tys.push(ptr_layout.scalar_pair_element_llvm_type(cx, 1, true)); + continue; + } + PassMode::Cast(cast) => cast.llvm_type(cx), + PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { + cx.type_ptr_to(arg.memory_ty(cx)) + } + }; + llargument_tys.push(llarg_ty); + } + + if self.c_variadic { + cx.type_variadic_func(&llargument_tys, llreturn_ty) + } else { + cx.type_func(&llargument_tys, llreturn_ty) + } + } + + fn ptr_to_llvm_type(&self, cx: &CodegenCx<'ll, 'tcx>) -> &'ll Type { + unsafe { + llvm::LLVMPointerType( + self.llvm_type(cx), + cx.data_layout().instruction_address_space.0 as c_uint, + ) + } + } + + fn llvm_cconv(&self) -> llvm::CallConv { + match self.conv { + Conv::C | Conv::Rust | Conv::CCmseNonSecureCall => llvm::CCallConv, + Conv::RustCold => llvm::ColdCallConv, + Conv::AmdGpuKernel => llvm::AmdGpuKernel, + Conv::AvrInterrupt => llvm::AvrInterrupt, + Conv::AvrNonBlockingInterrupt => llvm::AvrNonBlockingInterrupt, + Conv::ArmAapcs => llvm::ArmAapcsCallConv, + Conv::Msp430Intr => llvm::Msp430Intr, + Conv::PtxKernel => llvm::PtxKernel, + Conv::X86Fastcall => llvm::X86FastcallCallConv, + Conv::X86Intr => llvm::X86_Intr, + Conv::X86Stdcall => llvm::X86StdcallCallConv, + Conv::X86ThisCall => llvm::X86_ThisCall, + Conv::X86VectorCall => llvm::X86_VectorCall, + Conv::X86_64SysV => llvm::X86_64_SysV, + Conv::X86_64Win64 => llvm::X86_64_Win64, + } + } + + fn apply_attrs_llfn(&self, cx: &CodegenCx<'ll, 'tcx>, llfn: &'ll Value) { + let mut func_attrs = SmallVec::<[_; 2]>::new(); + if self.ret.layout.abi.is_uninhabited() { + func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(cx.llcx)); + } + if !self.can_unwind { + func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(cx.llcx)); + } + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &{ func_attrs }); + + let mut i = 0; + let mut apply = |attrs: &ArgAttributes| { + attrs.apply_attrs_to_llfn(llvm::AttributePlace::Argument(i), cx, llfn); + i += 1; + i - 1 + }; + match self.ret.mode { + PassMode::Direct(ref attrs) => { + attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); + } + PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + assert!(!on_stack); + let i = apply(attrs); + let sret = llvm::CreateStructRetAttr(cx.llcx, self.ret.layout.llvm_type(cx)); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[sret]); + } + PassMode::Cast(cast) => { + cast.attrs.apply_attrs_to_llfn(llvm::AttributePlace::ReturnValue, cx, llfn); + } + _ => {} + } + for arg in &self.args { + if arg.pad.is_some() { + apply(&ArgAttributes::new()); + } + match arg.mode { + PassMode::Ignore => {} + PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + let i = apply(attrs); + let byval = llvm::CreateByValAttr(cx.llcx, arg.layout.llvm_type(cx)); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Argument(i), &[byval]); + } + PassMode::Direct(ref attrs) + | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + apply(attrs); + } + PassMode::Indirect { ref attrs, extra_attrs: Some(ref extra_attrs), on_stack } => { + assert!(!on_stack); + apply(attrs); + apply(extra_attrs); + } + PassMode::Pair(ref a, ref b) => { + apply(a); + apply(b); + } + PassMode::Cast(cast) => { + apply(&cast.attrs); + } + } + } + } + + fn apply_attrs_callsite(&self, bx: &mut Builder<'_, 'll, 'tcx>, callsite: &'ll Value) { + let mut func_attrs = SmallVec::<[_; 2]>::new(); + if self.ret.layout.abi.is_uninhabited() { + func_attrs.push(llvm::AttributeKind::NoReturn.create_attr(bx.cx.llcx)); + } + if !self.can_unwind { + func_attrs.push(llvm::AttributeKind::NoUnwind.create_attr(bx.cx.llcx)); + } + attributes::apply_to_callsite(callsite, llvm::AttributePlace::Function, &{ func_attrs }); + + let mut i = 0; + let mut apply = |cx: &CodegenCx<'_, '_>, attrs: &ArgAttributes| { + attrs.apply_attrs_to_callsite(llvm::AttributePlace::Argument(i), cx, callsite); + i += 1; + i - 1 + }; + match self.ret.mode { + PassMode::Direct(ref attrs) => { + attrs.apply_attrs_to_callsite(llvm::AttributePlace::ReturnValue, bx.cx, callsite); + } + PassMode::Indirect { ref attrs, extra_attrs: _, on_stack } => { + assert!(!on_stack); + let i = apply(bx.cx, attrs); + let sret = llvm::CreateStructRetAttr(bx.cx.llcx, self.ret.layout.llvm_type(bx)); + attributes::apply_to_callsite(callsite, llvm::AttributePlace::Argument(i), &[sret]); + } + PassMode::Cast(cast) => { + cast.attrs.apply_attrs_to_callsite( + llvm::AttributePlace::ReturnValue, + &bx.cx, + callsite, + ); + } + _ => {} + } + if let abi::Abi::Scalar(scalar) = self.ret.layout.abi { + // If the value is a boolean, the range is 0..2 and that ultimately + // become 0..0 when the type becomes i1, which would be rejected + // by the LLVM verifier. + if let Int(..) = scalar.primitive() { + if !scalar.is_bool() && !scalar.is_always_valid(bx) { + bx.range_metadata(callsite, scalar.valid_range(bx)); + } + } + } + for arg in &self.args { + if arg.pad.is_some() { + apply(bx.cx, &ArgAttributes::new()); + } + match arg.mode { + PassMode::Ignore => {} + PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: true } => { + let i = apply(bx.cx, attrs); + let byval = llvm::CreateByValAttr(bx.cx.llcx, arg.layout.llvm_type(bx)); + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(i), + &[byval], + ); + } + PassMode::Direct(ref attrs) + | PassMode::Indirect { ref attrs, extra_attrs: None, on_stack: false } => { + apply(bx.cx, attrs); + } + PassMode::Indirect { + ref attrs, + extra_attrs: Some(ref extra_attrs), + on_stack: _, + } => { + apply(bx.cx, attrs); + apply(bx.cx, extra_attrs); + } + PassMode::Pair(ref a, ref b) => { + apply(bx.cx, a); + apply(bx.cx, b); + } + PassMode::Cast(cast) => { + apply(bx.cx, &cast.attrs); + } + } + } + + let cconv = self.llvm_cconv(); + if cconv != llvm::CCallConv { + llvm::SetInstructionCallConv(callsite, cconv); + } + + if self.conv == Conv::CCmseNonSecureCall { + // This will probably get ignored on all targets but those supporting the TrustZone-M + // extension (thumbv8m targets). + let cmse_nonsecure_call = llvm::CreateAttrString(bx.cx.llcx, "cmse_nonsecure_call"); + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Function, + &[cmse_nonsecure_call], + ); + } + + // Some intrinsics require that an elementtype attribute (with the pointee type of a + // pointer argument) is added to the callsite. + let element_type_index = unsafe { llvm::LLVMRustGetElementTypeArgIndex(callsite) }; + if element_type_index >= 0 { + let arg_ty = self.args[element_type_index as usize].layout.ty; + let pointee_ty = arg_ty.builtin_deref(true).expect("Must be pointer argument").ty; + let element_type_attr = unsafe { + llvm::LLVMRustCreateElementTypeAttr(bx.llcx, bx.layout_of(pointee_ty).llvm_type(bx)) + }; + attributes::apply_to_callsite( + callsite, + llvm::AttributePlace::Argument(element_type_index as u32), + &[element_type_attr], + ); + } + } +} + +impl<'tcx> AbiBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn apply_attrs_callsite(&mut self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, callsite: Self::Value) { + fn_abi.apply_attrs_callsite(self, callsite) + } + + fn get_param(&mut self, index: usize) -> Self::Value { + llvm::get_param(self.llfn(), index as c_uint) + } +} diff --git a/compiler/rustc_codegen_llvm/src/allocator.rs b/compiler/rustc_codegen_llvm/src/allocator.rs new file mode 100644 index 000000000..72961ae88 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/allocator.rs @@ -0,0 +1,157 @@ +use crate::attributes; +use libc::c_uint; +use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; +use rustc_middle::bug; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::{DebugInfo, OomStrategy}; +use rustc_span::symbol::sym; + +use crate::debuginfo; +use crate::llvm::{self, False, True}; +use crate::ModuleLlvm; + +pub(crate) unsafe fn codegen( + tcx: TyCtxt<'_>, + module_llvm: &mut ModuleLlvm, + module_name: &str, + kind: AllocatorKind, + has_alloc_error_handler: bool, +) { + let llcx = &*module_llvm.llcx; + let llmod = module_llvm.llmod(); + let usize = match tcx.sess.target.pointer_width { + 16 => llvm::LLVMInt16TypeInContext(llcx), + 32 => llvm::LLVMInt32TypeInContext(llcx), + 64 => llvm::LLVMInt64TypeInContext(llcx), + tws => bug!("Unsupported target word size for int: {}", tws), + }; + let i8 = llvm::LLVMInt8TypeInContext(llcx); + let i8p = llvm::LLVMPointerType(i8, 0); + let void = llvm::LLVMVoidTypeInContext(llcx); + + for method in ALLOCATOR_METHODS { + let mut args = Vec::with_capacity(method.inputs.len()); + for ty in method.inputs.iter() { + match *ty { + AllocatorTy::Layout => { + args.push(usize); // size + args.push(usize); // align + } + AllocatorTy::Ptr => args.push(i8p), + AllocatorTy::Usize => args.push(usize), + + AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + } + } + let output = match method.output { + AllocatorTy::ResultPtr => Some(i8p), + AllocatorTy::Unit => None, + + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; + let ty = llvm::LLVMFunctionType( + output.unwrap_or(void), + args.as_ptr(), + args.len() as c_uint, + False, + ); + let name = format!("__rust_{}", method.name); + let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); + + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + if tcx.sess.must_emit_unwind_tables() { + let uwtable = attributes::uwtable_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); + } + + let callee = kind.fn_name(method.name); + let callee = + llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); + llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); + + let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); + llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); + let args = args + .iter() + .enumerate() + .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) + .collect::>(); + let ret = llvm::LLVMRustBuildCall( + llbuilder, + ty, + callee, + args.as_ptr(), + args.len() as c_uint, + None, + ); + llvm::LLVMSetTailCall(ret, True); + if output.is_some() { + llvm::LLVMBuildRet(llbuilder, ret); + } else { + llvm::LLVMBuildRetVoid(llbuilder); + } + llvm::LLVMDisposeBuilder(llbuilder); + } + + // rust alloc error handler + let args = [usize, usize]; // size, align + + let ty = llvm::LLVMFunctionType(void, args.as_ptr(), args.len() as c_uint, False); + let name = "__rust_alloc_error_handler"; + let llfn = llvm::LLVMRustGetOrInsertFunction(llmod, name.as_ptr().cast(), name.len(), ty); + // -> ! DIFlagNoReturn + let no_return = llvm::AttributeKind::NoReturn.create_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[no_return]); + + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + if tcx.sess.must_emit_unwind_tables() { + let uwtable = attributes::uwtable_attr(llcx); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[uwtable]); + } + + let kind = if has_alloc_error_handler { AllocatorKind::Global } else { AllocatorKind::Default }; + let callee = kind.fn_name(sym::oom); + let callee = llvm::LLVMRustGetOrInsertFunction(llmod, callee.as_ptr().cast(), callee.len(), ty); + // -> ! DIFlagNoReturn + attributes::apply_to_llfn(callee, llvm::AttributePlace::Function, &[no_return]); + llvm::LLVMRustSetVisibility(callee, llvm::Visibility::Hidden); + + let llbb = llvm::LLVMAppendBasicBlockInContext(llcx, llfn, "entry\0".as_ptr().cast()); + + let llbuilder = llvm::LLVMCreateBuilderInContext(llcx); + llvm::LLVMPositionBuilderAtEnd(llbuilder, llbb); + let args = args + .iter() + .enumerate() + .map(|(i, _)| llvm::LLVMGetParam(llfn, i as c_uint)) + .collect::>(); + let ret = + llvm::LLVMRustBuildCall(llbuilder, ty, callee, args.as_ptr(), args.len() as c_uint, None); + llvm::LLVMSetTailCall(ret, True); + llvm::LLVMBuildRetVoid(llbuilder); + llvm::LLVMDisposeBuilder(llbuilder); + + // __rust_alloc_error_handler_should_panic + let name = OomStrategy::SYMBOL; + let ll_g = llvm::LLVMRustGetOrInsertGlobal(llmod, name.as_ptr().cast(), name.len(), i8); + if tcx.sess.target.default_hidden_visibility { + llvm::LLVMRustSetVisibility(ll_g, llvm::Visibility::Hidden); + } + let val = tcx.sess.opts.unstable_opts.oom.should_panic(); + let llval = llvm::LLVMConstInt(i8, val as u64, False); + llvm::LLVMSetInitializer(ll_g, llval); + + if tcx.sess.opts.debuginfo != DebugInfo::None { + let dbg_cx = debuginfo::CodegenUnitDebugContext::new(llmod); + debuginfo::metadata::build_compile_unit_di_node(tcx, module_name, &dbg_cx); + dbg_cx.finalize(tcx.sess); + } +} diff --git a/compiler/rustc_codegen_llvm/src/asm.rs b/compiler/rustc_codegen_llvm/src/asm.rs new file mode 100644 index 000000000..a53946995 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/asm.rs @@ -0,0 +1,1037 @@ +use crate::attributes; +use crate::builder::Builder; +use crate::common::Funclet; +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm_util; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; + +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_codegen_ssa::mir::operand::OperandValue; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::fx::FxHashMap; +use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::{bug, span_bug, ty::Instance}; +use rustc_span::{Pos, Span}; +use rustc_target::abi::*; +use rustc_target::asm::*; + +use libc::{c_char, c_uint}; +use smallvec::SmallVec; +use tracing::debug; + +impl<'ll, 'tcx> AsmBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> { + fn codegen_inline_asm( + &mut self, + template: &[InlineAsmTemplatePiece], + operands: &[InlineAsmOperandRef<'tcx, Self>], + options: InlineAsmOptions, + line_spans: &[Span], + instance: Instance<'_>, + dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>, + ) { + let asm_arch = self.tcx.sess.asm_arch.unwrap(); + + // Collect the types of output operands + let mut constraints = vec![]; + let mut clobbers = vec![]; + let mut output_types = vec![]; + let mut op_idx = FxHashMap::default(); + let mut clobbered_x87 = false; + for (idx, op) in operands.iter().enumerate() { + match *op { + InlineAsmOperandRef::Out { reg, late, place } => { + let is_target_supported = |reg_class: InlineAsmRegClass| { + for &(_, feature) in reg_class.supported_types(asm_arch) { + if let Some(feature) = feature { + let codegen_fn_attrs = self.tcx.codegen_fn_attrs(instance.def_id()); + if self.tcx.sess.target_features.contains(&feature) + || codegen_fn_attrs.target_features.contains(&feature) + { + return true; + } + } else { + // Register class is unconditionally supported + return true; + } + } + false + }; + + let mut layout = None; + let ty = if let Some(ref place) = place { + layout = Some(&place.layout); + llvm_fixup_output_type(self.cx, reg.reg_class(), &place.layout) + } else if matches!( + reg.reg_class(), + InlineAsmRegClass::X86( + X86InlineAsmRegClass::mmx_reg | X86InlineAsmRegClass::x87_reg + ) + ) { + // Special handling for x87/mmx registers: we always + // clobber the whole set if one register is marked as + // clobbered. This is due to the way LLVM handles the + // FP stack in inline assembly. + if !clobbered_x87 { + clobbered_x87 = true; + clobbers.push("~{st}".to_string()); + for i in 1..=7 { + clobbers.push(format!("~{{st({})}}", i)); + } + } + continue; + } else if !is_target_supported(reg.reg_class()) + || reg.reg_class().is_clobber_only(asm_arch) + { + // We turn discarded outputs into clobber constraints + // if the target feature needed by the register class is + // disabled. This is necessary otherwise LLVM will try + // to actually allocate a register for the dummy output. + assert!(matches!(reg, InlineAsmRegOrRegClass::Reg(_))); + clobbers.push(format!("~{}", reg_to_llvm(reg, None))); + continue; + } else { + // If the output is discarded, we don't really care what + // type is used. We're just using this to tell LLVM to + // reserve the register. + dummy_output_type(self.cx, reg.reg_class()) + }; + output_types.push(ty); + op_idx.insert(idx, constraints.len()); + let prefix = if late { "=" } else { "=&" }; + constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, layout))); + } + InlineAsmOperandRef::InOut { reg, late, in_value, out_place } => { + let layout = if let Some(ref out_place) = out_place { + &out_place.layout + } else { + // LLVM required tied operands to have the same type, + // so we just use the type of the input. + &in_value.layout + }; + let ty = llvm_fixup_output_type(self.cx, reg.reg_class(), layout); + output_types.push(ty); + op_idx.insert(idx, constraints.len()); + let prefix = if late { "=" } else { "=&" }; + constraints.push(format!("{}{}", prefix, reg_to_llvm(reg, Some(layout)))); + } + _ => {} + } + } + + // Collect input operands + let mut inputs = vec![]; + for (idx, op) in operands.iter().enumerate() { + match *op { + InlineAsmOperandRef::In { reg, value } => { + let llval = + llvm_fixup_input(self, value.immediate(), reg.reg_class(), &value.layout); + inputs.push(llval); + op_idx.insert(idx, constraints.len()); + constraints.push(reg_to_llvm(reg, Some(&value.layout))); + } + InlineAsmOperandRef::InOut { reg, late: _, in_value, out_place: _ } => { + let value = llvm_fixup_input( + self, + in_value.immediate(), + reg.reg_class(), + &in_value.layout, + ); + inputs.push(value); + constraints.push(format!("{}", op_idx[&idx])); + } + InlineAsmOperandRef::SymFn { instance } => { + inputs.push(self.cx.get_fn(instance)); + op_idx.insert(idx, constraints.len()); + constraints.push("s".to_string()); + } + InlineAsmOperandRef::SymStatic { def_id } => { + inputs.push(self.cx.get_static(def_id)); + op_idx.insert(idx, constraints.len()); + constraints.push("s".to_string()); + } + _ => {} + } + } + + // Build the template string + let mut template_str = String::new(); + for piece in template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => { + if s.contains('$') { + for c in s.chars() { + if c == '$' { + template_str.push_str("$$"); + } else { + template_str.push(c); + } + } + } else { + template_str.push_str(s) + } + } + InlineAsmTemplatePiece::Placeholder { operand_idx, modifier, span: _ } => { + match operands[operand_idx] { + InlineAsmOperandRef::In { reg, .. } + | InlineAsmOperandRef::Out { reg, .. } + | InlineAsmOperandRef::InOut { reg, .. } => { + let modifier = modifier_to_llvm(asm_arch, reg.reg_class(), modifier); + if let Some(modifier) = modifier { + template_str.push_str(&format!( + "${{{}:{}}}", + op_idx[&operand_idx], modifier + )); + } else { + template_str.push_str(&format!("${{{}}}", op_idx[&operand_idx])); + } + } + InlineAsmOperandRef::Const { ref string } => { + // Const operands get injected directly into the template + template_str.push_str(string); + } + InlineAsmOperandRef::SymFn { .. } + | InlineAsmOperandRef::SymStatic { .. } => { + // Only emit the raw symbol name + template_str.push_str(&format!("${{{}:c}}", op_idx[&operand_idx])); + } + } + } + } + } + + constraints.append(&mut clobbers); + if !options.contains(InlineAsmOptions::PRESERVES_FLAGS) { + match asm_arch { + InlineAsmArch::AArch64 | InlineAsmArch::Arm => { + constraints.push("~{cc}".to_string()); + } + InlineAsmArch::X86 | InlineAsmArch::X86_64 => { + constraints.extend_from_slice(&[ + "~{dirflag}".to_string(), + "~{fpsr}".to_string(), + "~{flags}".to_string(), + ]); + } + InlineAsmArch::RiscV32 | InlineAsmArch::RiscV64 => { + constraints.extend_from_slice(&[ + "~{vtype}".to_string(), + "~{vl}".to_string(), + "~{vxsat}".to_string(), + "~{vxrm}".to_string(), + ]); + } + InlineAsmArch::Avr => { + constraints.push("~{sreg}".to_string()); + } + InlineAsmArch::Nvptx64 => {} + InlineAsmArch::PowerPC | InlineAsmArch::PowerPC64 => {} + InlineAsmArch::Hexagon => {} + InlineAsmArch::Mips | InlineAsmArch::Mips64 => {} + InlineAsmArch::S390x => {} + InlineAsmArch::SpirV => {} + InlineAsmArch::Wasm32 | InlineAsmArch::Wasm64 => {} + InlineAsmArch::Bpf => {} + InlineAsmArch::Msp430 => { + constraints.push("~{sr}".to_string()); + } + } + } + if !options.contains(InlineAsmOptions::NOMEM) { + // This is actually ignored by LLVM, but it's probably best to keep + // it just in case. LLVM instead uses the ReadOnly/ReadNone + // attributes on the call instruction to optimize. + constraints.push("~{memory}".to_string()); + } + let volatile = !options.contains(InlineAsmOptions::PURE); + let alignstack = !options.contains(InlineAsmOptions::NOSTACK); + let output_type = match &output_types[..] { + [] => self.type_void(), + [ty] => ty, + tys => self.type_struct(tys, false), + }; + let dialect = match asm_arch { + InlineAsmArch::X86 | InlineAsmArch::X86_64 + if !options.contains(InlineAsmOptions::ATT_SYNTAX) => + { + llvm::AsmDialect::Intel + } + _ => llvm::AsmDialect::Att, + }; + let result = inline_asm_call( + self, + &template_str, + &constraints.join(","), + &inputs, + output_type, + volatile, + alignstack, + dialect, + line_spans, + options.contains(InlineAsmOptions::MAY_UNWIND), + dest_catch_funclet, + ) + .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed")); + + let mut attrs = SmallVec::<[_; 2]>::new(); + if options.contains(InlineAsmOptions::PURE) { + if options.contains(InlineAsmOptions::NOMEM) { + attrs.push(llvm::AttributeKind::ReadNone.create_attr(self.cx.llcx)); + } else if options.contains(InlineAsmOptions::READONLY) { + attrs.push(llvm::AttributeKind::ReadOnly.create_attr(self.cx.llcx)); + } + attrs.push(llvm::AttributeKind::WillReturn.create_attr(self.cx.llcx)); + } else if options.contains(InlineAsmOptions::NOMEM) { + attrs.push(llvm::AttributeKind::InaccessibleMemOnly.create_attr(self.cx.llcx)); + } else { + // LLVM doesn't have an attribute to represent ReadOnly + SideEffect + } + attributes::apply_to_callsite(result, llvm::AttributePlace::Function, &{ attrs }); + + // Switch to the 'normal' basic block if we did an `invoke` instead of a `call` + if let Some((dest, _, _)) = dest_catch_funclet { + self.switch_to_block(dest); + } + + // Write results to outputs + for (idx, op) in operands.iter().enumerate() { + if let InlineAsmOperandRef::Out { reg, place: Some(place), .. } + | InlineAsmOperandRef::InOut { reg, out_place: Some(place), .. } = *op + { + let value = if output_types.len() == 1 { + result + } else { + self.extract_value(result, op_idx[&idx] as u64) + }; + let value = llvm_fixup_output(self, value, reg.reg_class(), &place.layout); + OperandValue::Immediate(value).store(self, place); + } + } + } +} + +impl<'tcx> AsmMethods<'tcx> for CodegenCx<'_, 'tcx> { + fn codegen_global_asm( + &self, + template: &[InlineAsmTemplatePiece], + operands: &[GlobalAsmOperandRef<'tcx>], + options: InlineAsmOptions, + _line_spans: &[Span], + ) { + let asm_arch = self.tcx.sess.asm_arch.unwrap(); + + // Default to Intel syntax on x86 + let intel_syntax = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64) + && !options.contains(InlineAsmOptions::ATT_SYNTAX); + + // Build the template string + let mut template_str = String::new(); + if intel_syntax { + template_str.push_str(".intel_syntax\n"); + } + for piece in template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => template_str.push_str(s), + InlineAsmTemplatePiece::Placeholder { operand_idx, modifier: _, span: _ } => { + match operands[operand_idx] { + GlobalAsmOperandRef::Const { ref string } => { + // Const operands get injected directly into the + // template. Note that we don't need to escape $ + // here unlike normal inline assembly. + template_str.push_str(string); + } + GlobalAsmOperandRef::SymFn { instance } => { + let llval = self.get_fn(instance); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } + GlobalAsmOperandRef::SymStatic { def_id } => { + let llval = self + .renamed_statics + .borrow() + .get(&def_id) + .copied() + .unwrap_or_else(|| self.get_static(def_id)); + self.add_compiler_used_global(llval); + let symbol = llvm::build_string(|s| unsafe { + llvm::LLVMRustGetMangledName(llval, s); + }) + .expect("symbol is not valid UTF-8"); + template_str.push_str(&symbol); + } + } + } + } + } + if intel_syntax { + template_str.push_str("\n.att_syntax\n"); + } + + unsafe { + llvm::LLVMRustAppendModuleInlineAsm( + self.llmod, + template_str.as_ptr().cast(), + template_str.len(), + ); + } + } +} + +pub(crate) fn inline_asm_call<'ll>( + bx: &mut Builder<'_, 'll, '_>, + asm: &str, + cons: &str, + inputs: &[&'ll Value], + output: &'ll llvm::Type, + volatile: bool, + alignstack: bool, + dia: llvm::AsmDialect, + line_spans: &[Span], + unwind: bool, + dest_catch_funclet: Option<( + &'ll llvm::BasicBlock, + &'ll llvm::BasicBlock, + Option<&Funclet<'ll>>, + )>, +) -> Option<&'ll Value> { + let volatile = if volatile { llvm::True } else { llvm::False }; + let alignstack = if alignstack { llvm::True } else { llvm::False }; + let can_throw = if unwind { llvm::True } else { llvm::False }; + + let argtys = inputs + .iter() + .map(|v| { + debug!("Asm Input Type: {:?}", *v); + bx.cx.val_ty(*v) + }) + .collect::>(); + + debug!("Asm Output Type: {:?}", output); + let fty = bx.cx.type_func(&argtys, output); + unsafe { + // Ask LLVM to verify that the constraints are well-formed. + let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len()); + debug!("constraint verification result: {:?}", constraints_ok); + if constraints_ok { + if unwind && llvm_util::get_version() < (13, 0, 0) { + bx.cx.sess().span_fatal( + line_spans[0], + "unwinding from inline assembly is only supported on llvm >= 13.", + ); + } + + let v = llvm::LLVMRustInlineAsm( + fty, + asm.as_ptr().cast(), + asm.len(), + cons.as_ptr().cast(), + cons.len(), + volatile, + alignstack, + dia, + can_throw, + ); + + let call = if let Some((dest, catch, funclet)) = dest_catch_funclet { + bx.invoke(fty, v, inputs, dest, catch, funclet) + } else { + bx.call(fty, v, inputs, None) + }; + + // Store mark in a metadata node so we can map LLVM errors + // back to source locations. See #17552. + let key = "srcloc"; + let kind = llvm::LLVMGetMDKindIDInContext( + bx.llcx, + key.as_ptr() as *const c_char, + key.len() as c_uint, + ); + + // srcloc contains one integer for each line of assembly code. + // Unfortunately this isn't enough to encode a full span so instead + // we just encode the start position of each line. + // FIXME: Figure out a way to pass the entire line spans. + let mut srcloc = vec![]; + if dia == llvm::AsmDialect::Intel && line_spans.len() > 1 { + // LLVM inserts an extra line to add the ".intel_syntax", so add + // a dummy srcloc entry for it. + // + // Don't do this if we only have 1 line span since that may be + // due to the asm template string coming from a macro. LLVM will + // default to the first srcloc for lines that don't have an + // associated srcloc. + srcloc.push(bx.const_i32(0)); + } + srcloc.extend(line_spans.iter().map(|span| bx.const_i32(span.lo().to_u32() as i32))); + let md = llvm::LLVMMDNodeInContext(bx.llcx, srcloc.as_ptr(), srcloc.len() as u32); + llvm::LLVMSetMetadata(call, kind, md); + + Some(call) + } else { + // LLVM has detected an issue with our constraints, bail out + None + } + } +} + +/// If the register is an xmm/ymm/zmm register then return its index. +fn xmm_reg_index(reg: InlineAsmReg) -> Option { + match reg { + InlineAsmReg::X86(reg) + if reg as u32 >= X86InlineAsmReg::xmm0 as u32 + && reg as u32 <= X86InlineAsmReg::xmm15 as u32 => + { + Some(reg as u32 - X86InlineAsmReg::xmm0 as u32) + } + InlineAsmReg::X86(reg) + if reg as u32 >= X86InlineAsmReg::ymm0 as u32 + && reg as u32 <= X86InlineAsmReg::ymm15 as u32 => + { + Some(reg as u32 - X86InlineAsmReg::ymm0 as u32) + } + InlineAsmReg::X86(reg) + if reg as u32 >= X86InlineAsmReg::zmm0 as u32 + && reg as u32 <= X86InlineAsmReg::zmm31 as u32 => + { + Some(reg as u32 - X86InlineAsmReg::zmm0 as u32) + } + _ => None, + } +} + +/// If the register is an AArch64 vector register then return its index. +fn a64_vreg_index(reg: InlineAsmReg) -> Option { + match reg { + InlineAsmReg::AArch64(reg) + if reg as u32 >= AArch64InlineAsmReg::v0 as u32 + && reg as u32 <= AArch64InlineAsmReg::v31 as u32 => + { + Some(reg as u32 - AArch64InlineAsmReg::v0 as u32) + } + _ => None, + } +} + +/// Converts a register class to an LLVM constraint code. +fn reg_to_llvm(reg: InlineAsmRegOrRegClass, layout: Option<&TyAndLayout<'_>>) -> String { + match reg { + // For vector registers LLVM wants the register name to match the type size. + InlineAsmRegOrRegClass::Reg(reg) => { + if let Some(idx) = xmm_reg_index(reg) { + let class = if let Some(layout) = layout { + match layout.size.bytes() { + 64 => 'z', + 32 => 'y', + _ => 'x', + } + } else { + // We use f32 as the type for discarded outputs + 'x' + }; + format!("{{{}mm{}}}", class, idx) + } else if let Some(idx) = a64_vreg_index(reg) { + let class = if let Some(layout) = layout { + match layout.size.bytes() { + 16 => 'q', + 8 => 'd', + 4 => 's', + 2 => 'h', + 1 => 'd', // We fixup i8 to i8x8 + _ => unreachable!(), + } + } else { + // We use i64x2 as the type for discarded outputs + 'q' + }; + format!("{{{}{}}}", class, idx) + } else if reg == InlineAsmReg::AArch64(AArch64InlineAsmReg::x30) { + // LLVM doesn't recognize x30 + "{lr}".to_string() + } else if reg == InlineAsmReg::Arm(ArmInlineAsmReg::r14) { + // LLVM doesn't recognize r14 + "{lr}".to_string() + } else { + format!("{{{}}}", reg.name()) + } + } + InlineAsmRegOrRegClass::RegClass(reg) => match reg { + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) => "w", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => "x", + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) => "t", + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => "x", + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) => "w", + InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => "f", + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => "h", + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => "r", + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => "l", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => "b", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => "f", + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) + | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => "f", + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => "Q", + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => "q", + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) => "x", + InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => "v", + InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => "^Yk", + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, + ) => unreachable!("clobber-only"), + InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => "r", + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => "w", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => "d", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => "r", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => "w", + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => "e", + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => "r", + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => "f", + InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => "r", + InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { + bug!("LLVM backend does not support SPIR-V") + } + InlineAsmRegClass::Err => unreachable!(), + } + .to_string(), + } +} + +/// Converts a modifier into LLVM's equivalent modifier. +fn modifier_to_llvm( + arch: InlineAsmArch, + reg: InlineAsmRegClass, + modifier: Option, +) -> Option { + match reg { + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => modifier, + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) + | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + if modifier == Some('v') { None } else { modifier } + } + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => None, + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => None, + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => Some('P'), + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { + if modifier.is_none() { + Some('q') + } else { + modifier + } + } + InlineAsmRegClass::Hexagon(_) => None, + InlineAsmRegClass::Mips(_) => None, + InlineAsmRegClass::Nvptx(_) => None, + InlineAsmRegClass::PowerPC(_) => None, + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) + | InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => None, + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => match modifier { + None if arch == InlineAsmArch::X86_64 => Some('q'), + None => Some('k'), + Some('l') => Some('b'), + Some('h') => Some('h'), + Some('x') => Some('w'), + Some('e') => Some('k'), + Some('r') => Some('q'), + _ => unreachable!(), + }, + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => None, + InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::xmm_reg) + | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::ymm_reg) + | InlineAsmRegClass::X86(reg @ X86InlineAsmRegClass::zmm_reg) => match (reg, modifier) { + (X86InlineAsmRegClass::xmm_reg, None) => Some('x'), + (X86InlineAsmRegClass::ymm_reg, None) => Some('t'), + (X86InlineAsmRegClass::zmm_reg, None) => Some('g'), + (_, Some('x')) => Some('x'), + (_, Some('y')) => Some('t'), + (_, Some('z')) => Some('g'), + _ => unreachable!(), + }, + InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => None, + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, + ) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => None, + InlineAsmRegClass::Bpf(_) => None, + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) + | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) + | InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => match modifier { + Some('h') => Some('B'), + Some('l') => Some('A'), + _ => None, + }, + InlineAsmRegClass::Avr(_) => None, + InlineAsmRegClass::S390x(_) => None, + InlineAsmRegClass::Msp430(_) => None, + InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { + bug!("LLVM backend does not support SPIR-V") + } + InlineAsmRegClass::Err => unreachable!(), + } +} + +/// Type to use for outputs that are discarded. It doesn't really matter what +/// the type is, as long as it is valid for the constraint code. +fn dummy_output_type<'ll>(cx: &CodegenCx<'ll, '_>, reg: InlineAsmRegClass) -> &'ll Type { + match reg { + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg) + | InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16) => { + cx.type_vector(cx.type_i64(), 2) + } + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::preg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg_low16) => cx.type_f32(), + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low16) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::dreg_low8) => cx.type_f64(), + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low8) + | InlineAsmRegClass::Arm(ArmInlineAsmRegClass::qreg_low4) => { + cx.type_vector(cx.type_i64(), 2) + } + InlineAsmRegClass::Hexagon(HexagonInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::Mips(MipsInlineAsmRegClass::freg) => cx.type_f32(), + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg16) => cx.type_i16(), + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg32) => cx.type_i32(), + InlineAsmRegClass::Nvptx(NvptxInlineAsmRegClass::reg64) => cx.type_i64(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::reg_nonzero) => cx.type_i32(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::freg) => cx.type_f64(), + InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::cr) + | InlineAsmRegClass::PowerPC(PowerPCInlineAsmRegClass::xer) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::freg) => cx.type_f32(), + InlineAsmRegClass::RiscV(RiscVInlineAsmRegClass::vreg) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd) => cx.type_i32(), + InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_byte) => cx.type_i8(), + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::ymm_reg) + | InlineAsmRegClass::X86(X86InlineAsmRegClass::zmm_reg) => cx.type_f32(), + InlineAsmRegClass::X86(X86InlineAsmRegClass::kreg) => cx.type_i16(), + InlineAsmRegClass::X86( + X86InlineAsmRegClass::x87_reg + | X86InlineAsmRegClass::mmx_reg + | X86InlineAsmRegClass::kreg0 + | X86InlineAsmRegClass::tmm_reg, + ) => { + unreachable!("clobber-only") + } + InlineAsmRegClass::Wasm(WasmInlineAsmRegClass::local) => cx.type_i32(), + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::reg) => cx.type_i64(), + InlineAsmRegClass::Bpf(BpfInlineAsmRegClass::wreg) => cx.type_i32(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg) => cx.type_i8(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_upper) => cx.type_i8(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_pair) => cx.type_i16(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_iw) => cx.type_i16(), + InlineAsmRegClass::Avr(AvrInlineAsmRegClass::reg_ptr) => cx.type_i16(), + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::reg) => cx.type_i32(), + InlineAsmRegClass::S390x(S390xInlineAsmRegClass::freg) => cx.type_f64(), + InlineAsmRegClass::Msp430(Msp430InlineAsmRegClass::reg) => cx.type_i16(), + InlineAsmRegClass::SpirV(SpirVInlineAsmRegClass::reg) => { + bug!("LLVM backend does not support SPIR-V") + } + InlineAsmRegClass::Err => unreachable!(), + } +} + +/// Helper function to get the LLVM type for a Scalar. Pointers are returned as +/// the equivalent integer type. +fn llvm_asm_scalar_type<'ll>(cx: &CodegenCx<'ll, '_>, scalar: Scalar) -> &'ll Type { + match scalar.primitive() { + Primitive::Int(Integer::I8, _) => cx.type_i8(), + Primitive::Int(Integer::I16, _) => cx.type_i16(), + Primitive::Int(Integer::I32, _) => cx.type_i32(), + Primitive::Int(Integer::I64, _) => cx.type_i64(), + Primitive::F32 => cx.type_f32(), + Primitive::F64 => cx.type_f64(), + Primitive::Pointer => cx.type_isize(), + _ => unreachable!(), + } +} + +/// Fix up an input value to work around LLVM bugs. +fn llvm_fixup_input<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + mut value: &'ll Value, + reg: InlineAsmRegClass, + layout: &TyAndLayout<'tcx>, +) -> &'ll Value { + match (reg, layout.abi) { + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + if let Primitive::Int(Integer::I8, _) = s.primitive() { + let vec_ty = bx.cx.type_vector(bx.cx.type_i8(), 8); + bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) + } else { + value + } + } + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { + let elem_ty = llvm_asm_scalar_type(bx.cx, s); + let count = 16 / layout.size.bytes(); + let vec_ty = bx.cx.type_vector(elem_ty, count); + if let Primitive::Pointer = s.primitive() { + value = bx.ptrtoint(value, bx.cx.type_isize()); + } + bx.insert_element(bx.const_undef(vec_ty), value, bx.const_i32(0)) + } + ( + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { + let elem_ty = llvm_asm_scalar_type(bx.cx, element); + let vec_ty = bx.cx.type_vector(elem_ty, count); + let indices: Vec<_> = (0..count * 2).map(|x| bx.const_i32(x as i32)).collect(); + bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) + } + (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + if s.primitive() == Primitive::F64 => + { + bx.bitcast(value, bx.cx.type_i64()) + } + ( + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), + Abi::Vector { .. }, + ) if layout.size.bytes() == 64 => bx.bitcast(value, bx.cx.type_vector(bx.cx.type_f64(), 8)), + ( + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I32, _) = s.primitive() { + bx.bitcast(value, bx.cx.type_f32()) + } else { + value + } + } + ( + InlineAsmRegClass::Arm( + ArmInlineAsmRegClass::dreg + | ArmInlineAsmRegClass::dreg_low8 + | ArmInlineAsmRegClass::dreg_low16, + ), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I64, _) = s.primitive() { + bx.bitcast(value, bx.cx.type_f64()) + } else { + value + } + } + (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + match s.primitive() { + // MIPS only supports register-length arithmetics. + Primitive::Int(Integer::I8 | Integer::I16, _) => bx.zext(value, bx.cx.type_i32()), + Primitive::F32 => bx.bitcast(value, bx.cx.type_i32()), + Primitive::F64 => bx.bitcast(value, bx.cx.type_i64()), + _ => value, + } + } + _ => value, + } +} + +/// Fix up an output value to work around LLVM bugs. +fn llvm_fixup_output<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + mut value: &'ll Value, + reg: InlineAsmRegClass, + layout: &TyAndLayout<'tcx>, +) -> &'ll Value { + match (reg, layout.abi) { + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + if let Primitive::Int(Integer::I8, _) = s.primitive() { + bx.extract_element(value, bx.const_i32(0)) + } else { + value + } + } + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { + value = bx.extract_element(value, bx.const_i32(0)); + if let Primitive::Pointer = s.primitive() { + value = bx.inttoptr(value, layout.llvm_type(bx.cx)); + } + value + } + ( + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { + let elem_ty = llvm_asm_scalar_type(bx.cx, element); + let vec_ty = bx.cx.type_vector(elem_ty, count * 2); + let indices: Vec<_> = (0..count).map(|x| bx.const_i32(x as i32)).collect(); + bx.shuffle_vector(value, bx.const_undef(vec_ty), bx.const_vector(&indices)) + } + (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + if s.primitive() == Primitive::F64 => + { + bx.bitcast(value, bx.cx.type_f64()) + } + ( + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), + Abi::Vector { .. }, + ) if layout.size.bytes() == 64 => bx.bitcast(value, layout.llvm_type(bx.cx)), + ( + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I32, _) = s.primitive() { + bx.bitcast(value, bx.cx.type_i32()) + } else { + value + } + } + ( + InlineAsmRegClass::Arm( + ArmInlineAsmRegClass::dreg + | ArmInlineAsmRegClass::dreg_low8 + | ArmInlineAsmRegClass::dreg_low16, + ), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I64, _) = s.primitive() { + bx.bitcast(value, bx.cx.type_i64()) + } else { + value + } + } + (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + match s.primitive() { + // MIPS only supports register-length arithmetics. + Primitive::Int(Integer::I8, _) => bx.trunc(value, bx.cx.type_i8()), + Primitive::Int(Integer::I16, _) => bx.trunc(value, bx.cx.type_i16()), + Primitive::F32 => bx.bitcast(value, bx.cx.type_f32()), + Primitive::F64 => bx.bitcast(value, bx.cx.type_f64()), + _ => value, + } + } + _ => value, + } +} + +/// Output type to use for llvm_fixup_output. +fn llvm_fixup_output_type<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + reg: InlineAsmRegClass, + layout: &TyAndLayout<'tcx>, +) -> &'ll Type { + match (reg, layout.abi) { + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg), Abi::Scalar(s)) => { + if let Primitive::Int(Integer::I8, _) = s.primitive() { + cx.type_vector(cx.type_i8(), 8) + } else { + layout.llvm_type(cx) + } + } + (InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), Abi::Scalar(s)) => { + let elem_ty = llvm_asm_scalar_type(cx, s); + let count = 16 / layout.size.bytes(); + cx.type_vector(elem_ty, count) + } + ( + InlineAsmRegClass::AArch64(AArch64InlineAsmRegClass::vreg_low16), + Abi::Vector { element, count }, + ) if layout.size.bytes() == 8 => { + let elem_ty = llvm_asm_scalar_type(cx, element); + cx.type_vector(elem_ty, count * 2) + } + (InlineAsmRegClass::X86(X86InlineAsmRegClass::reg_abcd), Abi::Scalar(s)) + if s.primitive() == Primitive::F64 => + { + cx.type_i64() + } + ( + InlineAsmRegClass::X86(X86InlineAsmRegClass::xmm_reg | X86InlineAsmRegClass::zmm_reg), + Abi::Vector { .. }, + ) if layout.size.bytes() == 64 => cx.type_vector(cx.type_f64(), 8), + ( + InlineAsmRegClass::Arm(ArmInlineAsmRegClass::sreg | ArmInlineAsmRegClass::sreg_low16), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I32, _) = s.primitive() { + cx.type_f32() + } else { + layout.llvm_type(cx) + } + } + ( + InlineAsmRegClass::Arm( + ArmInlineAsmRegClass::dreg + | ArmInlineAsmRegClass::dreg_low8 + | ArmInlineAsmRegClass::dreg_low16, + ), + Abi::Scalar(s), + ) => { + if let Primitive::Int(Integer::I64, _) = s.primitive() { + cx.type_f64() + } else { + layout.llvm_type(cx) + } + } + (InlineAsmRegClass::Mips(MipsInlineAsmRegClass::reg), Abi::Scalar(s)) => { + match s.primitive() { + // MIPS only supports register-length arithmetics. + Primitive::Int(Integer::I8 | Integer::I16, _) => cx.type_i32(), + Primitive::F32 => cx.type_i32(), + Primitive::F64 => cx.type_i64(), + _ => layout.llvm_type(cx), + } + } + _ => layout.llvm_type(cx), + } +} diff --git a/compiler/rustc_codegen_llvm/src/attributes.rs b/compiler/rustc_codegen_llvm/src/attributes.rs new file mode 100644 index 000000000..aabbe8ac2 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/attributes.rs @@ -0,0 +1,449 @@ +//! Set and unset common attributes on LLVM values. + +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::small_str::SmallStr; +use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::ty::{self, TyCtxt}; +use rustc_session::config::OptLevel; +use rustc_span::symbol::sym; +use rustc_target::spec::abi::Abi; +use rustc_target::spec::{FramePointer, SanitizerSet, StackProbeType, StackProtector}; +use smallvec::SmallVec; + +use crate::attributes; +use crate::llvm::AttributePlace::Function; +use crate::llvm::{self, AllocKindFlags, Attribute, AttributeKind, AttributePlace}; +use crate::llvm_util; +pub use rustc_attr::{InlineAttr, InstructionSetAttr, OptimizeAttr}; + +use crate::context::CodegenCx; +use crate::value::Value; + +pub fn apply_to_llfn(llfn: &Value, idx: AttributePlace, attrs: &[&Attribute]) { + if !attrs.is_empty() { + llvm::AddFunctionAttributes(llfn, idx, attrs); + } +} + +pub fn apply_to_callsite(callsite: &Value, idx: AttributePlace, attrs: &[&Attribute]) { + if !attrs.is_empty() { + llvm::AddCallSiteAttributes(callsite, idx, attrs); + } +} + +/// Get LLVM attribute for the provided inline heuristic. +#[inline] +fn inline_attr<'ll>(cx: &CodegenCx<'ll, '_>, inline: InlineAttr) -> Option<&'ll Attribute> { + match inline { + InlineAttr::Hint => Some(AttributeKind::InlineHint.create_attr(cx.llcx)), + InlineAttr::Always => Some(AttributeKind::AlwaysInline.create_attr(cx.llcx)), + InlineAttr::Never => { + if cx.sess().target.arch != "amdgpu" { + Some(AttributeKind::NoInline.create_attr(cx.llcx)) + } else { + None + } + } + InlineAttr::None => None, + } +} + +/// Get LLVM sanitize attributes. +#[inline] +pub fn sanitize_attrs<'ll>( + cx: &CodegenCx<'ll, '_>, + no_sanitize: SanitizerSet, +) -> SmallVec<[&'ll Attribute; 4]> { + let mut attrs = SmallVec::new(); + let enabled = cx.tcx.sess.opts.unstable_opts.sanitizer - no_sanitize; + if enabled.contains(SanitizerSet::ADDRESS) { + attrs.push(llvm::AttributeKind::SanitizeAddress.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::MEMORY) { + attrs.push(llvm::AttributeKind::SanitizeMemory.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::THREAD) { + attrs.push(llvm::AttributeKind::SanitizeThread.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::HWADDRESS) { + attrs.push(llvm::AttributeKind::SanitizeHWAddress.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::SHADOWCALLSTACK) { + attrs.push(llvm::AttributeKind::ShadowCallStack.create_attr(cx.llcx)); + } + if enabled.contains(SanitizerSet::MEMTAG) { + // Check to make sure the mte target feature is actually enabled. + let features = cx.tcx.global_backend_features(()); + let mte_feature = + features.iter().map(|s| &s[..]).rfind(|n| ["+mte", "-mte"].contains(&&n[..])); + if let None | Some("-mte") = mte_feature { + cx.tcx.sess.err("`-Zsanitizer=memtag` requires `-Ctarget-feature=+mte`"); + } + + attrs.push(llvm::AttributeKind::SanitizeMemTag.create_attr(cx.llcx)); + } + attrs +} + +/// Tell LLVM to emit or not emit the information necessary to unwind the stack for the function. +#[inline] +pub fn uwtable_attr(llcx: &llvm::Context) -> &Attribute { + // NOTE: We should determine if we even need async unwind tables, as they + // take have more overhead and if we can use sync unwind tables we + // probably should. + llvm::CreateUWTableAttr(llcx, true) +} + +pub fn frame_pointer_type_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + let mut fp = cx.sess().target.frame_pointer; + // "mcount" function relies on stack pointer. + // See . + if cx.sess().instrument_mcount() || matches!(cx.sess().opts.cg.force_frame_pointers, Some(true)) + { + fp = FramePointer::Always; + } + let attr_value = match fp { + FramePointer::Always => "all", + FramePointer::NonLeaf => "non-leaf", + FramePointer::MayOmit => return None, + }; + Some(llvm::CreateAttrStringValue(cx.llcx, "frame-pointer", attr_value)) +} + +/// Tell LLVM what instrument function to insert. +#[inline] +fn instrument_function_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + if cx.sess().instrument_mcount() { + // Similar to `clang -pg` behavior. Handled by the + // `post-inline-ee-instrument` LLVM pass. + + // The function name varies on platforms. + // See test/CodeGen/mcount.c in clang. + let mcount_name = cx.sess().target.mcount.as_ref(); + + Some(llvm::CreateAttrStringValue( + cx.llcx, + "instrument-function-entry-inlined", + &mcount_name, + )) + } else { + None + } +} + +fn probestack_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + // Currently stack probes seem somewhat incompatible with the address + // sanitizer and thread sanitizer. With asan we're already protected from + // stack overflow anyway so we don't really need stack probes regardless. + if cx + .sess() + .opts + .unstable_opts + .sanitizer + .intersects(SanitizerSet::ADDRESS | SanitizerSet::THREAD) + { + return None; + } + + // probestack doesn't play nice either with `-C profile-generate`. + if cx.sess().opts.cg.profile_generate.enabled() { + return None; + } + + // probestack doesn't play nice either with gcov profiling. + if cx.sess().opts.unstable_opts.profile { + return None; + } + + let attr_value = match cx.sess().target.stack_probes { + StackProbeType::None => return None, + // Request LLVM to generate the probes inline. If the given LLVM version does not support + // this, no probe is generated at all (even if the attribute is specified). + StackProbeType::Inline => "inline-asm", + // Flag our internal `__rust_probestack` function as the stack probe symbol. + // This is defined in the `compiler-builtins` crate for each architecture. + StackProbeType::Call => "__rust_probestack", + // Pick from the two above based on the LLVM version. + StackProbeType::InlineOrCall { min_llvm_version_for_inline } => { + if llvm_util::get_version() < min_llvm_version_for_inline { + "__rust_probestack" + } else { + "inline-asm" + } + } + }; + Some(llvm::CreateAttrStringValue(cx.llcx, "probe-stack", attr_value)) +} + +fn stackprotector_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + let sspattr = match cx.sess().stack_protector() { + StackProtector::None => return None, + StackProtector::All => AttributeKind::StackProtectReq, + StackProtector::Strong => AttributeKind::StackProtectStrong, + StackProtector::Basic => AttributeKind::StackProtect, + }; + + Some(sspattr.create_attr(cx.llcx)) +} + +pub fn target_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Attribute { + let target_cpu = llvm_util::target_cpu(cx.tcx.sess); + llvm::CreateAttrStringValue(cx.llcx, "target-cpu", target_cpu) +} + +pub fn tune_cpu_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + llvm_util::tune_cpu(cx.tcx.sess) + .map(|tune_cpu| llvm::CreateAttrStringValue(cx.llcx, "tune-cpu", tune_cpu)) +} + +/// Get the `NonLazyBind` LLVM attribute, +/// if the codegen options allow skipping the PLT. +pub fn non_lazy_bind_attr<'ll>(cx: &CodegenCx<'ll, '_>) -> Option<&'ll Attribute> { + // Don't generate calls through PLT if it's not necessary + if !cx.sess().needs_plt() { + Some(AttributeKind::NonLazyBind.create_attr(cx.llcx)) + } else { + None + } +} + +/// Get the default optimizations attrs for a function. +#[inline] +pub(crate) fn default_optimisation_attrs<'ll>( + cx: &CodegenCx<'ll, '_>, +) -> SmallVec<[&'ll Attribute; 2]> { + let mut attrs = SmallVec::new(); + match cx.sess().opts.optimize { + OptLevel::Size => { + attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); + } + OptLevel::SizeMin => { + attrs.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); + attrs.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); + } + _ => {} + } + attrs +} + +fn create_alloc_family_attr(llcx: &llvm::Context) -> &llvm::Attribute { + llvm::CreateAttrStringValue(llcx, "alloc-family", "__rust_alloc") +} + +/// Composite function which sets LLVM attributes for function depending on its AST (`#[attribute]`) +/// attributes. +pub fn from_fn_attrs<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + llfn: &'ll Value, + instance: ty::Instance<'tcx>, +) { + let codegen_fn_attrs = cx.tcx.codegen_fn_attrs(instance.def_id()); + + let mut to_add = SmallVec::<[_; 16]>::new(); + + match codegen_fn_attrs.optimize { + OptimizeAttr::None => { + to_add.extend(default_optimisation_attrs(cx)); + } + OptimizeAttr::Size => { + to_add.push(llvm::AttributeKind::MinSize.create_attr(cx.llcx)); + to_add.push(llvm::AttributeKind::OptimizeForSize.create_attr(cx.llcx)); + } + OptimizeAttr::Speed => {} + } + + let inline = if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { + InlineAttr::Never + } else if codegen_fn_attrs.inline == InlineAttr::None && instance.def.requires_inline(cx.tcx) { + InlineAttr::Hint + } else { + codegen_fn_attrs.inline + }; + to_add.extend(inline_attr(cx, inline)); + + // The `uwtable` attribute according to LLVM is: + // + // This attribute indicates that the ABI being targeted requires that an + // unwind table entry be produced for this function even if we can show + // that no exceptions passes by it. This is normally the case for the + // ELF x86-64 abi, but it can be disabled for some compilation units. + // + // Typically when we're compiling with `-C panic=abort` (which implies this + // `no_landing_pads` check) we don't need `uwtable` because we can't + // generate any exceptions! On Windows, however, exceptions include other + // events such as illegal instructions, segfaults, etc. This means that on + // Windows we end up still needing the `uwtable` attribute even if the `-C + // panic=abort` flag is passed. + // + // You can also find more info on why Windows always requires uwtables here: + // https://bugzilla.mozilla.org/show_bug.cgi?id=1302078 + if cx.sess().must_emit_unwind_tables() { + to_add.push(uwtable_attr(cx.llcx)); + } + + if cx.sess().opts.unstable_opts.profile_sample_use.is_some() { + to_add.push(llvm::CreateAttrString(cx.llcx, "use-sample-profile")); + } + + // FIXME: none of these three functions interact with source level attributes. + to_add.extend(frame_pointer_type_attr(cx)); + to_add.extend(instrument_function_attr(cx)); + to_add.extend(probestack_attr(cx)); + to_add.extend(stackprotector_attr(cx)); + + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) { + to_add.push(AttributeKind::Cold.create_attr(cx.llcx)); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) { + to_add.push(AttributeKind::ReturnsTwice.create_attr(cx.llcx)); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_PURE) { + to_add.push(AttributeKind::ReadOnly.create_attr(cx.llcx)); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_CONST) { + to_add.push(AttributeKind::ReadNone.create_attr(cx.llcx)); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) { + to_add.push(AttributeKind::Naked.create_attr(cx.llcx)); + // HACK(jubilee): "indirect branch tracking" works by attaching prologues to functions. + // And it is a module-level attribute, so the alternative is pulling naked functions into new LLVM modules. + // Otherwise LLVM's "naked" functions come with endbr prefixes per https://github.com/rust-lang/rust/issues/98768 + to_add.push(AttributeKind::NoCfCheck.create_attr(cx.llcx)); + // Need this for AArch64. + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "branch-target-enforcement", "false")); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) + || codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR_ZEROED) + { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(1), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 0)); + let mut flags = AllocKindFlags::Alloc | AllocKindFlags::Aligned; + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) { + flags |= AllocKindFlags::Uninitialized; + } else { + flags |= AllocKindFlags::Zeroed; + } + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, flags)); + } + // apply to return place instead of function (unlike all other attributes applied in this function) + let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::REALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr( + cx.llcx, + AllocKindFlags::Realloc | AllocKindFlags::Aligned, + )); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + // apply to argument place instead of function + let alloc_align = AttributeKind::AllocAlign.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(2), &[alloc_align]); + to_add.push(llvm::CreateAllocSizeAttr(cx.llcx, 3)); + } + let no_alias = AttributeKind::NoAlias.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::ReturnValue, &[no_alias]); + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::DEALLOCATOR) { + if llvm_util::get_version() >= (15, 0, 0) { + to_add.push(create_alloc_family_attr(cx.llcx)); + to_add.push(llvm::CreateAllocKindAttr(cx.llcx, AllocKindFlags::Free)); + // applies to argument place instead of function place + let allocated_pointer = AttributeKind::AllocatedPointer.create_attr(cx.llcx); + attributes::apply_to_llfn(llfn, AttributePlace::Argument(0), &[allocated_pointer]); + } + } + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::CMSE_NONSECURE_ENTRY) { + to_add.push(llvm::CreateAttrString(cx.llcx, "cmse_nonsecure_entry")); + } + if let Some(align) = codegen_fn_attrs.alignment { + llvm::set_alignment(llfn, align as usize); + } + to_add.extend(sanitize_attrs(cx, codegen_fn_attrs.no_sanitize)); + + // Always annotate functions with the target-cpu they are compiled for. + // Without this, ThinLTO won't inline Rust functions into Clang generated + // functions (because Clang annotates functions this way too). + to_add.push(target_cpu_attr(cx)); + // tune-cpu is only conveyed through the attribute for our purpose. + // The target doesn't care; the subtarget reads our attribute. + to_add.extend(tune_cpu_attr(cx)); + + let function_features = + codegen_fn_attrs.target_features.iter().map(|f| f.as_str()).collect::>(); + + if let Some(f) = llvm_util::check_tied_features( + cx.tcx.sess, + &function_features.iter().map(|f| (*f, true)).collect(), + ) { + let span = cx + .tcx + .get_attr(instance.def_id(), sym::target_feature) + .map_or_else(|| cx.tcx.def_span(instance.def_id()), |a| a.span); + let msg = format!( + "the target features {} must all be either enabled or disabled together", + f.join(", ") + ); + let mut err = cx.tcx.sess.struct_span_err(span, &msg); + err.help("add the missing features in a `target_feature` attribute"); + err.emit(); + return; + } + + let mut function_features = function_features + .iter() + .flat_map(|feat| { + llvm_util::to_llvm_features(cx.tcx.sess, feat).into_iter().map(|f| format!("+{}", f)) + }) + .chain(codegen_fn_attrs.instruction_set.iter().map(|x| match x { + InstructionSetAttr::ArmA32 => "-thumb-mode".to_string(), + InstructionSetAttr::ArmT32 => "+thumb-mode".to_string(), + })) + .collect::>(); + + if cx.tcx.sess.target.is_like_wasm { + // If this function is an import from the environment but the wasm + // import has a specific module/name, apply them here. + if let Some(module) = wasm_import_module(cx.tcx, instance.def_id()) { + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-module", &module)); + + let name = + codegen_fn_attrs.link_name.unwrap_or_else(|| cx.tcx.item_name(instance.def_id())); + let name = name.as_str(); + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "wasm-import-name", name)); + } + + // The `"wasm"` abi on wasm targets automatically enables the + // `+multivalue` feature because the purpose of the wasm abi is to match + // the WebAssembly specification, which has this feature. This won't be + // needed when LLVM enables this `multivalue` feature by default. + if !cx.tcx.is_closure(instance.def_id()) { + let abi = cx.tcx.fn_sig(instance.def_id()).abi(); + if abi == Abi::Wasm { + function_features.push("+multivalue".to_string()); + } + } + } + + let global_features = cx.tcx.global_backend_features(()).iter().map(|s| s.as_str()); + let function_features = function_features.iter().map(|s| s.as_str()); + let target_features = + global_features.chain(function_features).intersperse(",").collect::>(); + if !target_features.is_empty() { + to_add.push(llvm::CreateAttrStringValue(cx.llcx, "target-features", &target_features)); + } + + attributes::apply_to_llfn(llfn, Function, &to_add); +} + +fn wasm_import_module(tcx: TyCtxt<'_>, id: DefId) -> Option<&String> { + tcx.wasm_import_module_map(id.krate).get(&id) +} diff --git a/compiler/rustc_codegen_llvm/src/back/archive.rs b/compiler/rustc_codegen_llvm/src/back/archive.rs new file mode 100644 index 000000000..27039cda2 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/archive.rs @@ -0,0 +1,383 @@ +//! A helper class for dealing with static archives + +use std::env; +use std::ffi::{CStr, CString, OsString}; +use std::io; +use std::mem; +use std::path::{Path, PathBuf}; +use std::ptr; +use std::str; + +use crate::llvm::archive_ro::{ArchiveRO, Child}; +use crate::llvm::{self, ArchiveKind, LLVMMachineType, LLVMRustCOFFShortExport}; +use rustc_codegen_ssa::back::archive::{ArchiveBuilder, ArchiveBuilderBuilder}; +use rustc_session::cstore::{DllCallingConvention, DllImport}; +use rustc_session::Session; + +/// Helper for adding many files to an archive. +#[must_use = "must call build() to finish building the archive"] +pub struct LlvmArchiveBuilder<'a> { + sess: &'a Session, + additions: Vec, +} + +enum Addition { + File { path: PathBuf, name_in_archive: String }, + Archive { path: PathBuf, archive: ArchiveRO, skip: Box bool> }, +} + +impl Addition { + fn path(&self) -> &Path { + match self { + Addition::File { path, .. } | Addition::Archive { path, .. } => path, + } + } +} + +fn is_relevant_child(c: &Child<'_>) -> bool { + match c.name() { + Some(name) => !name.contains("SYMDEF"), + None => false, + } +} + +/// Map machine type strings to values of LLVM's MachineTypes enum. +fn llvm_machine_type(cpu: &str) -> LLVMMachineType { + match cpu { + "x86_64" => LLVMMachineType::AMD64, + "x86" => LLVMMachineType::I386, + "aarch64" => LLVMMachineType::ARM64, + "arm" => LLVMMachineType::ARM, + _ => panic!("unsupported cpu type {}", cpu), + } +} + +impl<'a> ArchiveBuilder<'a> for LlvmArchiveBuilder<'a> { + fn add_archive( + &mut self, + archive: &Path, + skip: Box bool + 'static>, + ) -> io::Result<()> { + let archive_ro = match ArchiveRO::open(archive) { + Ok(ar) => ar, + Err(e) => return Err(io::Error::new(io::ErrorKind::Other, e)), + }; + if self.additions.iter().any(|ar| ar.path() == archive) { + return Ok(()); + } + self.additions.push(Addition::Archive { + path: archive.to_path_buf(), + archive: archive_ro, + skip: Box::new(skip), + }); + Ok(()) + } + + /// Adds an arbitrary file to this archive + fn add_file(&mut self, file: &Path) { + let name = file.file_name().unwrap().to_str().unwrap(); + self.additions + .push(Addition::File { path: file.to_path_buf(), name_in_archive: name.to_owned() }); + } + + /// Combine the provided files, rlibs, and native libraries into a single + /// `Archive`. + fn build(mut self: Box, output: &Path) -> bool { + match self.build_with_llvm(output) { + Ok(any_members) => any_members, + Err(e) => self.sess.fatal(&format!("failed to build archive: {}", e)), + } + } +} + +pub struct LlvmArchiveBuilderBuilder; + +impl ArchiveBuilderBuilder for LlvmArchiveBuilderBuilder { + fn new_archive_builder<'a>(&self, sess: &'a Session) -> Box + 'a> { + Box::new(LlvmArchiveBuilder { sess, additions: Vec::new() }) + } + + fn create_dll_import_lib( + &self, + sess: &Session, + lib_name: &str, + dll_imports: &[DllImport], + tmpdir: &Path, + ) -> PathBuf { + let output_path = { + let mut output_path: PathBuf = tmpdir.to_path_buf(); + output_path.push(format!("{}_imports", lib_name)); + output_path.with_extension("lib") + }; + + let target = &sess.target; + let mingw_gnu_toolchain = target.vendor == "pc" + && target.os == "windows" + && target.env == "gnu" + && target.abi.is_empty(); + + let import_name_and_ordinal_vector: Vec<(String, Option)> = dll_imports + .iter() + .map(|import: &DllImport| { + if sess.target.arch == "x86" { + ( + LlvmArchiveBuilder::i686_decorated_name(import, mingw_gnu_toolchain), + import.ordinal, + ) + } else { + (import.name.to_string(), import.ordinal) + } + }) + .collect(); + + if mingw_gnu_toolchain { + // The binutils linker used on -windows-gnu targets cannot read the import + // libraries generated by LLVM: in our attempts, the linker produced an .EXE + // that loaded but crashed with an AV upon calling one of the imported + // functions. Therefore, use binutils to create the import library instead, + // by writing a .DEF file to the temp dir and calling binutils's dlltool. + let def_file_path = tmpdir.join(format!("{}_imports", lib_name)).with_extension("def"); + + let def_file_content = format!( + "EXPORTS\n{}", + import_name_and_ordinal_vector + .into_iter() + .map(|(name, ordinal)| { + match ordinal { + Some(n) => format!("{} @{} NONAME", name, n), + None => name, + } + }) + .collect::>() + .join("\n") + ); + + match std::fs::write(&def_file_path, def_file_content) { + Ok(_) => {} + Err(e) => { + sess.fatal(&format!("Error writing .DEF file: {}", e)); + } + }; + + let dlltool = find_binutils_dlltool(sess); + let result = std::process::Command::new(dlltool) + .args([ + "-d", + def_file_path.to_str().unwrap(), + "-D", + lib_name, + "-l", + output_path.to_str().unwrap(), + ]) + .output(); + + match result { + Err(e) => { + sess.fatal(&format!("Error calling dlltool: {}", e)); + } + Ok(output) if !output.status.success() => sess.fatal(&format!( + "Dlltool could not create import library: {}\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr) + )), + _ => {} + } + } else { + // we've checked for \0 characters in the library name already + let dll_name_z = CString::new(lib_name).unwrap(); + + let output_path_z = rustc_fs_util::path_to_c_string(&output_path); + + tracing::trace!("invoking LLVMRustWriteImportLibrary"); + tracing::trace!(" dll_name {:#?}", dll_name_z); + tracing::trace!(" output_path {}", output_path.display()); + tracing::trace!( + " import names: {}", + dll_imports + .iter() + .map(|import| import.name.to_string()) + .collect::>() + .join(", "), + ); + + // All import names are Rust identifiers and therefore cannot contain \0 characters. + // FIXME: when support for #[link_name] is implemented, ensure that the import names + // still don't contain any \0 characters. Also need to check that the names don't + // contain substrings like " @" or "NONAME" that are keywords or otherwise reserved + // in definition files. + let cstring_import_name_and_ordinal_vector: Vec<(CString, Option)> = + import_name_and_ordinal_vector + .into_iter() + .map(|(name, ordinal)| (CString::new(name).unwrap(), ordinal)) + .collect(); + + let ffi_exports: Vec = cstring_import_name_and_ordinal_vector + .iter() + .map(|(name_z, ordinal)| LLVMRustCOFFShortExport::new(name_z.as_ptr(), *ordinal)) + .collect(); + let result = unsafe { + crate::llvm::LLVMRustWriteImportLibrary( + dll_name_z.as_ptr(), + output_path_z.as_ptr(), + ffi_exports.as_ptr(), + ffi_exports.len(), + llvm_machine_type(&sess.target.arch) as u16, + !sess.target.is_like_msvc, + ) + }; + + if result == crate::llvm::LLVMRustResult::Failure { + sess.fatal(&format!( + "Error creating import library for {}: {}", + lib_name, + llvm::last_error().unwrap_or("unknown LLVM error".to_string()) + )); + } + }; + + output_path + } +} + +impl<'a> LlvmArchiveBuilder<'a> { + fn build_with_llvm(&mut self, output: &Path) -> io::Result { + let kind = &*self.sess.target.archive_format; + let kind = kind.parse::().map_err(|_| kind).unwrap_or_else(|kind| { + self.sess.fatal(&format!("Don't know how to build archive of type: {}", kind)) + }); + + let mut additions = mem::take(&mut self.additions); + let mut strings = Vec::new(); + let mut members = Vec::new(); + + let dst = CString::new(output.to_str().unwrap())?; + + unsafe { + for addition in &mut additions { + match addition { + Addition::File { path, name_in_archive } => { + let path = CString::new(path.to_str().unwrap())?; + let name = CString::new(name_in_archive.clone())?; + members.push(llvm::LLVMRustArchiveMemberNew( + path.as_ptr(), + name.as_ptr(), + None, + )); + strings.push(path); + strings.push(name); + } + Addition::Archive { archive, skip, .. } => { + for child in archive.iter() { + let child = child.map_err(string_to_io_error)?; + if !is_relevant_child(&child) { + continue; + } + let child_name = child.name().unwrap(); + if skip(child_name) { + continue; + } + + // It appears that LLVM's archive writer is a little + // buggy if the name we pass down isn't just the + // filename component, so chop that off here and + // pass it in. + // + // See LLVM bug 25877 for more info. + let child_name = + Path::new(child_name).file_name().unwrap().to_str().unwrap(); + let name = CString::new(child_name)?; + let m = llvm::LLVMRustArchiveMemberNew( + ptr::null(), + name.as_ptr(), + Some(child.raw), + ); + members.push(m); + strings.push(name); + } + } + } + } + + let r = llvm::LLVMRustWriteArchive( + dst.as_ptr(), + members.len() as libc::size_t, + members.as_ptr() as *const &_, + true, + kind, + ); + let ret = if r.into_result().is_err() { + let err = llvm::LLVMRustGetLastError(); + let msg = if err.is_null() { + "failed to write archive".into() + } else { + String::from_utf8_lossy(CStr::from_ptr(err).to_bytes()) + }; + Err(io::Error::new(io::ErrorKind::Other, msg)) + } else { + Ok(!members.is_empty()) + }; + for member in members { + llvm::LLVMRustArchiveMemberFree(member); + } + ret + } + } + + fn i686_decorated_name(import: &DllImport, mingw: bool) -> String { + let name = import.name; + let prefix = if mingw { "" } else { "_" }; + + match import.calling_convention { + DllCallingConvention::C => format!("{}{}", prefix, name), + DllCallingConvention::Stdcall(arg_list_size) => { + format!("{}{}@{}", prefix, name, arg_list_size) + } + DllCallingConvention::Fastcall(arg_list_size) => format!("@{}@{}", name, arg_list_size), + DllCallingConvention::Vectorcall(arg_list_size) => { + format!("{}@@{}", name, arg_list_size) + } + } + } +} + +fn string_to_io_error(s: String) -> io::Error { + io::Error::new(io::ErrorKind::Other, format!("bad archive: {}", s)) +} + +fn find_binutils_dlltool(sess: &Session) -> OsString { + assert!(sess.target.options.is_like_windows && !sess.target.options.is_like_msvc); + if let Some(dlltool_path) = &sess.opts.unstable_opts.dlltool { + return dlltool_path.clone().into_os_string(); + } + + let mut tool_name: OsString = if sess.host.arch != sess.target.arch { + // We are cross-compiling, so we need the tool with the prefix matching our target + if sess.target.arch == "x86" { + "i686-w64-mingw32-dlltool" + } else { + "x86_64-w64-mingw32-dlltool" + } + } else { + // We are not cross-compiling, so we just want `dlltool` + "dlltool" + } + .into(); + + if sess.host.options.is_like_windows { + // If we're compiling on Windows, add the .exe suffix + tool_name.push(".exe"); + } + + // NOTE: it's not clear how useful it is to explicitly search PATH. + for dir in env::split_paths(&env::var_os("PATH").unwrap_or_default()) { + let full_path = dir.join(&tool_name); + if full_path.is_file() { + return full_path.into_os_string(); + } + } + + // The user didn't specify the location of the dlltool binary, and we weren't able + // to find the appropriate one on the PATH. Just return the name of the tool + // and let the invocation fail with a hopefully useful error message. + tool_name +} diff --git a/compiler/rustc_codegen_llvm/src/back/lto.rs b/compiler/rustc_codegen_llvm/src/back/lto.rs new file mode 100644 index 000000000..3731c6bcf --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/lto.rs @@ -0,0 +1,936 @@ +use crate::back::write::{ + self, save_temp_bitcode, to_llvm_opt_settings, with_llvm_pmb, DiagnosticHandlers, +}; +use crate::llvm::archive_ro::ArchiveRO; +use crate::llvm::{self, build_string, False, True}; +use crate::{llvm_util, LlvmCodegenBackend, ModuleLlvm}; +use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared}; +use rustc_codegen_ssa::back::symbol_export; +use rustc_codegen_ssa::back::write::{CodegenContext, FatLTOInput, TargetMachineFactoryConfig}; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind}; +use rustc_data_structures::fx::FxHashMap; +use rustc_errors::{FatalError, Handler}; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::bug; +use rustc_middle::dep_graph::WorkProduct; +use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel}; +use rustc_session::cgu_reuse_tracker::CguReuse; +use rustc_session::config::{self, CrateType, Lto}; +use tracing::{debug, info}; + +use std::ffi::{CStr, CString}; +use std::fs::File; +use std::io; +use std::iter; +use std::path::Path; +use std::ptr; +use std::slice; +use std::sync::Arc; + +/// We keep track of the computed LTO cache keys from the previous +/// session to determine which CGUs we can reuse. +pub const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin"; + +pub fn crate_type_allows_lto(crate_type: CrateType) -> bool { + match crate_type { + CrateType::Executable | CrateType::Staticlib | CrateType::Cdylib => true, + CrateType::Dylib | CrateType::Rlib | CrateType::ProcMacro => false, + } +} + +fn prepare_lto( + cgcx: &CodegenContext, + diag_handler: &Handler, +) -> Result<(Vec, Vec<(SerializedModule, CString)>), FatalError> { + let export_threshold = match cgcx.lto { + // We're just doing LTO for our one crate + Lto::ThinLocal => SymbolExportLevel::Rust, + + // We're doing LTO for the entire crate graph + Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types), + + Lto::No => panic!("didn't request LTO but we're doing LTO"), + }; + + let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| { + if info.level.is_below_threshold(export_threshold) || info.used { + Some(CString::new(name.as_str()).unwrap()) + } else { + None + } + }; + let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); + let mut symbols_below_threshold = { + let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); + exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::>() + }; + info!("{} symbols to preserve in this crate", symbols_below_threshold.len()); + + // If we're performing LTO for the entire crate graph, then for each of our + // upstream dependencies, find the corresponding rlib and load the bitcode + // from the archive. + // + // We save off all the bytecode and LLVM module ids for later processing + // with either fat or thin LTO + let mut upstream_modules = Vec::new(); + if cgcx.lto != Lto::ThinLocal { + if cgcx.opts.cg.prefer_dynamic { + diag_handler + .struct_err("cannot prefer dynamic linking when performing LTO") + .note( + "only 'staticlib', 'bin', and 'cdylib' outputs are \ + supported with LTO", + ) + .emit(); + return Err(FatalError); + } + + // Make sure we actually can run LTO + for crate_type in cgcx.crate_types.iter() { + if !crate_type_allows_lto(*crate_type) { + let e = diag_handler.fatal( + "lto can only be run for executables, cdylibs and \ + static library outputs", + ); + return Err(e); + } + } + + for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() { + let exported_symbols = + cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO"); + { + let _timer = + cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold"); + symbols_below_threshold + .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter)); + } + + let archive = ArchiveRO::open(path).expect("wanted an rlib"); + let obj_files = archive + .iter() + .filter_map(|child| child.ok().and_then(|c| c.name().map(|name| (name, c)))) + .filter(|&(name, _)| looks_like_rust_object_file(name)); + for (name, child) in obj_files { + info!("adding bitcode from {}", name); + match get_bitcode_slice_from_object_data(child.data()) { + Ok(data) => { + let module = SerializedModule::FromRlib(data.to_vec()); + upstream_modules.push((module, CString::new(name).unwrap())); + } + Err(msg) => return Err(diag_handler.fatal(&msg)), + } + } + } + } + + Ok((symbols_below_threshold, upstream_modules)) +} + +fn get_bitcode_slice_from_object_data(obj: &[u8]) -> Result<&[u8], String> { + let mut len = 0; + let data = + unsafe { llvm::LLVMRustGetBitcodeSliceFromObjectData(obj.as_ptr(), obj.len(), &mut len) }; + if !data.is_null() { + assert!(len != 0); + let bc = unsafe { slice::from_raw_parts(data, len) }; + + // `bc` must be a sub-slice of `obj`. + assert!(obj.as_ptr() <= bc.as_ptr()); + assert!(bc[bc.len()..bc.len()].as_ptr() <= obj[obj.len()..obj.len()].as_ptr()); + + Ok(bc) + } else { + assert!(len == 0); + let msg = llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()); + Err(format!("failed to get bitcode from object file for LTO ({})", msg)) + } +} + +/// Performs fat LTO by merging all modules into a single one and returning it +/// for further optimization. +pub(crate) fn run_fat( + cgcx: &CodegenContext, + modules: Vec>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, +) -> Result, FatalError> { + let diag_handler = cgcx.create_diag_handler(); + let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; + let symbols_below_threshold = + symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); + fat_lto( + cgcx, + &diag_handler, + modules, + cached_modules, + upstream_modules, + &symbols_below_threshold, + ) +} + +/// Performs thin LTO by performing necessary global analysis and returning two +/// lists, one of the modules that need optimization and another for modules that +/// can simply be copied over from the incr. comp. cache. +pub(crate) fn run_thin( + cgcx: &CodegenContext, + modules: Vec<(String, ThinBuffer)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, +) -> Result<(Vec>, Vec), FatalError> { + let diag_handler = cgcx.create_diag_handler(); + let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, &diag_handler)?; + let symbols_below_threshold = + symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::>(); + if cgcx.opts.cg.linker_plugin_lto.enabled() { + unreachable!( + "We should never reach this case if the LTO step \ + is deferred to the linker" + ); + } + thin_lto( + cgcx, + &diag_handler, + modules, + upstream_modules, + cached_modules, + &symbols_below_threshold, + ) +} + +pub(crate) fn prepare_thin(module: ModuleCodegen) -> (String, ThinBuffer) { + let name = module.name.clone(); + let buffer = ThinBuffer::new(module.module_llvm.llmod(), true); + (name, buffer) +} + +fn fat_lto( + cgcx: &CodegenContext, + diag_handler: &Handler, + modules: Vec>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + mut serialized_modules: Vec<(SerializedModule, CString)>, + symbols_below_threshold: &[*const libc::c_char], +) -> Result, FatalError> { + let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module"); + info!("going for a fat lto"); + + // Sort out all our lists of incoming modules into two lists. + // + // * `serialized_modules` (also and argument to this function) contains all + // modules that are serialized in-memory. + // * `in_memory` contains modules which are already parsed and in-memory, + // such as from multi-CGU builds. + // + // All of `cached_modules` (cached from previous incremental builds) can + // immediately go onto the `serialized_modules` modules list and then we can + // split the `modules` array into these two lists. + let mut in_memory = Vec::new(); + serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| { + info!("pushing cached module {:?}", wp.cgu_name); + (buffer, CString::new(wp.cgu_name).unwrap()) + })); + for module in modules { + match module { + FatLTOInput::InMemory(m) => in_memory.push(m), + FatLTOInput::Serialized { name, buffer } => { + info!("pushing serialized module {:?}", name); + let buffer = SerializedModule::Local(buffer); + serialized_modules.push((buffer, CString::new(name).unwrap())); + } + } + } + + // Find the "costliest" module and merge everything into that codegen unit. + // All the other modules will be serialized and reparsed into the new + // context, so this hopefully avoids serializing and parsing the largest + // codegen unit. + // + // Additionally use a regular module as the base here to ensure that various + // file copy operations in the backend work correctly. The only other kind + // of module here should be an allocator one, and if your crate is smaller + // than the allocator module then the size doesn't really matter anyway. + let costliest_module = in_memory + .iter() + .enumerate() + .filter(|&(_, module)| module.kind == ModuleKind::Regular) + .map(|(i, module)| { + let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) }; + (cost, i) + }) + .max(); + + // If we found a costliest module, we're good to go. Otherwise all our + // inputs were serialized which could happen in the case, for example, that + // all our inputs were incrementally reread from the cache and we're just + // re-executing the LTO passes. If that's the case deserialize the first + // module and create a linker with it. + let module: ModuleCodegen = match costliest_module { + Some((_cost, i)) => in_memory.remove(i), + None => { + assert!(!serialized_modules.is_empty(), "must have at least one serialized module"); + let (buffer, name) = serialized_modules.remove(0); + info!("no in-memory regular modules to choose from, parsing {:?}", name); + ModuleCodegen { + module_llvm: ModuleLlvm::parse(cgcx, &name, buffer.data(), diag_handler)?, + name: name.into_string().unwrap(), + kind: ModuleKind::Regular, + } + } + }; + let mut serialized_bitcode = Vec::new(); + { + let (llcx, llmod) = { + let llvm = &module.module_llvm; + (&llvm.llcx, llvm.llmod()) + }; + info!("using {:?} as a base module", module.name); + + // The linking steps below may produce errors and diagnostics within LLVM + // which we'd like to handle and print, so set up our diagnostic handlers + // (which get unregistered when they go out of scope below). + let _handler = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + + // For all other modules we codegened we'll need to link them into our own + // bitcode. All modules were codegened in their own LLVM context, however, + // and we want to move everything to the same LLVM context. Currently the + // way we know of to do that is to serialize them to a string and them parse + // them later. Not great but hey, that's why it's "fat" LTO, right? + for module in in_memory { + let buffer = ModuleBuffer::new(module.module_llvm.llmod()); + let llmod_id = CString::new(&module.name[..]).unwrap(); + serialized_modules.push((SerializedModule::Local(buffer), llmod_id)); + } + // Sort the modules to ensure we produce deterministic results. + serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1)); + + // For all serialized bitcode files we parse them and link them in as we did + // above, this is all mostly handled in C++. Like above, though, we don't + // know much about the memory management here so we err on the side of being + // save and persist everything with the original module. + let mut linker = Linker::new(llmod); + for (bc_decoded, name) in serialized_modules { + let _timer = cgcx + .prof + .generic_activity_with_arg_recorder("LLVM_fat_lto_link_module", |recorder| { + recorder.record_arg(format!("{:?}", name)) + }); + info!("linking {:?}", name); + let data = bc_decoded.data(); + linker.add(data).map_err(|()| { + let msg = format!("failed to load bitcode of module {:?}", name); + write::llvm_err(diag_handler, &msg) + })?; + serialized_bitcode.push(bc_decoded); + } + drop(linker); + save_temp_bitcode(cgcx, &module, "lto.input"); + + // Internalize everything below threshold to help strip out more modules and such. + unsafe { + let ptr = symbols_below_threshold.as_ptr(); + llvm::LLVMRustRunRestrictionPass( + llmod, + ptr as *const *const libc::c_char, + symbols_below_threshold.len() as libc::size_t, + ); + save_temp_bitcode(cgcx, &module, "lto.after-restriction"); + } + } + + Ok(LtoModuleCodegen::Fat { module, _serialized_bitcode: serialized_bitcode }) +} + +pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>); + +impl<'a> Linker<'a> { + pub(crate) fn new(llmod: &'a llvm::Module) -> Self { + unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) } + } + + pub(crate) fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> { + unsafe { + if llvm::LLVMRustLinkerAdd( + self.0, + bytecode.as_ptr() as *const libc::c_char, + bytecode.len(), + ) { + Ok(()) + } else { + Err(()) + } + } + } +} + +impl Drop for Linker<'_> { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustLinkerFree(&mut *(self.0 as *mut _)); + } + } +} + +/// Prepare "thin" LTO to get run on these modules. +/// +/// The general structure of ThinLTO is quite different from the structure of +/// "fat" LTO above. With "fat" LTO all LLVM modules in question are merged into +/// one giant LLVM module, and then we run more optimization passes over this +/// big module after internalizing most symbols. Thin LTO, on the other hand, +/// avoid this large bottleneck through more targeted optimization. +/// +/// At a high level Thin LTO looks like: +/// +/// 1. Prepare a "summary" of each LLVM module in question which describes +/// the values inside, cost of the values, etc. +/// 2. Merge the summaries of all modules in question into one "index" +/// 3. Perform some global analysis on this index +/// 4. For each module, use the index and analysis calculated previously to +/// perform local transformations on the module, for example inlining +/// small functions from other modules. +/// 5. Run thin-specific optimization passes over each module, and then code +/// generate everything at the end. +/// +/// The summary for each module is intended to be quite cheap, and the global +/// index is relatively quite cheap to create as well. As a result, the goal of +/// ThinLTO is to reduce the bottleneck on LTO and enable LTO to be used in more +/// situations. For example one cheap optimization is that we can parallelize +/// all codegen modules, easily making use of all the cores on a machine. +/// +/// With all that in mind, the function here is designed at specifically just +/// calculating the *index* for ThinLTO. This index will then be shared amongst +/// all of the `LtoModuleCodegen` units returned below and destroyed once +/// they all go out of scope. +fn thin_lto( + cgcx: &CodegenContext, + diag_handler: &Handler, + modules: Vec<(String, ThinBuffer)>, + serialized_modules: Vec<(SerializedModule, CString)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + symbols_below_threshold: &[*const libc::c_char], +) -> Result<(Vec>, Vec), FatalError> { + let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis"); + unsafe { + info!("going for that thin, thin LTO"); + + let green_modules: FxHashMap<_, _> = + cached_modules.iter().map(|&(_, ref wp)| (wp.cgu_name.clone(), wp.clone())).collect(); + + let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len(); + let mut thin_buffers = Vec::with_capacity(modules.len()); + let mut module_names = Vec::with_capacity(full_scope_len); + let mut thin_modules = Vec::with_capacity(full_scope_len); + + for (i, (name, buffer)) in modules.into_iter().enumerate() { + info!("local module: {} - {}", i, name); + let cname = CString::new(name.clone()).unwrap(); + thin_modules.push(llvm::ThinLTOModule { + identifier: cname.as_ptr(), + data: buffer.data().as_ptr(), + len: buffer.data().len(), + }); + thin_buffers.push(buffer); + module_names.push(cname); + } + + // FIXME: All upstream crates are deserialized internally in the + // function below to extract their summary and modules. Note that + // unlike the loop above we *must* decode and/or read something + // here as these are all just serialized files on disk. An + // improvement, however, to make here would be to store the + // module summary separately from the actual module itself. Right + // now this is store in one large bitcode file, and the entire + // file is deflate-compressed. We could try to bypass some of the + // decompression by storing the index uncompressed and only + // lazily decompressing the bytecode if necessary. + // + // Note that truly taking advantage of this optimization will + // likely be further down the road. We'd have to implement + // incremental ThinLTO first where we could actually avoid + // looking at upstream modules entirely sometimes (the contents, + // we must always unconditionally look at the index). + let mut serialized = Vec::with_capacity(serialized_modules.len() + cached_modules.len()); + + let cached_modules = + cached_modules.into_iter().map(|(sm, wp)| (sm, CString::new(wp.cgu_name).unwrap())); + + for (module, name) in serialized_modules.into_iter().chain(cached_modules) { + info!("upstream or cached module {:?}", name); + thin_modules.push(llvm::ThinLTOModule { + identifier: name.as_ptr(), + data: module.data().as_ptr(), + len: module.data().len(), + }); + serialized.push(module); + module_names.push(name); + } + + // Sanity check + assert_eq!(thin_modules.len(), module_names.len()); + + // Delegate to the C++ bindings to create some data here. Once this is a + // tried-and-true interface we may wish to try to upstream some of this + // to LLVM itself, right now we reimplement a lot of what they do + // upstream... + let data = llvm::LLVMRustCreateThinLTOData( + thin_modules.as_ptr(), + thin_modules.len() as u32, + symbols_below_threshold.as_ptr(), + symbols_below_threshold.len() as u32, + ) + .ok_or_else(|| write::llvm_err(diag_handler, "failed to prepare thin LTO context"))?; + + let data = ThinData(data); + + info!("thin LTO data created"); + + let (key_map_path, prev_key_map, curr_key_map) = if let Some(ref incr_comp_session_dir) = + cgcx.incr_comp_session_dir + { + let path = incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME); + // If the previous file was deleted, or we get an IO error + // reading the file, then we'll just use `None` as the + // prev_key_map, which will force the code to be recompiled. + let prev = + if path.exists() { ThinLTOKeysMap::load_from_file(&path).ok() } else { None }; + let curr = ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names); + (Some(path), prev, curr) + } else { + // If we don't compile incrementally, we don't need to load the + // import data from LLVM. + assert!(green_modules.is_empty()); + let curr = ThinLTOKeysMap::default(); + (None, None, curr) + }; + info!("thin LTO cache key map loaded"); + info!("prev_key_map: {:#?}", prev_key_map); + info!("curr_key_map: {:#?}", curr_key_map); + + // Throw our data in an `Arc` as we'll be sharing it across threads. We + // also put all memory referenced by the C++ data (buffers, ids, etc) + // into the arc as well. After this we'll create a thin module + // codegen per module in this data. + let shared = Arc::new(ThinShared { + data, + thin_buffers, + serialized_modules: serialized, + module_names, + }); + + let mut copy_jobs = vec![]; + let mut opt_jobs = vec![]; + + info!("checking which modules can be-reused and which have to be re-optimized."); + for (module_index, module_name) in shared.module_names.iter().enumerate() { + let module_name = module_name_to_str(module_name); + if let (Some(prev_key_map), true) = + (prev_key_map.as_ref(), green_modules.contains_key(module_name)) + { + assert!(cgcx.incr_comp_session_dir.is_some()); + + // If a module exists in both the current and the previous session, + // and has the same LTO cache key in both sessions, then we can re-use it + if prev_key_map.keys.get(module_name) == curr_key_map.keys.get(module_name) { + let work_product = green_modules[module_name].clone(); + copy_jobs.push(work_product); + info!(" - {}: re-used", module_name); + assert!(cgcx.incr_comp_session_dir.is_some()); + cgcx.cgu_reuse_tracker.set_actual_reuse(module_name, CguReuse::PostLto); + continue; + } + } + + info!(" - {}: re-compiled", module_name); + opt_jobs.push(LtoModuleCodegen::Thin(ThinModule { + shared: shared.clone(), + idx: module_index, + })); + } + + // Save the current ThinLTO import information for the next compilation + // session, overwriting the previous serialized data (if any). + if let Some(path) = key_map_path { + if let Err(err) = curr_key_map.save_to_file(&path) { + let msg = format!("Error while writing ThinLTO key data: {}", err); + return Err(write::llvm_err(diag_handler, &msg)); + } + } + + Ok((opt_jobs, copy_jobs)) + } +} + +pub(crate) fn run_pass_manager( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: &mut ModuleCodegen, + thin: bool, +) -> Result<(), FatalError> { + let _timer = cgcx.prof.extra_verbose_generic_activity("LLVM_lto_optimize", &*module.name); + let config = cgcx.config(module.kind); + + // Now we have one massive module inside of llmod. Time to run the + // LTO-specific optimization passes that LLVM provides. + // + // This code is based off the code found in llvm's LTO code generator: + // llvm/lib/LTO/LTOCodeGenerator.cpp + debug!("running the pass manager"); + unsafe { + if !llvm::LLVMRustHasModuleFlag( + module.module_llvm.llmod(), + "LTOPostLink".as_ptr().cast(), + 11, + ) { + llvm::LLVMRustAddModuleFlag( + module.module_llvm.llmod(), + llvm::LLVMModFlagBehavior::Error, + "LTOPostLink\0".as_ptr().cast(), + 1, + ); + } + if llvm_util::should_use_new_llvm_pass_manager( + &config.new_llvm_pass_manager, + &cgcx.target_arch, + ) { + let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO }; + let opt_level = config.opt_level.unwrap_or(config::OptLevel::No); + write::optimize_with_new_llvm_pass_manager( + cgcx, + diag_handler, + module, + config, + opt_level, + opt_stage, + )?; + debug!("lto done"); + return Ok(()); + } + + let pm = llvm::LLVMCreatePassManager(); + llvm::LLVMAddAnalysisPasses(module.module_llvm.tm, pm); + + if config.verify_llvm_ir { + let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); + llvm::LLVMRustAddPass(pm, pass.unwrap()); + } + + let opt_level = config + .opt_level + .map(|x| to_llvm_opt_settings(x).0) + .unwrap_or(llvm::CodeGenOptLevel::None); + with_llvm_pmb(module.module_llvm.llmod(), config, opt_level, false, &mut |b| { + if thin { + llvm::LLVMRustPassManagerBuilderPopulateThinLTOPassManager(b, pm); + } else { + llvm::LLVMRustPassManagerBuilderPopulateLTOPassManager( + b, pm, /* Internalize = */ False, /* RunInliner = */ True, + ); + } + }); + + // We always generate bitcode through ThinLTOBuffers, + // which do not support anonymous globals + if config.bitcode_needed() { + let pass = llvm::LLVMRustFindAndCreatePass("name-anon-globals\0".as_ptr().cast()); + llvm::LLVMRustAddPass(pm, pass.unwrap()); + } + + if config.verify_llvm_ir { + let pass = llvm::LLVMRustFindAndCreatePass("verify\0".as_ptr().cast()); + llvm::LLVMRustAddPass(pm, pass.unwrap()); + } + + llvm::LLVMRunPassManager(pm, module.module_llvm.llmod()); + + llvm::LLVMDisposePassManager(pm); + } + debug!("lto done"); + Ok(()) +} + +pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer); + +unsafe impl Send for ModuleBuffer {} +unsafe impl Sync for ModuleBuffer {} + +impl ModuleBuffer { + pub fn new(m: &llvm::Module) -> ModuleBuffer { + ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) }) + } +} + +impl ModuleBufferMethods for ModuleBuffer { + fn data(&self) -> &[u8] { + unsafe { + let ptr = llvm::LLVMRustModuleBufferPtr(self.0); + let len = llvm::LLVMRustModuleBufferLen(self.0); + slice::from_raw_parts(ptr, len) + } + } +} + +impl Drop for ModuleBuffer { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _)); + } + } +} + +pub struct ThinData(&'static mut llvm::ThinLTOData); + +unsafe impl Send for ThinData {} +unsafe impl Sync for ThinData {} + +impl Drop for ThinData { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustFreeThinLTOData(&mut *(self.0 as *mut _)); + } + } +} + +pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer); + +unsafe impl Send for ThinBuffer {} +unsafe impl Sync for ThinBuffer {} + +impl ThinBuffer { + pub fn new(m: &llvm::Module, is_thin: bool) -> ThinBuffer { + unsafe { + let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin); + ThinBuffer(buffer) + } + } +} + +impl ThinBufferMethods for ThinBuffer { + fn data(&self) -> &[u8] { + unsafe { + let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _; + let len = llvm::LLVMRustThinLTOBufferLen(self.0); + slice::from_raw_parts(ptr, len) + } + } +} + +impl Drop for ThinBuffer { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _)); + } + } +} + +pub unsafe fn optimize_thin_module( + thin_module: ThinModule, + cgcx: &CodegenContext, +) -> Result, FatalError> { + let diag_handler = cgcx.create_diag_handler(); + + let module_name = &thin_module.shared.module_names[thin_module.idx]; + let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap()); + let tm = + (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&diag_handler, &e))?; + + // Right now the implementation we've got only works over serialized + // modules, so we create a fresh new LLVM context and parse the module + // into that context. One day, however, we may do this for upstream + // crates but for locally codegened modules we may be able to reuse + // that LLVM Context and Module. + let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); + let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &diag_handler)? as *const _; + let mut module = ModuleCodegen { + module_llvm: ModuleLlvm { llmod_raw, llcx, tm }, + name: thin_module.name().to_string(), + kind: ModuleKind::Regular, + }; + { + let target = &*module.module_llvm.tm; + let llmod = module.module_llvm.llmod(); + save_temp_bitcode(cgcx, &module, "thin-lto-input"); + + // Before we do much else find the "main" `DICompileUnit` that we'll be + // using below. If we find more than one though then rustc has changed + // in a way we're not ready for, so generate an ICE by returning + // an error. + let mut cu1 = ptr::null_mut(); + let mut cu2 = ptr::null_mut(); + llvm::LLVMRustThinLTOGetDICompileUnit(llmod, &mut cu1, &mut cu2); + if !cu2.is_null() { + let msg = "multiple source DICompileUnits found"; + return Err(write::llvm_err(&diag_handler, msg)); + } + + // Up next comes the per-module local analyses that we do for Thin LTO. + // Each of these functions is basically copied from the LLVM + // implementation and then tailored to suit this implementation. Ideally + // each of these would be supported by upstream LLVM but that's perhaps + // a patch for another day! + // + // You can find some more comments about these functions in the LLVM + // bindings we've got (currently `PassWrapper.cpp`) + { + let _timer = + cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name()); + if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)); + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-rename"); + } + + { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name()); + if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)); + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve"); + } + + { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name()); + if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)); + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize"); + } + + { + let _timer = + cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name()); + if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) { + let msg = "failed to prepare thin LTO module"; + return Err(write::llvm_err(&diag_handler, msg)); + } + save_temp_bitcode(cgcx, &module, "thin-lto-after-import"); + } + + // Ok now this is a bit unfortunate. This is also something you won't + // find upstream in LLVM's ThinLTO passes! This is a hack for now to + // work around bugs in LLVM. + // + // First discovered in #45511 it was found that as part of ThinLTO + // importing passes LLVM will import `DICompileUnit` metadata + // information across modules. This means that we'll be working with one + // LLVM module that has multiple `DICompileUnit` instances in it (a + // bunch of `llvm.dbg.cu` members). Unfortunately there's a number of + // bugs in LLVM's backend which generates invalid DWARF in a situation + // like this: + // + // https://bugs.llvm.org/show_bug.cgi?id=35212 + // https://bugs.llvm.org/show_bug.cgi?id=35562 + // + // While the first bug there is fixed the second ended up causing #46346 + // which was basically a resurgence of #45511 after LLVM's bug 35212 was + // fixed. + // + // This function below is a huge hack around this problem. The function + // below is defined in `PassWrapper.cpp` and will basically "merge" + // all `DICompileUnit` instances in a module. Basically it'll take all + // the objects, rewrite all pointers of `DISubprogram` to point to the + // first `DICompileUnit`, and then delete all the other units. + // + // This is probably mangling to the debug info slightly (but hopefully + // not too much) but for now at least gets LLVM to emit valid DWARF (or + // so it appears). Hopefully we can remove this once upstream bugs are + // fixed in LLVM. + { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_thin_lto_patch_debuginfo", thin_module.name()); + llvm::LLVMRustThinLTOPatchDICompileUnit(llmod, cu1); + save_temp_bitcode(cgcx, &module, "thin-lto-after-patch"); + } + + // Alright now that we've done everything related to the ThinLTO + // analysis it's time to run some optimizations! Here we use the same + // `run_pass_manager` as the "fat" LTO above except that we tell it to + // populate a thin-specific pass manager, which presumably LLVM treats a + // little differently. + { + info!("running thin lto passes over {}", module.name); + run_pass_manager(cgcx, &diag_handler, &mut module, true)?; + save_temp_bitcode(cgcx, &module, "thin-lto-after-pm"); + } + } + Ok(module) +} + +/// Maps LLVM module identifiers to their corresponding LLVM LTO cache keys +#[derive(Debug, Default)] +pub struct ThinLTOKeysMap { + // key = llvm name of importing module, value = LLVM cache key + keys: FxHashMap, +} + +impl ThinLTOKeysMap { + fn save_to_file(&self, path: &Path) -> io::Result<()> { + use std::io::Write; + let file = File::create(path)?; + let mut writer = io::BufWriter::new(file); + for (module, key) in &self.keys { + writeln!(writer, "{} {}", module, key)?; + } + Ok(()) + } + + fn load_from_file(path: &Path) -> io::Result { + use std::io::BufRead; + let mut keys = FxHashMap::default(); + let file = File::open(path)?; + for line in io::BufReader::new(file).lines() { + let line = line?; + let mut split = line.split(' '); + let module = split.next().unwrap(); + let key = split.next().unwrap(); + assert_eq!(split.next(), None, "Expected two space-separated values, found {:?}", line); + keys.insert(module.to_string(), key.to_string()); + } + Ok(Self { keys }) + } + + fn from_thin_lto_modules( + data: &ThinData, + modules: &[llvm::ThinLTOModule], + names: &[CString], + ) -> Self { + let keys = iter::zip(modules, names) + .map(|(module, name)| { + let key = build_string(|rust_str| unsafe { + llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0); + }) + .expect("Invalid ThinLTO module key"); + (name.clone().into_string().unwrap(), key) + }) + .collect(); + Self { keys } + } +} + +fn module_name_to_str(c_str: &CStr) -> &str { + c_str.to_str().unwrap_or_else(|e| { + bug!("Encountered non-utf8 LLVM module name `{}`: {}", c_str.to_string_lossy(), e) + }) +} + +pub fn parse_module<'a>( + cx: &'a llvm::Context, + name: &CStr, + data: &[u8], + diag_handler: &Handler, +) -> Result<&'a llvm::Module, FatalError> { + unsafe { + llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr()).ok_or_else( + || { + let msg = "failed to parse bitcode for LTO module"; + write::llvm_err(diag_handler, msg) + }, + ) + } +} diff --git a/compiler/rustc_codegen_llvm/src/back/profiling.rs b/compiler/rustc_codegen_llvm/src/back/profiling.rs new file mode 100644 index 000000000..2741f7d84 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/profiling.rs @@ -0,0 +1,58 @@ +use measureme::{event_id::SEPARATOR_BYTE, EventId, StringComponent, StringId}; +use rustc_data_structures::profiling::{SelfProfiler, TimingGuard}; +use std::ffi::{c_void, CStr}; +use std::os::raw::c_char; +use std::sync::Arc; + +fn llvm_args_to_string_id(profiler: &SelfProfiler, pass_name: &str, ir_name: &str) -> EventId { + let pass_name = profiler.get_or_alloc_cached_string(pass_name); + let mut components = vec![StringComponent::Ref(pass_name)]; + // handle that LazyCallGraph::SCC is a comma separated list within parentheses + let parentheses: &[_] = &['(', ')']; + let trimmed = ir_name.trim_matches(parentheses); + for part in trimmed.split(", ") { + let demangled_ir_name = rustc_demangle::demangle(part).to_string(); + let ir_name = profiler.get_or_alloc_cached_string(demangled_ir_name); + components.push(StringComponent::Value(SEPARATOR_BYTE)); + components.push(StringComponent::Ref(ir_name)); + } + EventId::from_label(profiler.alloc_string(components.as_slice())) +} + +pub struct LlvmSelfProfiler<'a> { + profiler: Arc, + stack: Vec>, + llvm_pass_event_kind: StringId, +} + +impl<'a> LlvmSelfProfiler<'a> { + pub fn new(profiler: Arc) -> Self { + let llvm_pass_event_kind = profiler.alloc_string("LLVM Pass"); + Self { profiler, stack: Vec::default(), llvm_pass_event_kind } + } + + fn before_pass_callback(&'a mut self, pass_name: &str, ir_name: &str) { + let event_id = llvm_args_to_string_id(&self.profiler, pass_name, ir_name); + + self.stack.push(TimingGuard::start(&self.profiler, self.llvm_pass_event_kind, event_id)); + } + fn after_pass_callback(&mut self) { + self.stack.pop(); + } +} + +pub unsafe extern "C" fn selfprofile_before_pass_callback( + llvm_self_profiler: *mut c_void, + pass_name: *const c_char, + ir_name: *const c_char, +) { + let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>); + let pass_name = CStr::from_ptr(pass_name).to_str().expect("valid UTF-8"); + let ir_name = CStr::from_ptr(ir_name).to_str().expect("valid UTF-8"); + llvm_self_profiler.before_pass_callback(pass_name, ir_name); +} + +pub unsafe extern "C" fn selfprofile_after_pass_callback(llvm_self_profiler: *mut c_void) { + let llvm_self_profiler = &mut *(llvm_self_profiler as *mut LlvmSelfProfiler<'_>); + llvm_self_profiler.after_pass_callback(); +} diff --git a/compiler/rustc_codegen_llvm/src/back/write.rs b/compiler/rustc_codegen_llvm/src/back/write.rs new file mode 100644 index 000000000..534d32e8a --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/back/write.rs @@ -0,0 +1,1212 @@ +use crate::back::lto::ThinBuffer; +use crate::back::profiling::{ + selfprofile_after_pass_callback, selfprofile_before_pass_callback, LlvmSelfProfiler, +}; +use crate::base; +use crate::common; +use crate::consts; +use crate::llvm::{self, DiagnosticInfo, PassManager, SMDiagnostic}; +use crate::llvm_util; +use crate::type_::Type; +use crate::LlvmCodegenBackend; +use crate::ModuleLlvm; +use rustc_codegen_ssa::back::link::ensure_removed; +use rustc_codegen_ssa::back::write::{ + BitcodeSection, CodegenContext, EmitObj, ModuleConfig, TargetMachineFactoryConfig, + TargetMachineFactoryFn, +}; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::{CompiledModule, ModuleCodegen}; +use rustc_data_structures::profiling::SelfProfilerRef; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_errors::{FatalError, Handler, Level}; +use rustc_fs_util::{link_or_copy, path_to_c_string}; +use rustc_middle::bug; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::{self, Lto, OutputType, Passes, SplitDwarfKind, SwitchWithOptPath}; +use rustc_session::Session; +use rustc_span::symbol::sym; +use rustc_span::InnerSpan; +use rustc_target::spec::{CodeModel, RelocModel, SanitizerSet, SplitDebuginfo}; +use tracing::debug; + +use libc::{c_char, c_int, c_uint, c_void, size_t}; +use std::ffi::CString; +use std::fs; +use std::io::{self, Write}; +use std::path::{Path, PathBuf}; +use std::slice; +use std::str; +use std::sync::Arc; + +pub fn llvm_err(handler: &rustc_errors::Handler, msg: &str) -> FatalError { + match llvm::last_error() { + Some(err) => handler.fatal(&format!("{}: {}", msg, err)), + None => handler.fatal(msg), + } +} + +pub fn write_output_file<'ll>( + handler: &rustc_errors::Handler, + target: &'ll llvm::TargetMachine, + pm: &llvm::PassManager<'ll>, + m: &'ll llvm::Module, + output: &Path, + dwo_output: Option<&Path>, + file_type: llvm::FileType, + self_profiler_ref: &SelfProfilerRef, +) -> Result<(), FatalError> { + debug!("write_output_file output={:?} dwo_output={:?}", output, dwo_output); + unsafe { + let output_c = path_to_c_string(output); + let dwo_output_c; + let dwo_output_ptr = if let Some(dwo_output) = dwo_output { + dwo_output_c = path_to_c_string(dwo_output); + dwo_output_c.as_ptr() + } else { + std::ptr::null() + }; + let result = llvm::LLVMRustWriteOutputFile( + target, + pm, + m, + output_c.as_ptr(), + dwo_output_ptr, + file_type, + ); + + // Record artifact sizes for self-profiling + if result == llvm::LLVMRustResult::Success { + let artifact_kind = match file_type { + llvm::FileType::ObjectFile => "object_file", + llvm::FileType::AssemblyFile => "assembly_file", + }; + record_artifact_size(self_profiler_ref, artifact_kind, output); + if let Some(dwo_file) = dwo_output { + record_artifact_size(self_profiler_ref, "dwo_file", dwo_file); + } + } + + result.into_result().map_err(|()| { + let msg = format!("could not write output to {}", output.display()); + llvm_err(handler, &msg) + }) + } +} + +pub fn create_informational_target_machine(sess: &Session) -> &'static mut llvm::TargetMachine { + let config = TargetMachineFactoryConfig { split_dwarf_file: None }; + // Can't use query system here quite yet because this function is invoked before the query + // system/tcx is set up. + let features = llvm_util::global_llvm_features(sess, false); + target_machine_factory(sess, config::OptLevel::No, &features)(config) + .unwrap_or_else(|err| llvm_err(sess.diagnostic(), &err).raise()) +} + +pub fn create_target_machine(tcx: TyCtxt<'_>, mod_name: &str) -> &'static mut llvm::TargetMachine { + let split_dwarf_file = if tcx.sess.target_can_use_split_dwarf() { + tcx.output_filenames(()).split_dwarf_path( + tcx.sess.split_debuginfo(), + tcx.sess.opts.unstable_opts.split_dwarf_kind, + Some(mod_name), + ) + } else { + None + }; + let config = TargetMachineFactoryConfig { split_dwarf_file }; + target_machine_factory( + &tcx.sess, + tcx.backend_optimization_level(()), + tcx.global_backend_features(()), + )(config) + .unwrap_or_else(|err| llvm_err(tcx.sess.diagnostic(), &err).raise()) +} + +pub fn to_llvm_opt_settings( + cfg: config::OptLevel, +) -> (llvm::CodeGenOptLevel, llvm::CodeGenOptSize) { + use self::config::OptLevel::*; + match cfg { + No => (llvm::CodeGenOptLevel::None, llvm::CodeGenOptSizeNone), + Less => (llvm::CodeGenOptLevel::Less, llvm::CodeGenOptSizeNone), + Default => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeNone), + Aggressive => (llvm::CodeGenOptLevel::Aggressive, llvm::CodeGenOptSizeNone), + Size => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeDefault), + SizeMin => (llvm::CodeGenOptLevel::Default, llvm::CodeGenOptSizeAggressive), + } +} + +fn to_pass_builder_opt_level(cfg: config::OptLevel) -> llvm::PassBuilderOptLevel { + use config::OptLevel::*; + match cfg { + No => llvm::PassBuilderOptLevel::O0, + Less => llvm::PassBuilderOptLevel::O1, + Default => llvm::PassBuilderOptLevel::O2, + Aggressive => llvm::PassBuilderOptLevel::O3, + Size => llvm::PassBuilderOptLevel::Os, + SizeMin => llvm::PassBuilderOptLevel::Oz, + } +} + +fn to_llvm_relocation_model(relocation_model: RelocModel) -> llvm::RelocModel { + match relocation_model { + RelocModel::Static => llvm::RelocModel::Static, + // LLVM doesn't have a PIE relocation model, it represents PIE as PIC with an extra attribute. + RelocModel::Pic | RelocModel::Pie => llvm::RelocModel::PIC, + RelocModel::DynamicNoPic => llvm::RelocModel::DynamicNoPic, + RelocModel::Ropi => llvm::RelocModel::ROPI, + RelocModel::Rwpi => llvm::RelocModel::RWPI, + RelocModel::RopiRwpi => llvm::RelocModel::ROPI_RWPI, + } +} + +pub(crate) fn to_llvm_code_model(code_model: Option) -> llvm::CodeModel { + match code_model { + Some(CodeModel::Tiny) => llvm::CodeModel::Tiny, + Some(CodeModel::Small) => llvm::CodeModel::Small, + Some(CodeModel::Kernel) => llvm::CodeModel::Kernel, + Some(CodeModel::Medium) => llvm::CodeModel::Medium, + Some(CodeModel::Large) => llvm::CodeModel::Large, + None => llvm::CodeModel::None, + } +} + +pub fn target_machine_factory( + sess: &Session, + optlvl: config::OptLevel, + target_features: &[String], +) -> TargetMachineFactoryFn { + let reloc_model = to_llvm_relocation_model(sess.relocation_model()); + + let (opt_level, _) = to_llvm_opt_settings(optlvl); + let use_softfp = sess.opts.cg.soft_float; + + let ffunction_sections = + sess.opts.unstable_opts.function_sections.unwrap_or(sess.target.function_sections); + let fdata_sections = ffunction_sections; + let funique_section_names = !sess.opts.unstable_opts.no_unique_section_names; + + let code_model = to_llvm_code_model(sess.code_model()); + + let mut singlethread = sess.target.singlethread; + + // On the wasm target once the `atomics` feature is enabled that means that + // we're no longer single-threaded, or otherwise we don't want LLVM to + // lower atomic operations to single-threaded operations. + if singlethread && sess.target.is_like_wasm && sess.target_features.contains(&sym::atomics) { + singlethread = false; + } + + let triple = SmallCStr::new(&sess.target.llvm_target); + let cpu = SmallCStr::new(llvm_util::target_cpu(sess)); + let features = CString::new(target_features.join(",")).unwrap(); + let abi = SmallCStr::new(&sess.target.llvm_abiname); + let trap_unreachable = + sess.opts.unstable_opts.trap_unreachable.unwrap_or(sess.target.trap_unreachable); + let emit_stack_size_section = sess.opts.unstable_opts.emit_stack_sizes; + + let asm_comments = sess.asm_comments(); + let relax_elf_relocations = + sess.opts.unstable_opts.relax_elf_relocations.unwrap_or(sess.target.relax_elf_relocations); + + let use_init_array = + !sess.opts.unstable_opts.use_ctors_section.unwrap_or(sess.target.use_ctors_section); + + let path_mapping = sess.source_map().path_mapping().clone(); + + Arc::new(move |config: TargetMachineFactoryConfig| { + let split_dwarf_file = + path_mapping.map_prefix(config.split_dwarf_file.unwrap_or_default()).0; + let split_dwarf_file = CString::new(split_dwarf_file.to_str().unwrap()).unwrap(); + + let tm = unsafe { + llvm::LLVMRustCreateTargetMachine( + triple.as_ptr(), + cpu.as_ptr(), + features.as_ptr(), + abi.as_ptr(), + code_model, + reloc_model, + opt_level, + use_softfp, + ffunction_sections, + fdata_sections, + funique_section_names, + trap_unreachable, + singlethread, + asm_comments, + emit_stack_size_section, + relax_elf_relocations, + use_init_array, + split_dwarf_file.as_ptr(), + ) + }; + + tm.ok_or_else(|| { + format!("Could not create LLVM TargetMachine for triple: {}", triple.to_str().unwrap()) + }) + }) +} + +pub(crate) fn save_temp_bitcode( + cgcx: &CodegenContext, + module: &ModuleCodegen, + name: &str, +) { + if !cgcx.save_temps { + return; + } + unsafe { + let ext = format!("{}.bc", name); + let cgu = Some(&module.name[..]); + let path = cgcx.output_filenames.temp_path_ext(&ext, cgu); + let cstr = path_to_c_string(&path); + let llmod = module.module_llvm.llmod(); + llvm::LLVMWriteBitcodeToFile(llmod, cstr.as_ptr()); + } +} + +pub struct DiagnosticHandlers<'a> { + data: *mut (&'a CodegenContext, &'a Handler), + llcx: &'a llvm::Context, + old_handler: Option<&'a llvm::DiagnosticHandler>, +} + +impl<'a> DiagnosticHandlers<'a> { + pub fn new( + cgcx: &'a CodegenContext, + handler: &'a Handler, + llcx: &'a llvm::Context, + ) -> Self { + let remark_passes_all: bool; + let remark_passes: Vec; + match &cgcx.remark { + Passes::All => { + remark_passes_all = true; + remark_passes = Vec::new(); + } + Passes::Some(passes) => { + remark_passes_all = false; + remark_passes = + passes.iter().map(|name| CString::new(name.as_str()).unwrap()).collect(); + } + }; + let remark_passes: Vec<*const c_char> = + remark_passes.iter().map(|name: &CString| name.as_ptr()).collect(); + let data = Box::into_raw(Box::new((cgcx, handler))); + unsafe { + let old_handler = llvm::LLVMRustContextGetDiagnosticHandler(llcx); + llvm::LLVMRustContextConfigureDiagnosticHandler( + llcx, + diagnostic_handler, + data.cast(), + remark_passes_all, + remark_passes.as_ptr(), + remark_passes.len(), + ); + llvm::LLVMRustSetInlineAsmDiagnosticHandler(llcx, inline_asm_handler, data.cast()); + DiagnosticHandlers { data, llcx, old_handler } + } + } +} + +impl<'a> Drop for DiagnosticHandlers<'a> { + fn drop(&mut self) { + use std::ptr::null_mut; + unsafe { + llvm::LLVMRustSetInlineAsmDiagnosticHandler(self.llcx, inline_asm_handler, null_mut()); + llvm::LLVMRustContextSetDiagnosticHandler(self.llcx, self.old_handler); + drop(Box::from_raw(self.data)); + } + } +} + +fn report_inline_asm( + cgcx: &CodegenContext, + msg: String, + level: llvm::DiagnosticLevel, + mut cookie: c_uint, + source: Option<(String, Vec)>, +) { + // In LTO build we may get srcloc values from other crates which are invalid + // since they use a different source map. To be safe we just suppress these + // in LTO builds. + if matches!(cgcx.lto, Lto::Fat | Lto::Thin) { + cookie = 0; + } + let level = match level { + llvm::DiagnosticLevel::Error => Level::Error { lint: false }, + llvm::DiagnosticLevel::Warning => Level::Warning(None), + llvm::DiagnosticLevel::Note | llvm::DiagnosticLevel::Remark => Level::Note, + }; + cgcx.diag_emitter.inline_asm_error(cookie as u32, msg, level, source); +} + +unsafe extern "C" fn inline_asm_handler(diag: &SMDiagnostic, user: *const c_void, cookie: c_uint) { + if user.is_null() { + return; + } + let (cgcx, _) = *(user as *const (&CodegenContext, &Handler)); + + let smdiag = llvm::diagnostic::SrcMgrDiagnostic::unpack(diag); + report_inline_asm(cgcx, smdiag.message, smdiag.level, cookie, smdiag.source); +} + +unsafe extern "C" fn diagnostic_handler(info: &DiagnosticInfo, user: *mut c_void) { + if user.is_null() { + return; + } + let (cgcx, diag_handler) = *(user as *const (&CodegenContext, &Handler)); + + match llvm::diagnostic::Diagnostic::unpack(info) { + llvm::diagnostic::InlineAsm(inline) => { + report_inline_asm(cgcx, inline.message, inline.level, inline.cookie, inline.source); + } + + llvm::diagnostic::Optimization(opt) => { + let enabled = match cgcx.remark { + Passes::All => true, + Passes::Some(ref v) => v.iter().any(|s| *s == opt.pass_name), + }; + + if enabled { + diag_handler.note_without_error(&format!( + "{}:{}:{}: {}: {}", + opt.filename, opt.line, opt.column, opt.pass_name, opt.message, + )); + } + } + llvm::diagnostic::PGO(diagnostic_ref) | llvm::diagnostic::Linker(diagnostic_ref) => { + let msg = llvm::build_string(|s| { + llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) + }) + .expect("non-UTF8 diagnostic"); + diag_handler.warn(&msg); + } + llvm::diagnostic::Unsupported(diagnostic_ref) => { + let msg = llvm::build_string(|s| { + llvm::LLVMRustWriteDiagnosticInfoToString(diagnostic_ref, s) + }) + .expect("non-UTF8 diagnostic"); + diag_handler.err(&msg); + } + llvm::diagnostic::UnknownDiagnostic(..) => {} + } +} + +fn get_pgo_gen_path(config: &ModuleConfig) -> Option { + match config.pgo_gen { + SwitchWithOptPath::Enabled(ref opt_dir_path) => { + let path = if let Some(dir_path) = opt_dir_path { + dir_path.join("default_%m.profraw") + } else { + PathBuf::from("default_%m.profraw") + }; + + Some(CString::new(format!("{}", path.display())).unwrap()) + } + SwitchWithOptPath::Disabled => None, + } +} + +fn get_pgo_use_path(config: &ModuleConfig) -> Option { + config + .pgo_use + .as_ref() + .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) +} + +fn get_pgo_sample_use_path(config: &ModuleConfig) -> Option { + config + .pgo_sample_use + .as_ref() + .map(|path_buf| CString::new(path_buf.to_string_lossy().as_bytes()).unwrap()) +} + +pub(crate) unsafe fn optimize_with_new_llvm_pass_manager( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: &ModuleCodegen, + config: &ModuleConfig, + opt_level: config::OptLevel, + opt_stage: llvm::OptStage, +) -> Result<(), FatalError> { + let unroll_loops = + opt_level != config::OptLevel::Size && opt_level != config::OptLevel::SizeMin; + let using_thin_buffers = opt_stage == llvm::OptStage::PreLinkThinLTO || config.bitcode_needed(); + let pgo_gen_path = get_pgo_gen_path(config); + let pgo_use_path = get_pgo_use_path(config); + let pgo_sample_use_path = get_pgo_sample_use_path(config); + let is_lto = opt_stage == llvm::OptStage::ThinLTO || opt_stage == llvm::OptStage::FatLTO; + // Sanitizer instrumentation is only inserted during the pre-link optimization stage. + let sanitizer_options = if !is_lto { + Some(llvm::SanitizerOptions { + sanitize_address: config.sanitizer.contains(SanitizerSet::ADDRESS), + sanitize_address_recover: config.sanitizer_recover.contains(SanitizerSet::ADDRESS), + sanitize_memory: config.sanitizer.contains(SanitizerSet::MEMORY), + sanitize_memory_recover: config.sanitizer_recover.contains(SanitizerSet::MEMORY), + sanitize_memory_track_origins: config.sanitizer_memory_track_origins as c_int, + sanitize_thread: config.sanitizer.contains(SanitizerSet::THREAD), + sanitize_hwaddress: config.sanitizer.contains(SanitizerSet::HWADDRESS), + sanitize_hwaddress_recover: config.sanitizer_recover.contains(SanitizerSet::HWADDRESS), + }) + } else { + None + }; + + let mut llvm_profiler = if cgcx.prof.llvm_recording_enabled() { + Some(LlvmSelfProfiler::new(cgcx.prof.get_self_profiler().unwrap())) + } else { + None + }; + + let llvm_selfprofiler = + llvm_profiler.as_mut().map(|s| s as *mut _ as *mut c_void).unwrap_or(std::ptr::null_mut()); + + let extra_passes = if !is_lto { config.passes.join(",") } else { "".to_string() }; + + let llvm_plugins = config.llvm_plugins.join(","); + + // FIXME: NewPM doesn't provide a facility to pass custom InlineParams. + // We would have to add upstream support for this first, before we can support + // config.inline_threshold and our more aggressive default thresholds. + let result = llvm::LLVMRustOptimizeWithNewPassManager( + module.module_llvm.llmod(), + &*module.module_llvm.tm, + to_pass_builder_opt_level(opt_level), + opt_stage, + config.no_prepopulate_passes, + config.verify_llvm_ir, + using_thin_buffers, + config.merge_functions, + unroll_loops, + config.vectorize_slp, + config.vectorize_loop, + config.no_builtins, + config.emit_lifetime_markers, + sanitizer_options.as_ref(), + pgo_gen_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + pgo_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + config.instrument_coverage, + config.instrument_gcov, + pgo_sample_use_path.as_ref().map_or(std::ptr::null(), |s| s.as_ptr()), + config.debug_info_for_profiling, + llvm_selfprofiler, + selfprofile_before_pass_callback, + selfprofile_after_pass_callback, + extra_passes.as_ptr().cast(), + extra_passes.len(), + llvm_plugins.as_ptr().cast(), + llvm_plugins.len(), + ); + result.into_result().map_err(|()| llvm_err(diag_handler, "failed to run LLVM passes")) +} + +// Unsafe due to LLVM calls. +pub(crate) unsafe fn optimize( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: &ModuleCodegen, + config: &ModuleConfig, +) -> Result<(), FatalError> { + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_optimize", &*module.name); + + let llmod = module.module_llvm.llmod(); + let llcx = &*module.module_llvm.llcx; + let tm = &*module.module_llvm.tm; + let _handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + + let module_name = module.name.clone(); + let module_name = Some(&module_name[..]); + + if let Some(false) = config.new_llvm_pass_manager && llvm_util::get_version() >= (15, 0, 0) { + diag_handler.warn( + "ignoring `-Z new-llvm-pass-manager=no`, which is no longer supported with LLVM 15", + ); + } + + if config.emit_no_opt_bc { + let out = cgcx.output_filenames.temp_path_ext("no-opt.bc", module_name); + let out = path_to_c_string(&out); + llvm::LLVMWriteBitcodeToFile(llmod, out.as_ptr()); + } + + if let Some(opt_level) = config.opt_level { + if llvm_util::should_use_new_llvm_pass_manager( + &config.new_llvm_pass_manager, + &cgcx.target_arch, + ) { + let opt_stage = match cgcx.lto { + Lto::Fat => llvm::OptStage::PreLinkFatLTO, + Lto::Thin | Lto::ThinLocal => llvm::OptStage::PreLinkThinLTO, + _ if cgcx.opts.cg.linker_plugin_lto.enabled() => llvm::OptStage::PreLinkThinLTO, + _ => llvm::OptStage::PreLinkNoLTO, + }; + return optimize_with_new_llvm_pass_manager( + cgcx, + diag_handler, + module, + config, + opt_level, + opt_stage, + ); + } + + if cgcx.prof.llvm_recording_enabled() { + diag_handler + .warn("`-Z self-profile-events = llvm` requires `-Z new-llvm-pass-manager`"); + } + + // Create the two optimizing pass managers. These mirror what clang + // does, and are by populated by LLVM's default PassManagerBuilder. + // Each manager has a different set of passes, but they also share + // some common passes. + let fpm = llvm::LLVMCreateFunctionPassManagerForModule(llmod); + let mpm = llvm::LLVMCreatePassManager(); + + { + let find_pass = |pass_name: &str| { + let pass_name = SmallCStr::new(pass_name); + llvm::LLVMRustFindAndCreatePass(pass_name.as_ptr()) + }; + + if config.verify_llvm_ir { + // Verification should run as the very first pass. + llvm::LLVMRustAddPass(fpm, find_pass("verify").unwrap()); + } + + let mut extra_passes = Vec::new(); + let mut have_name_anon_globals_pass = false; + + for pass_name in &config.passes { + if pass_name == "lint" { + // Linting should also be performed early, directly on the generated IR. + llvm::LLVMRustAddPass(fpm, find_pass("lint").unwrap()); + continue; + } + + if let Some(pass) = find_pass(pass_name) { + extra_passes.push(pass); + } else { + diag_handler.warn(&format!("unknown pass `{}`, ignoring", pass_name)); + } + + if pass_name == "name-anon-globals" { + have_name_anon_globals_pass = true; + } + } + + // Instrumentation must be inserted before optimization, + // otherwise LLVM may optimize some functions away which + // breaks llvm-cov. + // + // This mirrors what Clang does in lib/CodeGen/BackendUtil.cpp. + if config.instrument_gcov { + llvm::LLVMRustAddPass(mpm, find_pass("insert-gcov-profiling").unwrap()); + } + if config.instrument_coverage { + llvm::LLVMRustAddPass(mpm, find_pass("instrprof").unwrap()); + } + if config.debug_info_for_profiling { + llvm::LLVMRustAddPass(mpm, find_pass("add-discriminators").unwrap()); + } + + add_sanitizer_passes(config, &mut extra_passes); + + // Some options cause LLVM bitcode to be emitted, which uses ThinLTOBuffers, so we need + // to make sure we run LLVM's NameAnonGlobals pass when emitting bitcode; otherwise + // we'll get errors in LLVM. + let using_thin_buffers = config.bitcode_needed(); + if !config.no_prepopulate_passes { + llvm::LLVMAddAnalysisPasses(tm, fpm); + llvm::LLVMAddAnalysisPasses(tm, mpm); + let opt_level = to_llvm_opt_settings(opt_level).0; + let prepare_for_thin_lto = cgcx.lto == Lto::Thin + || cgcx.lto == Lto::ThinLocal + || (cgcx.lto != Lto::Fat && cgcx.opts.cg.linker_plugin_lto.enabled()); + with_llvm_pmb(llmod, config, opt_level, prepare_for_thin_lto, &mut |b| { + llvm::LLVMRustAddLastExtensionPasses( + b, + extra_passes.as_ptr(), + extra_passes.len() as size_t, + ); + llvm::LLVMRustPassManagerBuilderPopulateFunctionPassManager(b, fpm); + llvm::LLVMRustPassManagerBuilderPopulateModulePassManager(b, mpm); + }); + + have_name_anon_globals_pass = have_name_anon_globals_pass || prepare_for_thin_lto; + if using_thin_buffers && !prepare_for_thin_lto { + llvm::LLVMRustAddPass(mpm, find_pass("name-anon-globals").unwrap()); + have_name_anon_globals_pass = true; + } + } else { + // If we don't use the standard pipeline, directly populate the MPM + // with the extra passes. + for pass in extra_passes { + llvm::LLVMRustAddPass(mpm, pass); + } + } + + if using_thin_buffers && !have_name_anon_globals_pass { + // As described above, this will probably cause an error in LLVM + if config.no_prepopulate_passes { + diag_handler.err( + "The current compilation is going to use thin LTO buffers \ + without running LLVM's NameAnonGlobals pass. \ + This will likely cause errors in LLVM. Consider adding \ + -C passes=name-anon-globals to the compiler command line.", + ); + } else { + bug!( + "We are using thin LTO buffers without running the NameAnonGlobals pass. \ + This will likely cause errors in LLVM and should never happen." + ); + } + } + } + + diag_handler.abort_if_errors(); + + // Finally, run the actual optimization passes + { + let _timer = cgcx.prof.extra_verbose_generic_activity( + "LLVM_module_optimize_function_passes", + &*module.name, + ); + llvm::LLVMRustRunFunctionPassManager(fpm, llmod); + } + { + let _timer = cgcx.prof.extra_verbose_generic_activity( + "LLVM_module_optimize_module_passes", + &*module.name, + ); + llvm::LLVMRunPassManager(mpm, llmod); + } + + // Deallocate managers that we're now done with + llvm::LLVMDisposePassManager(fpm); + llvm::LLVMDisposePassManager(mpm); + } + Ok(()) +} + +unsafe fn add_sanitizer_passes(config: &ModuleConfig, passes: &mut Vec<&'static mut llvm::Pass>) { + if config.sanitizer.contains(SanitizerSet::ADDRESS) { + let recover = config.sanitizer_recover.contains(SanitizerSet::ADDRESS); + passes.push(llvm::LLVMRustCreateAddressSanitizerFunctionPass(recover)); + passes.push(llvm::LLVMRustCreateModuleAddressSanitizerPass(recover)); + } + if config.sanitizer.contains(SanitizerSet::MEMORY) { + let track_origins = config.sanitizer_memory_track_origins as c_int; + let recover = config.sanitizer_recover.contains(SanitizerSet::MEMORY); + passes.push(llvm::LLVMRustCreateMemorySanitizerPass(track_origins, recover)); + } + if config.sanitizer.contains(SanitizerSet::THREAD) { + passes.push(llvm::LLVMRustCreateThreadSanitizerPass()); + } + if config.sanitizer.contains(SanitizerSet::HWADDRESS) { + let recover = config.sanitizer_recover.contains(SanitizerSet::HWADDRESS); + passes.push(llvm::LLVMRustCreateHWAddressSanitizerPass(recover)); + } +} + +pub(crate) fn link( + cgcx: &CodegenContext, + diag_handler: &Handler, + mut modules: Vec>, +) -> Result, FatalError> { + use super::lto::{Linker, ModuleBuffer}; + // Sort the modules by name to ensure to ensure deterministic behavior. + modules.sort_by(|a, b| a.name.cmp(&b.name)); + let (first, elements) = + modules.split_first().expect("Bug! modules must contain at least one module."); + + let mut linker = Linker::new(first.module_llvm.llmod()); + for module in elements { + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_link_module", &*module.name); + let buffer = ModuleBuffer::new(module.module_llvm.llmod()); + linker.add(buffer.data()).map_err(|()| { + let msg = format!("failed to serialize module {:?}", module.name); + llvm_err(diag_handler, &msg) + })?; + } + drop(linker); + Ok(modules.remove(0)) +} + +pub(crate) unsafe fn codegen( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: ModuleCodegen, + config: &ModuleConfig, +) -> Result { + let _timer = cgcx.prof.generic_activity_with_arg("LLVM_module_codegen", &*module.name); + { + let llmod = module.module_llvm.llmod(); + let llcx = &*module.module_llvm.llcx; + let tm = &*module.module_llvm.tm; + let module_name = module.name.clone(); + let module_name = Some(&module_name[..]); + let handlers = DiagnosticHandlers::new(cgcx, diag_handler, llcx); + + if cgcx.msvc_imps_needed { + create_msvc_imps(cgcx, llcx, llmod); + } + + // A codegen-specific pass manager is used to generate object + // files for an LLVM module. + // + // Apparently each of these pass managers is a one-shot kind of + // thing, so we create a new one for each type of output. The + // pass manager passed to the closure should be ensured to not + // escape the closure itself, and the manager should only be + // used once. + unsafe fn with_codegen<'ll, F, R>( + tm: &'ll llvm::TargetMachine, + llmod: &'ll llvm::Module, + no_builtins: bool, + f: F, + ) -> R + where + F: FnOnce(&'ll mut PassManager<'ll>) -> R, + { + let cpm = llvm::LLVMCreatePassManager(); + llvm::LLVMAddAnalysisPasses(tm, cpm); + llvm::LLVMRustAddLibraryInfo(cpm, llmod, no_builtins); + f(cpm) + } + + // Two things to note: + // - If object files are just LLVM bitcode we write bitcode, copy it to + // the .o file, and delete the bitcode if it wasn't otherwise + // requested. + // - If we don't have the integrated assembler then we need to emit + // asm from LLVM and use `gcc` to create the object file. + + let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name); + let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, module_name); + + if config.bitcode_needed() { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_module_codegen_make_bitcode", &*module.name); + let thin = ThinBuffer::new(llmod, config.emit_thin_lto); + let data = thin.data(); + + if let Some(bitcode_filename) = bc_out.file_name() { + cgcx.prof.artifact_size( + "llvm_bitcode", + bitcode_filename.to_string_lossy(), + data.len() as u64, + ); + } + + if config.emit_bc || config.emit_obj == EmitObj::Bitcode { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_module_codegen_emit_bitcode", &*module.name); + if let Err(e) = fs::write(&bc_out, data) { + let msg = format!("failed to write bytecode to {}: {}", bc_out.display(), e); + diag_handler.err(&msg); + } + } + + if config.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full) { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_module_codegen_embed_bitcode", &*module.name); + embed_bitcode(cgcx, llcx, llmod, &config.bc_cmdline, data); + } + } + + if config.emit_ir { + let _timer = + cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_ir", &*module.name); + let out = cgcx.output_filenames.temp_path(OutputType::LlvmAssembly, module_name); + let out_c = path_to_c_string(&out); + + extern "C" fn demangle_callback( + input_ptr: *const c_char, + input_len: size_t, + output_ptr: *mut c_char, + output_len: size_t, + ) -> size_t { + let input = + unsafe { slice::from_raw_parts(input_ptr as *const u8, input_len as usize) }; + + let Ok(input) = str::from_utf8(input) else { return 0 }; + + let output = unsafe { + slice::from_raw_parts_mut(output_ptr as *mut u8, output_len as usize) + }; + let mut cursor = io::Cursor::new(output); + + let Ok(demangled) = rustc_demangle::try_demangle(input) else { return 0 }; + + if write!(cursor, "{:#}", demangled).is_err() { + // Possible only if provided buffer is not big enough + return 0; + } + + cursor.position() as size_t + } + + let result = llvm::LLVMRustPrintModule(llmod, out_c.as_ptr(), demangle_callback); + + if result == llvm::LLVMRustResult::Success { + record_artifact_size(&cgcx.prof, "llvm_ir", &out); + } + + result.into_result().map_err(|()| { + let msg = format!("failed to write LLVM IR to {}", out.display()); + llvm_err(diag_handler, &msg) + })?; + } + + if config.emit_asm { + let _timer = + cgcx.prof.generic_activity_with_arg("LLVM_module_codegen_emit_asm", &*module.name); + let path = cgcx.output_filenames.temp_path(OutputType::Assembly, module_name); + + // We can't use the same module for asm and object code output, + // because that triggers various errors like invalid IR or broken + // binaries. So we must clone the module to produce the asm output + // if we are also producing object code. + let llmod = if let EmitObj::ObjectCode(_) = config.emit_obj { + llvm::LLVMCloneModule(llmod) + } else { + llmod + }; + with_codegen(tm, llmod, config.no_builtins, |cpm| { + write_output_file( + diag_handler, + tm, + cpm, + llmod, + &path, + None, + llvm::FileType::AssemblyFile, + &cgcx.prof, + ) + })?; + } + + match config.emit_obj { + EmitObj::ObjectCode(_) => { + let _timer = cgcx + .prof + .generic_activity_with_arg("LLVM_module_codegen_emit_obj", &*module.name); + + let dwo_out = cgcx.output_filenames.temp_path_dwo(module_name); + let dwo_out = match (cgcx.split_debuginfo, cgcx.split_dwarf_kind) { + // Don't change how DWARF is emitted when disabled. + (SplitDebuginfo::Off, _) => None, + // Don't provide a DWARF object path if split debuginfo is enabled but this is + // a platform that doesn't support Split DWARF. + _ if !cgcx.target_can_use_split_dwarf => None, + // Don't provide a DWARF object path in single mode, sections will be written + // into the object as normal but ignored by linker. + (_, SplitDwarfKind::Single) => None, + // Emit (a subset of the) DWARF into a separate dwarf object file in split + // mode. + (_, SplitDwarfKind::Split) => Some(dwo_out.as_path()), + }; + + with_codegen(tm, llmod, config.no_builtins, |cpm| { + write_output_file( + diag_handler, + tm, + cpm, + llmod, + &obj_out, + dwo_out, + llvm::FileType::ObjectFile, + &cgcx.prof, + ) + })?; + } + + EmitObj::Bitcode => { + debug!("copying bitcode {:?} to obj {:?}", bc_out, obj_out); + if let Err(e) = link_or_copy(&bc_out, &obj_out) { + diag_handler.err(&format!("failed to copy bitcode to object file: {}", e)); + } + + if !config.emit_bc { + debug!("removing_bitcode {:?}", bc_out); + ensure_removed(diag_handler, &bc_out); + } + } + + EmitObj::None => {} + } + + drop(handlers); + } + + Ok(module.into_compiled_module( + config.emit_obj != EmitObj::None, + cgcx.target_can_use_split_dwarf + && cgcx.split_debuginfo != SplitDebuginfo::Off + && cgcx.split_dwarf_kind == SplitDwarfKind::Split, + config.emit_bc, + &cgcx.output_filenames, + )) +} + +fn create_section_with_flags_asm(section_name: &str, section_flags: &str, data: &[u8]) -> Vec { + let mut asm = format!(".section {},\"{}\"\n", section_name, section_flags).into_bytes(); + asm.extend_from_slice(b".ascii \""); + asm.reserve(data.len()); + for &byte in data { + if byte == b'\\' || byte == b'"' { + asm.push(b'\\'); + asm.push(byte); + } else if byte < 0x20 || byte >= 0x80 { + // Avoid non UTF-8 inline assembly. Use octal escape sequence, because it is fixed + // width, while hex escapes will consume following characters. + asm.push(b'\\'); + asm.push(b'0' + ((byte >> 6) & 0x7)); + asm.push(b'0' + ((byte >> 3) & 0x7)); + asm.push(b'0' + ((byte >> 0) & 0x7)); + } else { + asm.push(byte); + } + } + asm.extend_from_slice(b"\"\n"); + asm +} + +/// Embed the bitcode of an LLVM module in the LLVM module itself. +/// +/// This is done primarily for iOS where it appears to be standard to compile C +/// code at least with `-fembed-bitcode` which creates two sections in the +/// executable: +/// +/// * __LLVM,__bitcode +/// * __LLVM,__cmdline +/// +/// It appears *both* of these sections are necessary to get the linker to +/// recognize what's going on. A suitable cmdline value is taken from the +/// target spec. +/// +/// Furthermore debug/O1 builds don't actually embed bitcode but rather just +/// embed an empty section. +/// +/// Basically all of this is us attempting to follow in the footsteps of clang +/// on iOS. See #35968 for lots more info. +unsafe fn embed_bitcode( + cgcx: &CodegenContext, + llcx: &llvm::Context, + llmod: &llvm::Module, + cmdline: &str, + bitcode: &[u8], +) { + // We're adding custom sections to the output object file, but we definitely + // do not want these custom sections to make their way into the final linked + // executable. The purpose of these custom sections is for tooling + // surrounding object files to work with the LLVM IR, if necessary. For + // example rustc's own LTO will look for LLVM IR inside of the object file + // in these sections by default. + // + // To handle this is a bit different depending on the object file format + // used by the backend, broken down into a few different categories: + // + // * Mach-O - this is for macOS. Inspecting the source code for the native + // linker here shows that the `.llvmbc` and `.llvmcmd` sections are + // automatically skipped by the linker. In that case there's nothing extra + // that we need to do here. + // + // * Wasm - the native LLD linker is hard-coded to skip `.llvmbc` and + // `.llvmcmd` sections, so there's nothing extra we need to do. + // + // * COFF - if we don't do anything the linker will by default copy all + // these sections to the output artifact, not what we want! To subvert + // this we want to flag the sections we inserted here as + // `IMAGE_SCN_LNK_REMOVE`. + // + // * ELF - this is very similar to COFF above. One difference is that these + // sections are removed from the output linked artifact when + // `--gc-sections` is passed, which we pass by default. If that flag isn't + // passed though then these sections will show up in the final output. + // Additionally the flag that we need to set here is `SHF_EXCLUDE`. + // + // Unfortunately, LLVM provides no way to set custom section flags. For ELF + // and COFF we emit the sections using module level inline assembly for that + // reason (see issue #90326 for historical background). + let is_apple = cgcx.opts.target_triple.triple().contains("-ios") + || cgcx.opts.target_triple.triple().contains("-darwin") + || cgcx.opts.target_triple.triple().contains("-tvos") + || cgcx.opts.target_triple.triple().contains("-watchos"); + if is_apple + || cgcx.opts.target_triple.triple().starts_with("wasm") + || cgcx.opts.target_triple.triple().starts_with("asmjs") + { + // We don't need custom section flags, create LLVM globals. + let llconst = common::bytes_in_context(llcx, bitcode); + let llglobal = llvm::LLVMAddGlobal( + llmod, + common::val_ty(llconst), + "rustc.embedded.module\0".as_ptr().cast(), + ); + llvm::LLVMSetInitializer(llglobal, llconst); + + let section = if is_apple { "__LLVM,__bitcode\0" } else { ".llvmbc\0" }; + llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); + llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::LLVMSetGlobalConstant(llglobal, llvm::True); + + let llconst = common::bytes_in_context(llcx, cmdline.as_bytes()); + let llglobal = llvm::LLVMAddGlobal( + llmod, + common::val_ty(llconst), + "rustc.embedded.cmdline\0".as_ptr().cast(), + ); + llvm::LLVMSetInitializer(llglobal, llconst); + let section = if is_apple { "__LLVM,__cmdline\0" } else { ".llvmcmd\0" }; + llvm::LLVMSetSection(llglobal, section.as_ptr().cast()); + llvm::LLVMRustSetLinkage(llglobal, llvm::Linkage::PrivateLinkage); + } else { + // We need custom section flags, so emit module-level inline assembly. + let section_flags = if cgcx.is_pe_coff { "n" } else { "e" }; + let asm = create_section_with_flags_asm(".llvmbc", section_flags, bitcode); + llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + let asm = create_section_with_flags_asm(".llvmcmd", section_flags, cmdline.as_bytes()); + llvm::LLVMRustAppendModuleInlineAsm(llmod, asm.as_ptr().cast(), asm.len()); + } +} + +pub unsafe fn with_llvm_pmb( + llmod: &llvm::Module, + config: &ModuleConfig, + opt_level: llvm::CodeGenOptLevel, + prepare_for_thin_lto: bool, + f: &mut dyn FnMut(&llvm::PassManagerBuilder), +) { + use std::ptr; + + // Create the PassManagerBuilder for LLVM. We configure it with + // reasonable defaults and prepare it to actually populate the pass + // manager. + let builder = llvm::LLVMRustPassManagerBuilderCreate(); + let opt_size = config.opt_size.map_or(llvm::CodeGenOptSizeNone, |x| to_llvm_opt_settings(x).1); + let inline_threshold = config.inline_threshold; + let pgo_gen_path = get_pgo_gen_path(config); + let pgo_use_path = get_pgo_use_path(config); + let pgo_sample_use_path = get_pgo_sample_use_path(config); + + llvm::LLVMRustConfigurePassManagerBuilder( + builder, + opt_level, + config.merge_functions, + config.vectorize_slp, + config.vectorize_loop, + prepare_for_thin_lto, + pgo_gen_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + pgo_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + pgo_sample_use_path.as_ref().map_or(ptr::null(), |s| s.as_ptr()), + opt_size as c_int, + ); + + llvm::LLVMRustAddBuilderLibraryInfo(builder, llmod, config.no_builtins); + + // Here we match what clang does (kinda). For O0 we only inline + // always-inline functions (but don't add lifetime intrinsics), at O1 we + // inline with lifetime intrinsics, and O2+ we add an inliner with a + // thresholds copied from clang. + match (opt_level, opt_size, inline_threshold) { + (.., Some(t)) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, t); + } + (llvm::CodeGenOptLevel::Aggressive, ..) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 275); + } + (_, llvm::CodeGenOptSizeDefault, _) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 75); + } + (_, llvm::CodeGenOptSizeAggressive, _) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 25); + } + (llvm::CodeGenOptLevel::None, ..) => { + llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); + } + (llvm::CodeGenOptLevel::Less, ..) => { + llvm::LLVMRustAddAlwaysInlinePass(builder, config.emit_lifetime_markers); + } + (llvm::CodeGenOptLevel::Default, ..) => { + llvm::LLVMRustPassManagerBuilderUseInlinerWithThreshold(builder, 225); + } + } + + f(builder); + llvm::LLVMRustPassManagerBuilderDispose(builder); +} + +// Create a `__imp_ = &symbol` global for every public static `symbol`. +// This is required to satisfy `dllimport` references to static data in .rlibs +// when using MSVC linker. We do this only for data, as linker can fix up +// code references on its own. +// See #26591, #27438 +fn create_msvc_imps( + cgcx: &CodegenContext, + llcx: &llvm::Context, + llmod: &llvm::Module, +) { + if !cgcx.msvc_imps_needed { + return; + } + // The x86 ABI seems to require that leading underscores are added to symbol + // names, so we need an extra underscore on x86. There's also a leading + // '\x01' here which disables LLVM's symbol mangling (e.g., no extra + // underscores added in front). + let prefix = if cgcx.target_arch == "x86" { "\x01__imp__" } else { "\x01__imp_" }; + + unsafe { + let i8p_ty = Type::i8p_llcx(llcx); + let globals = base::iter_globals(llmod) + .filter(|&val| { + llvm::LLVMRustGetLinkage(val) == llvm::Linkage::ExternalLinkage + && llvm::LLVMIsDeclaration(val) == 0 + }) + .filter_map(|val| { + // Exclude some symbols that we know are not Rust symbols. + let name = llvm::get_value_name(val); + if ignored(name) { None } else { Some((val, name)) } + }) + .map(move |(val, name)| { + let mut imp_name = prefix.as_bytes().to_vec(); + imp_name.extend(name); + let imp_name = CString::new(imp_name).unwrap(); + (imp_name, val) + }) + .collect::>(); + + for (imp_name, val) in globals { + let imp = llvm::LLVMAddGlobal(llmod, i8p_ty, imp_name.as_ptr().cast()); + llvm::LLVMSetInitializer(imp, consts::ptrcast(val, i8p_ty)); + llvm::LLVMRustSetLinkage(imp, llvm::Linkage::ExternalLinkage); + } + } + + // Use this function to exclude certain symbols from `__imp` generation. + fn ignored(symbol_name: &[u8]) -> bool { + // These are symbols generated by LLVM's profiling instrumentation + symbol_name.starts_with(b"__llvm_profile_") + } +} + +fn record_artifact_size( + self_profiler_ref: &SelfProfilerRef, + artifact_kind: &'static str, + path: &Path, +) { + // Don't stat the file if we are not going to record its size. + if !self_profiler_ref.enabled() { + return; + } + + if let Some(artifact_name) = path.file_name() { + let file_size = std::fs::metadata(path).map(|m| m.len()).unwrap_or(0); + self_profiler_ref.artifact_size(artifact_kind, artifact_name.to_string_lossy(), file_size); + } +} diff --git a/compiler/rustc_codegen_llvm/src/base.rs b/compiler/rustc_codegen_llvm/src/base.rs new file mode 100644 index 000000000..86f92dc02 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/base.rs @@ -0,0 +1,173 @@ +//! Codegen the MIR to the LLVM IR. +//! +//! Hopefully useful general knowledge about codegen: +//! +//! * There's no way to find out the [`Ty`] type of a [`Value`]. Doing so +//! would be "trying to get the eggs out of an omelette" (credit: +//! pcwalton). You can, instead, find out its [`llvm::Type`] by calling [`val_ty`], +//! but one [`llvm::Type`] corresponds to many [`Ty`]s; for instance, `tup(int, int, +//! int)` and `rec(x=int, y=int, z=int)` will have the same [`llvm::Type`]. +//! +//! [`Ty`]: rustc_middle::ty::Ty +//! [`val_ty`]: crate::common::val_ty + +use super::ModuleLlvm; + +use crate::attributes; +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::llvm; +use crate::value::Value; + +use rustc_codegen_ssa::base::maybe_create_entry_wrapper; +use rustc_codegen_ssa::mono_item::MonoItemExt; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::{ModuleCodegen, ModuleKind}; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_middle::dep_graph; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrs; +use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::DebugInfo; +use rustc_span::symbol::Symbol; +use rustc_target::spec::SanitizerSet; + +use std::time::Instant; + +pub struct ValueIter<'ll> { + cur: Option<&'ll Value>, + step: unsafe extern "C" fn(&'ll Value) -> Option<&'ll Value>, +} + +impl<'ll> Iterator for ValueIter<'ll> { + type Item = &'ll Value; + + fn next(&mut self) -> Option<&'ll Value> { + let old = self.cur; + if let Some(old) = old { + self.cur = unsafe { (self.step)(old) }; + } + old + } +} + +pub fn iter_globals(llmod: &llvm::Module) -> ValueIter<'_> { + unsafe { ValueIter { cur: llvm::LLVMGetFirstGlobal(llmod), step: llvm::LLVMGetNextGlobal } } +} + +pub fn compile_codegen_unit(tcx: TyCtxt<'_>, cgu_name: Symbol) -> (ModuleCodegen, u64) { + let start_time = Instant::now(); + + let dep_node = tcx.codegen_unit(cgu_name).codegen_dep_node(tcx); + let (module, _) = tcx.dep_graph.with_task( + dep_node, + tcx, + cgu_name, + module_codegen, + Some(dep_graph::hash_result), + ); + let time_to_codegen = start_time.elapsed(); + + // We assume that the cost to run LLVM on a CGU is proportional to + // the time we needed for codegenning it. + let cost = time_to_codegen.as_nanos() as u64; + + fn module_codegen(tcx: TyCtxt<'_>, cgu_name: Symbol) -> ModuleCodegen { + let cgu = tcx.codegen_unit(cgu_name); + let _prof_timer = + tcx.prof.generic_activity_with_arg_recorder("codegen_module", |recorder| { + recorder.record_arg(cgu_name.to_string()); + recorder.record_arg(cgu.size_estimate().to_string()); + }); + // Instantiate monomorphizations without filling out definitions yet... + let llvm_module = ModuleLlvm::new(tcx, cgu_name.as_str()); + { + let cx = CodegenCx::new(tcx, cgu, &llvm_module); + let mono_items = cx.codegen_unit.items_in_deterministic_order(cx.tcx); + for &(mono_item, (linkage, visibility)) in &mono_items { + mono_item.predefine::>(&cx, linkage, visibility); + } + + // ... and now that we have everything pre-defined, fill out those definitions. + for &(mono_item, _) in &mono_items { + mono_item.define::>(&cx); + } + + // If this codegen unit contains the main function, also create the + // wrapper here + if let Some(entry) = maybe_create_entry_wrapper::>(&cx) { + let attrs = attributes::sanitize_attrs(&cx, SanitizerSet::empty()); + attributes::apply_to_llfn(entry, llvm::AttributePlace::Function, &attrs); + } + + // Finalize code coverage by injecting the coverage map. Note, the coverage map will + // also be added to the `llvm.compiler.used` variable, created next. + if cx.sess().instrument_coverage() { + cx.coverageinfo_finalize(); + } + + // Create the llvm.used and llvm.compiler.used variables. + if !cx.used_statics().borrow().is_empty() { + cx.create_used_variable() + } + if !cx.compiler_used_statics().borrow().is_empty() { + cx.create_compiler_used_variable() + } + + // Run replace-all-uses-with for statics that need it. This must + // happen after the llvm.used variables are created. + for &(old_g, new_g) in cx.statics_to_rauw().borrow().iter() { + unsafe { + let bitcast = llvm::LLVMConstPointerCast(new_g, cx.val_ty(old_g)); + llvm::LLVMReplaceAllUsesWith(old_g, bitcast); + llvm::LLVMDeleteGlobal(old_g); + } + } + + // Finalize debuginfo + if cx.sess().opts.debuginfo != DebugInfo::None { + cx.debuginfo_finalize(); + } + } + + ModuleCodegen { + name: cgu_name.to_string(), + module_llvm: llvm_module, + kind: ModuleKind::Regular, + } + } + + (module, cost) +} + +pub fn set_link_section(llval: &Value, attrs: &CodegenFnAttrs) { + let Some(sect) = attrs.link_section else { return }; + unsafe { + let buf = SmallCStr::new(sect.as_str()); + llvm::LLVMSetSection(llval, buf.as_ptr()); + } +} + +pub fn linkage_to_llvm(linkage: Linkage) -> llvm::Linkage { + match linkage { + Linkage::External => llvm::Linkage::ExternalLinkage, + Linkage::AvailableExternally => llvm::Linkage::AvailableExternallyLinkage, + Linkage::LinkOnceAny => llvm::Linkage::LinkOnceAnyLinkage, + Linkage::LinkOnceODR => llvm::Linkage::LinkOnceODRLinkage, + Linkage::WeakAny => llvm::Linkage::WeakAnyLinkage, + Linkage::WeakODR => llvm::Linkage::WeakODRLinkage, + Linkage::Appending => llvm::Linkage::AppendingLinkage, + Linkage::Internal => llvm::Linkage::InternalLinkage, + Linkage::Private => llvm::Linkage::PrivateLinkage, + Linkage::ExternalWeak => llvm::Linkage::ExternalWeakLinkage, + Linkage::Common => llvm::Linkage::CommonLinkage, + } +} + +pub fn visibility_to_llvm(linkage: Visibility) -> llvm::Visibility { + match linkage { + Visibility::Default => llvm::Visibility::Default, + Visibility::Hidden => llvm::Visibility::Hidden, + Visibility::Protected => llvm::Visibility::Protected, + } +} diff --git a/compiler/rustc_codegen_llvm/src/builder.rs b/compiler/rustc_codegen_llvm/src/builder.rs new file mode 100644 index 000000000..d3096c73a --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/builder.rs @@ -0,0 +1,1508 @@ +use crate::attributes; +use crate::common::Funclet; +use crate::context::CodegenCx; +use crate::llvm::{self, BasicBlock, False}; +use crate::llvm::{AtomicOrdering, AtomicRmwBinOp, SynchronizationScope}; +use crate::llvm_util; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; +use cstr::cstr; +use libc::{c_char, c_uint}; +use rustc_codegen_ssa::common::{IntPredicate, RealPredicate, TypeKind}; +use rustc_codegen_ssa::mir::operand::{OperandRef, OperandValue}; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::MemFlags; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::def_id::DefId; +use rustc_middle::ty::layout::{ + FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, TyAndLayout, +}; +use rustc_middle::ty::{self, Ty, TyCtxt}; +use rustc_span::Span; +use rustc_target::abi::{self, call::FnAbi, Align, Size, WrappingRange}; +use rustc_target::spec::{HasTargetSpec, Target}; +use std::borrow::Cow; +use std::ffi::CStr; +use std::iter; +use std::ops::Deref; +use std::ptr; +use tracing::{debug, instrument}; + +// All Builders must have an llfn associated with them +#[must_use] +pub struct Builder<'a, 'll, 'tcx> { + pub llbuilder: &'ll mut llvm::Builder<'ll>, + pub cx: &'a CodegenCx<'ll, 'tcx>, +} + +impl Drop for Builder<'_, '_, '_> { + fn drop(&mut self) { + unsafe { + llvm::LLVMDisposeBuilder(&mut *(self.llbuilder as *mut _)); + } + } +} + +// FIXME(eddyb) use a checked constructor when they become `const fn`. +const EMPTY_C_STR: &CStr = unsafe { CStr::from_bytes_with_nul_unchecked(b"\0") }; + +/// Empty string, to be used where LLVM expects an instruction name, indicating +/// that the instruction is to be left unnamed (i.e. numbered, in textual IR). +// FIXME(eddyb) pass `&CStr` directly to FFI once it's a thin pointer. +const UNNAMED: *const c_char = EMPTY_C_STR.as_ptr(); + +impl<'ll, 'tcx> BackendTypes for Builder<'_, 'll, 'tcx> { + type Value = as BackendTypes>::Value; + type Function = as BackendTypes>::Function; + type BasicBlock = as BackendTypes>::BasicBlock; + type Type = as BackendTypes>::Type; + type Funclet = as BackendTypes>::Funclet; + + type DIScope = as BackendTypes>::DIScope; + type DILocation = as BackendTypes>::DILocation; + type DIVariable = as BackendTypes>::DIVariable; +} + +impl abi::HasDataLayout for Builder<'_, '_, '_> { + fn data_layout(&self) -> &abi::TargetDataLayout { + self.cx.data_layout() + } +} + +impl<'tcx> ty::layout::HasTyCtxt<'tcx> for Builder<'_, '_, 'tcx> { + #[inline] + fn tcx(&self) -> TyCtxt<'tcx> { + self.cx.tcx + } +} + +impl<'tcx> ty::layout::HasParamEnv<'tcx> for Builder<'_, '_, 'tcx> { + fn param_env(&self) -> ty::ParamEnv<'tcx> { + self.cx.param_env() + } +} + +impl HasTargetSpec for Builder<'_, '_, '_> { + #[inline] + fn target_spec(&self) -> &Target { + self.cx.target_spec() + } +} + +impl<'tcx> LayoutOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { + type LayoutOfResult = TyAndLayout<'tcx>; + + #[inline] + fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { + self.cx.handle_layout_err(err, span, ty) + } +} + +impl<'tcx> FnAbiOfHelpers<'tcx> for Builder<'_, '_, 'tcx> { + type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; + + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + self.cx.handle_fn_abi_err(err, span, fn_abi_request) + } +} + +impl<'ll, 'tcx> Deref for Builder<'_, 'll, 'tcx> { + type Target = CodegenCx<'ll, 'tcx>; + + #[inline] + fn deref(&self) -> &Self::Target { + self.cx + } +} + +impl<'ll, 'tcx> HasCodegen<'tcx> for Builder<'_, 'll, 'tcx> { + type CodegenCx = CodegenCx<'ll, 'tcx>; +} + +macro_rules! builder_methods_for_value_instructions { + ($($name:ident($($arg:ident),*) => $llvm_capi:ident),+ $(,)?) => { + $(fn $name(&mut self, $($arg: &'ll Value),*) -> &'ll Value { + unsafe { + llvm::$llvm_capi(self.llbuilder, $($arg,)* UNNAMED) + } + })+ + } +} + +impl<'a, 'll, 'tcx> BuilderMethods<'a, 'tcx> for Builder<'a, 'll, 'tcx> { + fn build(cx: &'a CodegenCx<'ll, 'tcx>, llbb: &'ll BasicBlock) -> Self { + let bx = Builder::with_cx(cx); + unsafe { + llvm::LLVMPositionBuilderAtEnd(bx.llbuilder, llbb); + } + bx + } + + fn cx(&self) -> &CodegenCx<'ll, 'tcx> { + self.cx + } + + fn llbb(&self) -> &'ll BasicBlock { + unsafe { llvm::LLVMGetInsertBlock(self.llbuilder) } + } + + fn set_span(&mut self, _span: Span) {} + + fn append_block(cx: &'a CodegenCx<'ll, 'tcx>, llfn: &'ll Value, name: &str) -> &'ll BasicBlock { + unsafe { + let name = SmallCStr::new(name); + llvm::LLVMAppendBasicBlockInContext(cx.llcx, llfn, name.as_ptr()) + } + } + + fn append_sibling_block(&mut self, name: &str) -> &'ll BasicBlock { + Self::append_block(self.cx, self.llfn(), name) + } + + fn switch_to_block(&mut self, llbb: Self::BasicBlock) { + *self = Self::build(self.cx, llbb) + } + + fn ret_void(&mut self) { + unsafe { + llvm::LLVMBuildRetVoid(self.llbuilder); + } + } + + fn ret(&mut self, v: &'ll Value) { + unsafe { + llvm::LLVMBuildRet(self.llbuilder, v); + } + } + + fn br(&mut self, dest: &'ll BasicBlock) { + unsafe { + llvm::LLVMBuildBr(self.llbuilder, dest); + } + } + + fn cond_br( + &mut self, + cond: &'ll Value, + then_llbb: &'ll BasicBlock, + else_llbb: &'ll BasicBlock, + ) { + unsafe { + llvm::LLVMBuildCondBr(self.llbuilder, cond, then_llbb, else_llbb); + } + } + + fn switch( + &mut self, + v: &'ll Value, + else_llbb: &'ll BasicBlock, + cases: impl ExactSizeIterator, + ) { + let switch = + unsafe { llvm::LLVMBuildSwitch(self.llbuilder, v, else_llbb, cases.len() as c_uint) }; + for (on_val, dest) in cases { + let on_val = self.const_uint_big(self.val_ty(v), on_val); + unsafe { llvm::LLVMAddCase(switch, on_val, dest) } + } + } + + fn invoke( + &mut self, + llty: &'ll Type, + llfn: &'ll Value, + args: &[&'ll Value], + then: &'ll BasicBlock, + catch: &'ll BasicBlock, + funclet: Option<&Funclet<'ll>>, + ) -> &'ll Value { + debug!("invoke {:?} with args ({:?})", llfn, args); + + let args = self.check_call("invoke", llty, llfn, args); + let bundle = funclet.map(|funclet| funclet.bundle()); + let bundle = bundle.as_ref().map(|b| &*b.raw); + + unsafe { + llvm::LLVMRustBuildInvoke( + self.llbuilder, + llty, + llfn, + args.as_ptr(), + args.len() as c_uint, + then, + catch, + bundle, + UNNAMED, + ) + } + } + + fn unreachable(&mut self) { + unsafe { + llvm::LLVMBuildUnreachable(self.llbuilder); + } + } + + builder_methods_for_value_instructions! { + add(a, b) => LLVMBuildAdd, + fadd(a, b) => LLVMBuildFAdd, + sub(a, b) => LLVMBuildSub, + fsub(a, b) => LLVMBuildFSub, + mul(a, b) => LLVMBuildMul, + fmul(a, b) => LLVMBuildFMul, + udiv(a, b) => LLVMBuildUDiv, + exactudiv(a, b) => LLVMBuildExactUDiv, + sdiv(a, b) => LLVMBuildSDiv, + exactsdiv(a, b) => LLVMBuildExactSDiv, + fdiv(a, b) => LLVMBuildFDiv, + urem(a, b) => LLVMBuildURem, + srem(a, b) => LLVMBuildSRem, + frem(a, b) => LLVMBuildFRem, + shl(a, b) => LLVMBuildShl, + lshr(a, b) => LLVMBuildLShr, + ashr(a, b) => LLVMBuildAShr, + and(a, b) => LLVMBuildAnd, + or(a, b) => LLVMBuildOr, + xor(a, b) => LLVMBuildXor, + neg(x) => LLVMBuildNeg, + fneg(x) => LLVMBuildFNeg, + not(x) => LLVMBuildNot, + unchecked_sadd(x, y) => LLVMBuildNSWAdd, + unchecked_uadd(x, y) => LLVMBuildNUWAdd, + unchecked_ssub(x, y) => LLVMBuildNSWSub, + unchecked_usub(x, y) => LLVMBuildNUWSub, + unchecked_smul(x, y) => LLVMBuildNSWMul, + unchecked_umul(x, y) => LLVMBuildNUWMul, + } + + fn fadd_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFAdd(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn fsub_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFSub(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn fmul_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFMul(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn fdiv_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFDiv(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn frem_fast(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMBuildFRem(self.llbuilder, lhs, rhs, UNNAMED); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + + fn checked_binop( + &mut self, + oop: OverflowOp, + ty: Ty<'_>, + lhs: Self::Value, + rhs: Self::Value, + ) -> (Self::Value, Self::Value) { + use rustc_middle::ty::{Int, Uint}; + use rustc_middle::ty::{IntTy::*, UintTy::*}; + + let new_kind = match ty.kind() { + Int(t @ Isize) => Int(t.normalize(self.tcx.sess.target.pointer_width)), + Uint(t @ Usize) => Uint(t.normalize(self.tcx.sess.target.pointer_width)), + t @ (Uint(_) | Int(_)) => t.clone(), + _ => panic!("tried to get overflow intrinsic for op applied to non-int type"), + }; + + let name = match oop { + OverflowOp::Add => match new_kind { + Int(I8) => "llvm.sadd.with.overflow.i8", + Int(I16) => "llvm.sadd.with.overflow.i16", + Int(I32) => "llvm.sadd.with.overflow.i32", + Int(I64) => "llvm.sadd.with.overflow.i64", + Int(I128) => "llvm.sadd.with.overflow.i128", + + Uint(U8) => "llvm.uadd.with.overflow.i8", + Uint(U16) => "llvm.uadd.with.overflow.i16", + Uint(U32) => "llvm.uadd.with.overflow.i32", + Uint(U64) => "llvm.uadd.with.overflow.i64", + Uint(U128) => "llvm.uadd.with.overflow.i128", + + _ => unreachable!(), + }, + OverflowOp::Sub => match new_kind { + Int(I8) => "llvm.ssub.with.overflow.i8", + Int(I16) => "llvm.ssub.with.overflow.i16", + Int(I32) => "llvm.ssub.with.overflow.i32", + Int(I64) => "llvm.ssub.with.overflow.i64", + Int(I128) => "llvm.ssub.with.overflow.i128", + + Uint(U8) => "llvm.usub.with.overflow.i8", + Uint(U16) => "llvm.usub.with.overflow.i16", + Uint(U32) => "llvm.usub.with.overflow.i32", + Uint(U64) => "llvm.usub.with.overflow.i64", + Uint(U128) => "llvm.usub.with.overflow.i128", + + _ => unreachable!(), + }, + OverflowOp::Mul => match new_kind { + Int(I8) => "llvm.smul.with.overflow.i8", + Int(I16) => "llvm.smul.with.overflow.i16", + Int(I32) => "llvm.smul.with.overflow.i32", + Int(I64) => "llvm.smul.with.overflow.i64", + Int(I128) => "llvm.smul.with.overflow.i128", + + Uint(U8) => "llvm.umul.with.overflow.i8", + Uint(U16) => "llvm.umul.with.overflow.i16", + Uint(U32) => "llvm.umul.with.overflow.i32", + Uint(U64) => "llvm.umul.with.overflow.i64", + Uint(U128) => "llvm.umul.with.overflow.i128", + + _ => unreachable!(), + }, + }; + + let res = self.call_intrinsic(name, &[lhs, rhs]); + (self.extract_value(res, 0), self.extract_value(res, 1)) + } + + fn from_immediate(&mut self, val: Self::Value) -> Self::Value { + if self.cx().val_ty(val) == self.cx().type_i1() { + self.zext(val, self.cx().type_i8()) + } else { + val + } + } + fn to_immediate_scalar(&mut self, val: Self::Value, scalar: abi::Scalar) -> Self::Value { + if scalar.is_bool() { + return self.trunc(val, self.cx().type_i1()); + } + val + } + + fn alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { + let mut bx = Builder::with_cx(self.cx); + bx.position_at_start(unsafe { llvm::LLVMGetFirstBasicBlock(self.llfn()) }); + bx.dynamic_alloca(ty, align) + } + + fn dynamic_alloca(&mut self, ty: &'ll Type, align: Align) -> &'ll Value { + unsafe { + let alloca = llvm::LLVMBuildAlloca(self.llbuilder, ty, UNNAMED); + llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); + alloca + } + } + + fn array_alloca(&mut self, ty: &'ll Type, len: &'ll Value, align: Align) -> &'ll Value { + unsafe { + let alloca = llvm::LLVMBuildArrayAlloca(self.llbuilder, ty, len, UNNAMED); + llvm::LLVMSetAlignment(alloca, align.bytes() as c_uint); + alloca + } + } + + fn load(&mut self, ty: &'ll Type, ptr: &'ll Value, align: Align) -> &'ll Value { + unsafe { + let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); + llvm::LLVMSetAlignment(load, align.bytes() as c_uint); + load + } + } + + fn volatile_load(&mut self, ty: &'ll Type, ptr: &'ll Value) -> &'ll Value { + unsafe { + let load = llvm::LLVMBuildLoad2(self.llbuilder, ty, ptr, UNNAMED); + llvm::LLVMSetVolatile(load, llvm::True); + load + } + } + + fn atomic_load( + &mut self, + ty: &'ll Type, + ptr: &'ll Value, + order: rustc_codegen_ssa::common::AtomicOrdering, + size: Size, + ) -> &'ll Value { + unsafe { + let load = llvm::LLVMRustBuildAtomicLoad( + self.llbuilder, + ty, + ptr, + UNNAMED, + AtomicOrdering::from_generic(order), + ); + // LLVM requires the alignment of atomic loads to be at least the size of the type. + llvm::LLVMSetAlignment(load, size.bytes() as c_uint); + load + } + } + + #[instrument(level = "trace", skip(self))] + fn load_operand(&mut self, place: PlaceRef<'tcx, &'ll Value>) -> OperandRef<'tcx, &'ll Value> { + assert_eq!(place.llextra.is_some(), place.layout.is_unsized()); + + if place.layout.is_zst() { + return OperandRef::new_zst(self, place.layout); + } + + #[instrument(level = "trace", skip(bx))] + fn scalar_load_metadata<'a, 'll, 'tcx>( + bx: &mut Builder<'a, 'll, 'tcx>, + load: &'ll Value, + scalar: abi::Scalar, + layout: TyAndLayout<'tcx>, + offset: Size, + ) { + if !scalar.is_always_valid(bx) { + bx.noundef_metadata(load); + } + + match scalar.primitive() { + abi::Int(..) => { + if !scalar.is_always_valid(bx) { + bx.range_metadata(load, scalar.valid_range(bx)); + } + } + abi::Pointer => { + if !scalar.valid_range(bx).contains(0) { + bx.nonnull_metadata(load); + } + + if let Some(pointee) = layout.pointee_info_at(bx, offset) { + if let Some(_) = pointee.safe { + bx.align_metadata(load, pointee.align); + } + } + } + abi::F32 | abi::F64 => {} + } + } + + let val = if let Some(llextra) = place.llextra { + OperandValue::Ref(place.llval, Some(llextra), place.align) + } else if place.layout.is_llvm_immediate() { + let mut const_llval = None; + let llty = place.layout.llvm_type(self); + unsafe { + if let Some(global) = llvm::LLVMIsAGlobalVariable(place.llval) { + if llvm::LLVMIsGlobalConstant(global) == llvm::True { + if let Some(init) = llvm::LLVMGetInitializer(global) { + if self.val_ty(init) == llty { + const_llval = Some(init); + } + } + } + } + } + let llval = const_llval.unwrap_or_else(|| { + let load = self.load(llty, place.llval, place.align); + if let abi::Abi::Scalar(scalar) = place.layout.abi { + scalar_load_metadata(self, load, scalar, place.layout, Size::ZERO); + } + load + }); + OperandValue::Immediate(self.to_immediate(llval, place.layout)) + } else if let abi::Abi::ScalarPair(a, b) = place.layout.abi { + let b_offset = a.size(self).align_to(b.align(self).abi); + let pair_ty = place.layout.llvm_type(self); + + let mut load = |i, scalar: abi::Scalar, layout, align, offset| { + let llptr = self.struct_gep(pair_ty, place.llval, i as u64); + let llty = place.layout.scalar_pair_element_llvm_type(self, i, false); + let load = self.load(llty, llptr, align); + scalar_load_metadata(self, load, scalar, layout, offset); + self.to_immediate_scalar(load, scalar) + }; + + OperandValue::Pair( + load(0, a, place.layout, place.align, Size::ZERO), + load(1, b, place.layout, place.align.restrict_for_offset(b_offset), b_offset), + ) + } else { + OperandValue::Ref(place.llval, None, place.align) + }; + + OperandRef { val, layout: place.layout } + } + + fn write_operand_repeatedly( + mut self, + cg_elem: OperandRef<'tcx, &'ll Value>, + count: u64, + dest: PlaceRef<'tcx, &'ll Value>, + ) -> Self { + let zero = self.const_usize(0); + let count = self.const_usize(count); + let start = dest.project_index(&mut self, zero).llval; + let end = dest.project_index(&mut self, count).llval; + + let header_bb = self.append_sibling_block("repeat_loop_header"); + let body_bb = self.append_sibling_block("repeat_loop_body"); + let next_bb = self.append_sibling_block("repeat_loop_next"); + + self.br(header_bb); + + let mut header_bx = Self::build(self.cx, header_bb); + let current = header_bx.phi(self.val_ty(start), &[start], &[self.llbb()]); + + let keep_going = header_bx.icmp(IntPredicate::IntNE, current, end); + header_bx.cond_br(keep_going, body_bb, next_bb); + + let mut body_bx = Self::build(self.cx, body_bb); + let align = dest.align.restrict_for_offset(dest.layout.field(self.cx(), 0).size); + cg_elem + .val + .store(&mut body_bx, PlaceRef::new_sized_aligned(current, cg_elem.layout, align)); + + let next = body_bx.inbounds_gep( + self.backend_type(cg_elem.layout), + current, + &[self.const_usize(1)], + ); + body_bx.br(header_bb); + header_bx.add_incoming_to_phi(current, next, body_bb); + + Self::build(self.cx, next_bb) + } + + fn range_metadata(&mut self, load: &'ll Value, range: WrappingRange) { + if self.sess().target.arch == "amdgpu" { + // amdgpu/LLVM does something weird and thinks an i64 value is + // split into a v2i32, halving the bitwidth LLVM expects, + // tripping an assertion. So, for now, just disable this + // optimization. + return; + } + + unsafe { + let llty = self.cx.val_ty(load); + let v = [ + self.cx.const_uint_big(llty, range.start), + self.cx.const_uint_big(llty, range.end.wrapping_add(1)), + ]; + + llvm::LLVMSetMetadata( + load, + llvm::MD_range as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint), + ); + } + } + + fn nonnull_metadata(&mut self, load: &'ll Value) { + unsafe { + llvm::LLVMSetMetadata( + load, + llvm::MD_nonnull as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), + ); + } + } + + fn store(&mut self, val: &'ll Value, ptr: &'ll Value, align: Align) -> &'ll Value { + self.store_with_flags(val, ptr, align, MemFlags::empty()) + } + + fn store_with_flags( + &mut self, + val: &'ll Value, + ptr: &'ll Value, + align: Align, + flags: MemFlags, + ) -> &'ll Value { + debug!("Store {:?} -> {:?} ({:?})", val, ptr, flags); + let ptr = self.check_store(val, ptr); + unsafe { + let store = llvm::LLVMBuildStore(self.llbuilder, val, ptr); + let align = + if flags.contains(MemFlags::UNALIGNED) { 1 } else { align.bytes() as c_uint }; + llvm::LLVMSetAlignment(store, align); + if flags.contains(MemFlags::VOLATILE) { + llvm::LLVMSetVolatile(store, llvm::True); + } + if flags.contains(MemFlags::NONTEMPORAL) { + // According to LLVM [1] building a nontemporal store must + // *always* point to a metadata value of the integer 1. + // + // [1]: https://llvm.org/docs/LangRef.html#store-instruction + let one = self.cx.const_i32(1); + let node = llvm::LLVMMDNodeInContext(self.cx.llcx, &one, 1); + llvm::LLVMSetMetadata(store, llvm::MD_nontemporal as c_uint, node); + } + store + } + } + + fn atomic_store( + &mut self, + val: &'ll Value, + ptr: &'ll Value, + order: rustc_codegen_ssa::common::AtomicOrdering, + size: Size, + ) { + debug!("Store {:?} -> {:?}", val, ptr); + let ptr = self.check_store(val, ptr); + unsafe { + let store = llvm::LLVMRustBuildAtomicStore( + self.llbuilder, + val, + ptr, + AtomicOrdering::from_generic(order), + ); + // LLVM requires the alignment of atomic stores to be at least the size of the type. + llvm::LLVMSetAlignment(store, size.bytes() as c_uint); + } + } + + fn gep(&mut self, ty: &'ll Type, ptr: &'ll Value, indices: &[&'ll Value]) -> &'ll Value { + unsafe { + llvm::LLVMBuildGEP2( + self.llbuilder, + ty, + ptr, + indices.as_ptr(), + indices.len() as c_uint, + UNNAMED, + ) + } + } + + fn inbounds_gep( + &mut self, + ty: &'ll Type, + ptr: &'ll Value, + indices: &[&'ll Value], + ) -> &'ll Value { + unsafe { + llvm::LLVMBuildInBoundsGEP2( + self.llbuilder, + ty, + ptr, + indices.as_ptr(), + indices.len() as c_uint, + UNNAMED, + ) + } + } + + fn struct_gep(&mut self, ty: &'ll Type, ptr: &'ll Value, idx: u64) -> &'ll Value { + assert_eq!(idx as c_uint as u64, idx); + unsafe { llvm::LLVMBuildStructGEP2(self.llbuilder, ty, ptr, idx as c_uint, UNNAMED) } + } + + /* Casts */ + fn trunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildTrunc(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn sext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildSExt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fptoui_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + self.fptoint_sat(false, val, dest_ty) + } + + fn fptosi_sat(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> Option<&'ll Value> { + self.fptoint_sat(true, val, dest_ty) + } + + fn fptoui(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + // On WebAssembly the `fptoui` and `fptosi` instructions currently have + // poor codegen. The reason for this is that the corresponding wasm + // instructions, `i32.trunc_f32_s` for example, will trap when the float + // is out-of-bounds, infinity, or nan. This means that LLVM + // automatically inserts control flow around `fptoui` and `fptosi` + // because the LLVM instruction `fptoui` is defined as producing a + // poison value, not having UB on out-of-bounds values. + // + // This method, however, is only used with non-saturating casts that + // have UB on out-of-bounds values. This means that it's ok if we use + // the raw wasm instruction since out-of-bounds values can do whatever + // we like. To ensure that LLVM picks the right instruction we choose + // the raw wasm intrinsic functions which avoid LLVM inserting all the + // other control flow automatically. + if self.sess().target.is_like_wasm { + let src_ty = self.cx.val_ty(val); + if self.cx.type_kind(src_ty) != TypeKind::Vector { + let float_width = self.cx.float_width(src_ty); + let int_width = self.cx.int_width(dest_ty); + let name = match (int_width, float_width) { + (32, 32) => Some("llvm.wasm.trunc.unsigned.i32.f32"), + (32, 64) => Some("llvm.wasm.trunc.unsigned.i32.f64"), + (64, 32) => Some("llvm.wasm.trunc.unsigned.i64.f32"), + (64, 64) => Some("llvm.wasm.trunc.unsigned.i64.f64"), + _ => None, + }; + if let Some(name) = name { + return self.call_intrinsic(name, &[val]); + } + } + } + unsafe { llvm::LLVMBuildFPToUI(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fptosi(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + // see `fptoui` above for why wasm is different here + if self.sess().target.is_like_wasm { + let src_ty = self.cx.val_ty(val); + if self.cx.type_kind(src_ty) != TypeKind::Vector { + let float_width = self.cx.float_width(src_ty); + let int_width = self.cx.int_width(dest_ty); + let name = match (int_width, float_width) { + (32, 32) => Some("llvm.wasm.trunc.signed.i32.f32"), + (32, 64) => Some("llvm.wasm.trunc.signed.i32.f64"), + (64, 32) => Some("llvm.wasm.trunc.signed.i64.f32"), + (64, 64) => Some("llvm.wasm.trunc.signed.i64.f64"), + _ => None, + }; + if let Some(name) = name { + return self.call_intrinsic(name, &[val]); + } + } + } + unsafe { llvm::LLVMBuildFPToSI(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn uitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildUIToFP(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn sitofp(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildSIToFP(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fptrunc(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildFPTrunc(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn fpext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildFPExt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn ptrtoint(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildPtrToInt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn inttoptr(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildIntToPtr(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn bitcast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildBitCast(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn intcast(&mut self, val: &'ll Value, dest_ty: &'ll Type, is_signed: bool) -> &'ll Value { + unsafe { llvm::LLVMRustBuildIntCast(self.llbuilder, val, dest_ty, is_signed) } + } + + fn pointercast(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildPointerCast(self.llbuilder, val, dest_ty, UNNAMED) } + } + + /* Comparisons */ + fn icmp(&mut self, op: IntPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + let op = llvm::IntPredicate::from_generic(op); + unsafe { llvm::LLVMBuildICmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) } + } + + fn fcmp(&mut self, op: RealPredicate, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + let op = llvm::RealPredicate::from_generic(op); + unsafe { llvm::LLVMBuildFCmp(self.llbuilder, op as c_uint, lhs, rhs, UNNAMED) } + } + + /* Miscellaneous instructions */ + fn memcpy( + &mut self, + dst: &'ll Value, + dst_align: Align, + src: &'ll Value, + src_align: Align, + size: &'ll Value, + flags: MemFlags, + ) { + assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memcpy not supported"); + let size = self.intcast(size, self.type_isize(), false); + let is_volatile = flags.contains(MemFlags::VOLATILE); + let dst = self.pointercast(dst, self.type_i8p()); + let src = self.pointercast(src, self.type_i8p()); + unsafe { + llvm::LLVMRustBuildMemCpy( + self.llbuilder, + dst, + dst_align.bytes() as c_uint, + src, + src_align.bytes() as c_uint, + size, + is_volatile, + ); + } + } + + fn memmove( + &mut self, + dst: &'ll Value, + dst_align: Align, + src: &'ll Value, + src_align: Align, + size: &'ll Value, + flags: MemFlags, + ) { + assert!(!flags.contains(MemFlags::NONTEMPORAL), "non-temporal memmove not supported"); + let size = self.intcast(size, self.type_isize(), false); + let is_volatile = flags.contains(MemFlags::VOLATILE); + let dst = self.pointercast(dst, self.type_i8p()); + let src = self.pointercast(src, self.type_i8p()); + unsafe { + llvm::LLVMRustBuildMemMove( + self.llbuilder, + dst, + dst_align.bytes() as c_uint, + src, + src_align.bytes() as c_uint, + size, + is_volatile, + ); + } + } + + fn memset( + &mut self, + ptr: &'ll Value, + fill_byte: &'ll Value, + size: &'ll Value, + align: Align, + flags: MemFlags, + ) { + let is_volatile = flags.contains(MemFlags::VOLATILE); + let ptr = self.pointercast(ptr, self.type_i8p()); + unsafe { + llvm::LLVMRustBuildMemSet( + self.llbuilder, + ptr, + align.bytes() as c_uint, + fill_byte, + size, + is_volatile, + ); + } + } + + fn select( + &mut self, + cond: &'ll Value, + then_val: &'ll Value, + else_val: &'ll Value, + ) -> &'ll Value { + unsafe { llvm::LLVMBuildSelect(self.llbuilder, cond, then_val, else_val, UNNAMED) } + } + + fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } + } + + fn extract_element(&mut self, vec: &'ll Value, idx: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMBuildExtractElement(self.llbuilder, vec, idx, UNNAMED) } + } + + fn vector_splat(&mut self, num_elts: usize, elt: &'ll Value) -> &'ll Value { + unsafe { + let elt_ty = self.cx.val_ty(elt); + let undef = llvm::LLVMGetUndef(self.type_vector(elt_ty, num_elts as u64)); + let vec = self.insert_element(undef, elt, self.cx.const_i32(0)); + let vec_i32_ty = self.type_vector(self.type_i32(), num_elts as u64); + self.shuffle_vector(vec, undef, self.const_null(vec_i32_ty)) + } + } + + fn extract_value(&mut self, agg_val: &'ll Value, idx: u64) -> &'ll Value { + assert_eq!(idx as c_uint as u64, idx); + unsafe { llvm::LLVMBuildExtractValue(self.llbuilder, agg_val, idx as c_uint, UNNAMED) } + } + + fn insert_value(&mut self, agg_val: &'ll Value, elt: &'ll Value, idx: u64) -> &'ll Value { + assert_eq!(idx as c_uint as u64, idx); + unsafe { llvm::LLVMBuildInsertValue(self.llbuilder, agg_val, elt, idx as c_uint, UNNAMED) } + } + + fn set_personality_fn(&mut self, personality: &'ll Value) { + unsafe { + llvm::LLVMSetPersonalityFn(self.llfn(), personality); + } + } + + fn cleanup_landing_pad(&mut self, ty: &'ll Type, pers_fn: &'ll Value) -> &'ll Value { + let landing_pad = self.landing_pad(ty, pers_fn, 1 /* FIXME should this be 0? */); + unsafe { + llvm::LLVMSetCleanup(landing_pad, llvm::True); + } + landing_pad + } + + fn resume(&mut self, exn: &'ll Value) { + unsafe { + llvm::LLVMBuildResume(self.llbuilder, exn); + } + } + + fn cleanup_pad(&mut self, parent: Option<&'ll Value>, args: &[&'ll Value]) -> Funclet<'ll> { + let name = cstr!("cleanuppad"); + let ret = unsafe { + llvm::LLVMRustBuildCleanupPad( + self.llbuilder, + parent, + args.len() as c_uint, + args.as_ptr(), + name.as_ptr(), + ) + }; + Funclet::new(ret.expect("LLVM does not have support for cleanuppad")) + } + + fn cleanup_ret(&mut self, funclet: &Funclet<'ll>, unwind: Option<&'ll BasicBlock>) { + unsafe { + llvm::LLVMRustBuildCleanupRet(self.llbuilder, funclet.cleanuppad(), unwind) + .expect("LLVM does not have support for cleanupret"); + } + } + + fn catch_pad(&mut self, parent: &'ll Value, args: &[&'ll Value]) -> Funclet<'ll> { + let name = cstr!("catchpad"); + let ret = unsafe { + llvm::LLVMRustBuildCatchPad( + self.llbuilder, + parent, + args.len() as c_uint, + args.as_ptr(), + name.as_ptr(), + ) + }; + Funclet::new(ret.expect("LLVM does not have support for catchpad")) + } + + fn catch_switch( + &mut self, + parent: Option<&'ll Value>, + unwind: Option<&'ll BasicBlock>, + handlers: &[&'ll BasicBlock], + ) -> &'ll Value { + let name = cstr!("catchswitch"); + let ret = unsafe { + llvm::LLVMRustBuildCatchSwitch( + self.llbuilder, + parent, + unwind, + handlers.len() as c_uint, + name.as_ptr(), + ) + }; + let ret = ret.expect("LLVM does not have support for catchswitch"); + for handler in handlers { + unsafe { + llvm::LLVMRustAddHandler(ret, handler); + } + } + ret + } + + // Atomic Operations + fn atomic_cmpxchg( + &mut self, + dst: &'ll Value, + cmp: &'ll Value, + src: &'ll Value, + mut order: rustc_codegen_ssa::common::AtomicOrdering, + failure_order: rustc_codegen_ssa::common::AtomicOrdering, + weak: bool, + ) -> &'ll Value { + let weak = if weak { llvm::True } else { llvm::False }; + if llvm_util::get_version() < (13, 0, 0) { + use rustc_codegen_ssa::common::AtomicOrdering::*; + // Older llvm has the pre-C++17 restriction on + // success and failure memory ordering, + // requiring the former to be at least as strong as the latter. + // So, for llvm 12, we upgrade the success ordering to a stronger + // one if necessary. + match (order, failure_order) { + (Relaxed, Acquire) => order = Acquire, + (Release, Acquire) => order = AcquireRelease, + (_, SequentiallyConsistent) => order = SequentiallyConsistent, + _ => {} + } + } + unsafe { + llvm::LLVMRustBuildAtomicCmpXchg( + self.llbuilder, + dst, + cmp, + src, + AtomicOrdering::from_generic(order), + AtomicOrdering::from_generic(failure_order), + weak, + ) + } + } + fn atomic_rmw( + &mut self, + op: rustc_codegen_ssa::common::AtomicRmwBinOp, + dst: &'ll Value, + src: &'ll Value, + order: rustc_codegen_ssa::common::AtomicOrdering, + ) -> &'ll Value { + unsafe { + llvm::LLVMBuildAtomicRMW( + self.llbuilder, + AtomicRmwBinOp::from_generic(op), + dst, + src, + AtomicOrdering::from_generic(order), + False, + ) + } + } + + fn atomic_fence( + &mut self, + order: rustc_codegen_ssa::common::AtomicOrdering, + scope: rustc_codegen_ssa::common::SynchronizationScope, + ) { + unsafe { + llvm::LLVMRustBuildAtomicFence( + self.llbuilder, + AtomicOrdering::from_generic(order), + SynchronizationScope::from_generic(scope), + ); + } + } + + fn set_invariant_load(&mut self, load: &'ll Value) { + unsafe { + llvm::LLVMSetMetadata( + load, + llvm::MD_invariant_load as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), + ); + } + } + + fn lifetime_start(&mut self, ptr: &'ll Value, size: Size) { + self.call_lifetime_intrinsic("llvm.lifetime.start.p0i8", ptr, size); + } + + fn lifetime_end(&mut self, ptr: &'ll Value, size: Size) { + self.call_lifetime_intrinsic("llvm.lifetime.end.p0i8", ptr, size); + } + + fn instrprof_increment( + &mut self, + fn_name: &'ll Value, + hash: &'ll Value, + num_counters: &'ll Value, + index: &'ll Value, + ) { + debug!( + "instrprof_increment() with args ({:?}, {:?}, {:?}, {:?})", + fn_name, hash, num_counters, index + ); + + let llfn = unsafe { llvm::LLVMRustGetInstrProfIncrementIntrinsic(self.cx().llmod) }; + let llty = self.cx.type_func( + &[self.cx.type_i8p(), self.cx.type_i64(), self.cx.type_i32(), self.cx.type_i32()], + self.cx.type_void(), + ); + let args = &[fn_name, hash, num_counters, index]; + let args = self.check_call("call", llty, llfn, args); + + unsafe { + let _ = llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + None, + ); + } + } + + fn call( + &mut self, + llty: &'ll Type, + llfn: &'ll Value, + args: &[&'ll Value], + funclet: Option<&Funclet<'ll>>, + ) -> &'ll Value { + debug!("call {:?} with args ({:?})", llfn, args); + + let args = self.check_call("call", llty, llfn, args); + let bundle = funclet.map(|funclet| funclet.bundle()); + let bundle = bundle.as_ref().map(|b| &*b.raw); + + unsafe { + llvm::LLVMRustBuildCall( + self.llbuilder, + llty, + llfn, + args.as_ptr() as *const &llvm::Value, + args.len() as c_uint, + bundle, + ) + } + } + + fn zext(&mut self, val: &'ll Value, dest_ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildZExt(self.llbuilder, val, dest_ty, UNNAMED) } + } + + fn do_not_inline(&mut self, llret: &'ll Value) { + let noinline = llvm::AttributeKind::NoInline.create_attr(self.llcx); + attributes::apply_to_callsite(llret, llvm::AttributePlace::Function, &[noinline]); + } +} + +impl<'ll> StaticBuilderMethods for Builder<'_, 'll, '_> { + fn get_static(&mut self, def_id: DefId) -> &'ll Value { + // Forward to the `get_static` method of `CodegenCx` + self.cx().get_static(def_id) + } +} + +impl<'a, 'll, 'tcx> Builder<'a, 'll, 'tcx> { + fn with_cx(cx: &'a CodegenCx<'ll, 'tcx>) -> Self { + // Create a fresh builder from the crate context. + let llbuilder = unsafe { llvm::LLVMCreateBuilderInContext(cx.llcx) }; + Builder { llbuilder, cx } + } + + pub fn llfn(&self) -> &'ll Value { + unsafe { llvm::LLVMGetBasicBlockParent(self.llbb()) } + } + + fn position_at_start(&mut self, llbb: &'ll BasicBlock) { + unsafe { + llvm::LLVMRustPositionBuilderAtStart(self.llbuilder, llbb); + } + } + + fn align_metadata(&mut self, load: &'ll Value, align: Align) { + unsafe { + let v = [self.cx.const_u64(align.bytes())]; + + llvm::LLVMSetMetadata( + load, + llvm::MD_align as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, v.as_ptr(), v.len() as c_uint), + ); + } + } + + fn noundef_metadata(&mut self, load: &'ll Value) { + unsafe { + llvm::LLVMSetMetadata( + load, + llvm::MD_noundef as c_uint, + llvm::LLVMMDNodeInContext(self.cx.llcx, ptr::null(), 0), + ); + } + } + + pub fn minnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildMinNum(self.llbuilder, lhs, rhs) } + } + + pub fn maxnum(&mut self, lhs: &'ll Value, rhs: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildMaxNum(self.llbuilder, lhs, rhs) } + } + + pub fn insert_element( + &mut self, + vec: &'ll Value, + elt: &'ll Value, + idx: &'ll Value, + ) -> &'ll Value { + unsafe { llvm::LLVMBuildInsertElement(self.llbuilder, vec, elt, idx, UNNAMED) } + } + + pub fn shuffle_vector( + &mut self, + v1: &'ll Value, + v2: &'ll Value, + mask: &'ll Value, + ) -> &'ll Value { + unsafe { llvm::LLVMBuildShuffleVector(self.llbuilder, v1, v2, mask, UNNAMED) } + } + + pub fn vector_reduce_fadd(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src) } + } + pub fn vector_reduce_fmul(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src) } + } + pub fn vector_reduce_fadd_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMRustBuildVectorReduceFAdd(self.llbuilder, acc, src); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + pub fn vector_reduce_fmul_fast(&mut self, acc: &'ll Value, src: &'ll Value) -> &'ll Value { + unsafe { + let instr = llvm::LLVMRustBuildVectorReduceFMul(self.llbuilder, acc, src); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + pub fn vector_reduce_add(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceAdd(self.llbuilder, src) } + } + pub fn vector_reduce_mul(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceMul(self.llbuilder, src) } + } + pub fn vector_reduce_and(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceAnd(self.llbuilder, src) } + } + pub fn vector_reduce_or(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceOr(self.llbuilder, src) } + } + pub fn vector_reduce_xor(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceXor(self.llbuilder, src) } + } + pub fn vector_reduce_fmin(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { + llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ false) + } + } + pub fn vector_reduce_fmax(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { + llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ false) + } + } + pub fn vector_reduce_fmin_fast(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { + let instr = + llvm::LLVMRustBuildVectorReduceFMin(self.llbuilder, src, /*NoNaNs:*/ true); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + pub fn vector_reduce_fmax_fast(&mut self, src: &'ll Value) -> &'ll Value { + unsafe { + let instr = + llvm::LLVMRustBuildVectorReduceFMax(self.llbuilder, src, /*NoNaNs:*/ true); + llvm::LLVMRustSetFastMath(instr); + instr + } + } + pub fn vector_reduce_min(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceMin(self.llbuilder, src, is_signed) } + } + pub fn vector_reduce_max(&mut self, src: &'ll Value, is_signed: bool) -> &'ll Value { + unsafe { llvm::LLVMRustBuildVectorReduceMax(self.llbuilder, src, is_signed) } + } + + pub fn add_clause(&mut self, landing_pad: &'ll Value, clause: &'ll Value) { + unsafe { + llvm::LLVMAddClause(landing_pad, clause); + } + } + + pub fn catch_ret(&mut self, funclet: &Funclet<'ll>, unwind: &'ll BasicBlock) -> &'ll Value { + let ret = + unsafe { llvm::LLVMRustBuildCatchRet(self.llbuilder, funclet.cleanuppad(), unwind) }; + ret.expect("LLVM does not have support for catchret") + } + + fn check_store(&mut self, val: &'ll Value, ptr: &'ll Value) -> &'ll Value { + let dest_ptr_ty = self.cx.val_ty(ptr); + let stored_ty = self.cx.val_ty(val); + let stored_ptr_ty = self.cx.type_ptr_to(stored_ty); + + assert_eq!(self.cx.type_kind(dest_ptr_ty), TypeKind::Pointer); + + if dest_ptr_ty == stored_ptr_ty { + ptr + } else { + debug!( + "type mismatch in store. \ + Expected {:?}, got {:?}; inserting bitcast", + dest_ptr_ty, stored_ptr_ty + ); + self.bitcast(ptr, stored_ptr_ty) + } + } + + fn check_call<'b>( + &mut self, + typ: &str, + fn_ty: &'ll Type, + llfn: &'ll Value, + args: &'b [&'ll Value], + ) -> Cow<'b, [&'ll Value]> { + assert!( + self.cx.type_kind(fn_ty) == TypeKind::Function, + "builder::{} not passed a function, but {:?}", + typ, + fn_ty + ); + + let param_tys = self.cx.func_params_types(fn_ty); + + let all_args_match = iter::zip(¶m_tys, args.iter().map(|&v| self.val_ty(v))) + .all(|(expected_ty, actual_ty)| *expected_ty == actual_ty); + + if all_args_match { + return Cow::Borrowed(args); + } + + let casted_args: Vec<_> = iter::zip(param_tys, args) + .enumerate() + .map(|(i, (expected_ty, &actual_val))| { + let actual_ty = self.val_ty(actual_val); + if expected_ty != actual_ty { + debug!( + "type mismatch in function call of {:?}. \ + Expected {:?} for param {}, got {:?}; injecting bitcast", + llfn, expected_ty, i, actual_ty + ); + self.bitcast(actual_val, expected_ty) + } else { + actual_val + } + }) + .collect(); + + Cow::Owned(casted_args) + } + + pub fn va_arg(&mut self, list: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMBuildVAArg(self.llbuilder, list, ty, UNNAMED) } + } + + pub(crate) fn call_intrinsic(&mut self, intrinsic: &str, args: &[&'ll Value]) -> &'ll Value { + let (ty, f) = self.cx.get_intrinsic(intrinsic); + self.call(ty, f, args, None) + } + + fn call_lifetime_intrinsic(&mut self, intrinsic: &str, ptr: &'ll Value, size: Size) { + let size = size.bytes(); + if size == 0 { + return; + } + + if !self.cx().sess().emit_lifetime_markers() { + return; + } + + let ptr = self.pointercast(ptr, self.cx.type_i8p()); + self.call_intrinsic(intrinsic, &[self.cx.const_u64(size), ptr]); + } + + pub(crate) fn phi( + &mut self, + ty: &'ll Type, + vals: &[&'ll Value], + bbs: &[&'ll BasicBlock], + ) -> &'ll Value { + assert_eq!(vals.len(), bbs.len()); + let phi = unsafe { llvm::LLVMBuildPhi(self.llbuilder, ty, UNNAMED) }; + unsafe { + llvm::LLVMAddIncoming(phi, vals.as_ptr(), bbs.as_ptr(), vals.len() as c_uint); + phi + } + } + + fn add_incoming_to_phi(&mut self, phi: &'ll Value, val: &'ll Value, bb: &'ll BasicBlock) { + unsafe { + llvm::LLVMAddIncoming(phi, &val, &bb, 1 as c_uint); + } + } + + fn fptoint_sat_broken_in_llvm(&self) -> bool { + match self.tcx.sess.target.arch.as_ref() { + // FIXME - https://bugs.llvm.org/show_bug.cgi?id=50083 + "riscv64" => llvm_util::get_version() < (13, 0, 0), + _ => false, + } + } + + fn fptoint_sat( + &mut self, + signed: bool, + val: &'ll Value, + dest_ty: &'ll Type, + ) -> Option<&'ll Value> { + if !self.fptoint_sat_broken_in_llvm() { + let src_ty = self.cx.val_ty(val); + let (float_ty, int_ty, vector_length) = if self.cx.type_kind(src_ty) == TypeKind::Vector + { + assert_eq!(self.cx.vector_length(src_ty), self.cx.vector_length(dest_ty)); + ( + self.cx.element_type(src_ty), + self.cx.element_type(dest_ty), + Some(self.cx.vector_length(src_ty)), + ) + } else { + (src_ty, dest_ty, None) + }; + let float_width = self.cx.float_width(float_ty); + let int_width = self.cx.int_width(int_ty); + + let instr = if signed { "fptosi" } else { "fptoui" }; + let name = if let Some(vector_length) = vector_length { + format!( + "llvm.{}.sat.v{}i{}.v{}f{}", + instr, vector_length, int_width, vector_length, float_width + ) + } else { + format!("llvm.{}.sat.i{}.f{}", instr, int_width, float_width) + }; + let f = + self.declare_cfn(&name, llvm::UnnamedAddr::No, self.type_func(&[src_ty], dest_ty)); + Some(self.call(self.type_func(&[src_ty], dest_ty), f, &[val], None)) + } else { + None + } + } + + pub(crate) fn landing_pad( + &mut self, + ty: &'ll Type, + pers_fn: &'ll Value, + num_clauses: usize, + ) -> &'ll Value { + // Use LLVMSetPersonalityFn to set the personality. It supports arbitrary Consts while, + // LLVMBuildLandingPad requires the argument to be a Function (as of LLVM 12). The + // personality lives on the parent function anyway. + self.set_personality_fn(pers_fn); + unsafe { + llvm::LLVMBuildLandingPad(self.llbuilder, ty, None, num_clauses as c_uint, UNNAMED) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/callee.rs b/compiler/rustc_codegen_llvm/src/callee.rs new file mode 100644 index 000000000..72155d874 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/callee.rs @@ -0,0 +1,194 @@ +//! Handles codegen of callees as well as other call-related +//! things. Callees are a superset of normal rust values and sometimes +//! have different representations. In particular, top-level fn items +//! and methods are represented as just a fn ptr and not a full +//! closure. + +use crate::abi::FnAbiLlvmExt; +use crate::attributes; +use crate::context::CodegenCx; +use crate::llvm; +use crate::value::Value; +use rustc_codegen_ssa::traits::*; +use tracing::debug; + +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt}; +use rustc_middle::ty::{self, Instance, TypeVisitable}; + +/// Codegens a reference to a fn/method item, monomorphizing and +/// inlining as it goes. +/// +/// # Parameters +/// +/// - `cx`: the crate context +/// - `instance`: the instance to be instantiated +pub fn get_fn<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, instance: Instance<'tcx>) -> &'ll Value { + let tcx = cx.tcx(); + + debug!("get_fn(instance={:?})", instance); + + assert!(!instance.substs.needs_infer()); + assert!(!instance.substs.has_escaping_bound_vars()); + + if let Some(&llfn) = cx.instances.borrow().get(&instance) { + return llfn; + } + + let sym = tcx.symbol_name(instance).name; + debug!( + "get_fn({:?}: {:?}) => {}", + instance, + instance.ty(cx.tcx(), ty::ParamEnv::reveal_all()), + sym + ); + + let fn_abi = cx.fn_abi_of_instance(instance, ty::List::empty()); + + let llfn = if let Some(llfn) = cx.get_declared_value(sym) { + // Create a fn pointer with the new signature. + let llptrty = fn_abi.ptr_to_llvm_type(cx); + + // This is subtle and surprising, but sometimes we have to bitcast + // the resulting fn pointer. The reason has to do with external + // functions. If you have two crates that both bind the same C + // library, they may not use precisely the same types: for + // example, they will probably each declare their own structs, + // which are distinct types from LLVM's point of view (nominal + // types). + // + // Now, if those two crates are linked into an application, and + // they contain inlined code, you can wind up with a situation + // where both of those functions wind up being loaded into this + // application simultaneously. In that case, the same function + // (from LLVM's point of view) requires two types. But of course + // LLVM won't allow one function to have two types. + // + // What we currently do, therefore, is declare the function with + // one of the two types (whichever happens to come first) and then + // bitcast as needed when the function is referenced to make sure + // it has the type we expect. + // + // This can occur on either a crate-local or crate-external + // reference. It also occurs when testing libcore and in some + // other weird situations. Annoying. + if cx.val_ty(llfn) != llptrty { + debug!("get_fn: casting {:?} to {:?}", llfn, llptrty); + cx.const_ptrcast(llfn, llptrty) + } else { + debug!("get_fn: not casting pointer!"); + llfn + } + } else { + let llfn = cx.declare_fn(sym, fn_abi); + debug!("get_fn: not casting pointer!"); + + attributes::from_fn_attrs(cx, llfn, instance); + + let instance_def_id = instance.def_id(); + + // Apply an appropriate linkage/visibility value to our item that we + // just declared. + // + // This is sort of subtle. Inside our codegen unit we started off + // compilation by predefining all our own `MonoItem` instances. That + // is, everything we're codegenning ourselves is already defined. That + // means that anything we're actually codegenning in this codegen unit + // will have hit the above branch in `get_declared_value`. As a result, + // we're guaranteed here that we're declaring a symbol that won't get + // defined, or in other words we're referencing a value from another + // codegen unit or even another crate. + // + // So because this is a foreign value we blanket apply an external + // linkage directive because it's coming from a different object file. + // The visibility here is where it gets tricky. This symbol could be + // referencing some foreign crate or foreign library (an `extern` + // block) in which case we want to leave the default visibility. We may + // also, though, have multiple codegen units. It could be a + // monomorphization, in which case its expected visibility depends on + // whether we are sharing generics or not. The important thing here is + // that the visibility we apply to the declaration is the same one that + // has been applied to the definition (wherever that definition may be). + unsafe { + llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::ExternalLinkage); + + let is_generic = instance.substs.non_erasable_generics().next().is_some(); + + if is_generic { + // This is a monomorphization. Its expected visibility depends + // on whether we are in share-generics mode. + + if cx.tcx.sess.opts.share_generics() { + // We are in share_generics mode. + + if let Some(instance_def_id) = instance_def_id.as_local() { + // This is a definition from the current crate. If the + // definition is unreachable for downstream crates or + // the current crate does not re-export generics, the + // definition of the instance will have been declared + // as `hidden`. + if cx.tcx.is_unreachable_local_definition(instance_def_id) + || !cx.tcx.local_crate_exports_generics() + { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } else { + // This is a monomorphization of a generic function + // defined in an upstream crate. + if instance.upstream_monomorphization(tcx).is_some() { + // This is instantiated in another crate. It cannot + // be `hidden`. + } else { + // This is a local instantiation of an upstream definition. + // If the current crate does not re-export it + // (because it is a C library or an executable), it + // will have been declared `hidden`. + if !cx.tcx.local_crate_exports_generics() { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } + } + } else { + // When not sharing generics, all instances are in the same + // crate and have hidden visibility + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } else { + // This is a non-generic function + if cx.tcx.is_codegened_item(instance_def_id) { + // This is a function that is instantiated in the local crate + + if instance_def_id.is_local() { + // This is function that is defined in the local crate. + // If it is not reachable, it is hidden. + if !cx.tcx.is_reachable_non_generic(instance_def_id) { + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } else { + // This is a function from an upstream crate that has + // been instantiated here. These are always hidden. + llvm::LLVMRustSetVisibility(llfn, llvm::Visibility::Hidden); + } + } + } + + // MinGW: For backward compatibility we rely on the linker to decide whether it + // should use dllimport for functions. + if cx.use_dll_storage_attrs + && tcx.is_dllimport_foreign_item(instance_def_id) + && !matches!(tcx.sess.target.env.as_ref(), "gnu" | "uclibc") + { + llvm::LLVMSetDLLStorageClass(llfn, llvm::DLLStorageClass::DllImport); + } + + if cx.should_assume_dso_local(llfn, true) { + llvm::LLVMRustSetDSOLocal(llfn, true); + } + } + + llfn + }; + + cx.instances.borrow_mut().insert(instance, llfn); + + llfn +} diff --git a/compiler/rustc_codegen_llvm/src/common.rs b/compiler/rustc_codegen_llvm/src/common.rs new file mode 100644 index 000000000..fb4da9a5f --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/common.rs @@ -0,0 +1,359 @@ +//! Code that is useful in various codegen modules. + +use crate::consts::{self, const_alloc_to_llvm}; +pub use crate::context::CodegenCx; +use crate::llvm::{self, BasicBlock, Bool, ConstantInt, False, OperandBundleDef, True}; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; + +use rustc_ast::Mutability; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::traits::*; +use rustc_middle::bug; +use rustc_middle::mir::interpret::{ConstAllocation, GlobalAlloc, Scalar}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_target::abi::{self, AddressSpace, HasDataLayout, Pointer, Size}; + +use libc::{c_char, c_uint}; +use tracing::debug; + +/* +* A note on nomenclature of linking: "extern", "foreign", and "upcall". +* +* An "extern" is an LLVM symbol we wind up emitting an undefined external +* reference to. This means "we don't have the thing in this compilation unit, +* please make sure you link it in at runtime". This could be a reference to +* C code found in a C library, or rust code found in a rust crate. +* +* Most "externs" are implicitly declared (automatically) as a result of a +* user declaring an extern _module_ dependency; this causes the rust driver +* to locate an extern crate, scan its compilation metadata, and emit extern +* declarations for any symbols used by the declaring crate. +* +* A "foreign" is an extern that references C (or other non-rust ABI) code. +* There is no metadata to scan for extern references so in these cases either +* a header-digester like bindgen, or manual function prototypes, have to +* serve as declarators. So these are usually given explicitly as prototype +* declarations, in rust code, with ABI attributes on them noting which ABI to +* link via. +* +* An "upcall" is a foreign call generated by the compiler (not corresponding +* to any user-written call in the code) into the runtime library, to perform +* some helper task such as bringing a task to life, allocating memory, etc. +* +*/ + +/// A structure representing an active landing pad for the duration of a basic +/// block. +/// +/// Each `Block` may contain an instance of this, indicating whether the block +/// is part of a landing pad or not. This is used to make decision about whether +/// to emit `invoke` instructions (e.g., in a landing pad we don't continue to +/// use `invoke`) and also about various function call metadata. +/// +/// For GNU exceptions (`landingpad` + `resume` instructions) this structure is +/// just a bunch of `None` instances (not too interesting), but for MSVC +/// exceptions (`cleanuppad` + `cleanupret` instructions) this contains data. +/// When inside of a landing pad, each function call in LLVM IR needs to be +/// annotated with which landing pad it's a part of. This is accomplished via +/// the `OperandBundleDef` value created for MSVC landing pads. +pub struct Funclet<'ll> { + cleanuppad: &'ll Value, + operand: OperandBundleDef<'ll>, +} + +impl<'ll> Funclet<'ll> { + pub fn new(cleanuppad: &'ll Value) -> Self { + Funclet { cleanuppad, operand: OperandBundleDef::new("funclet", &[cleanuppad]) } + } + + pub fn cleanuppad(&self) -> &'ll Value { + self.cleanuppad + } + + pub fn bundle(&self) -> &OperandBundleDef<'ll> { + &self.operand + } +} + +impl<'ll> BackendTypes for CodegenCx<'ll, '_> { + type Value = &'ll Value; + // FIXME(eddyb) replace this with a `Function` "subclass" of `Value`. + type Function = &'ll Value; + + type BasicBlock = &'ll BasicBlock; + type Type = &'ll Type; + type Funclet = Funclet<'ll>; + + type DIScope = &'ll llvm::debuginfo::DIScope; + type DILocation = &'ll llvm::debuginfo::DILocation; + type DIVariable = &'ll llvm::debuginfo::DIVariable; +} + +impl<'ll> CodegenCx<'ll, '_> { + pub fn const_array(&self, ty: &'ll Type, elts: &[&'ll Value]) -> &'ll Value { + unsafe { llvm::LLVMConstArray(ty, elts.as_ptr(), elts.len() as c_uint) } + } + + pub fn const_vector(&self, elts: &[&'ll Value]) -> &'ll Value { + unsafe { llvm::LLVMConstVector(elts.as_ptr(), elts.len() as c_uint) } + } + + pub fn const_bytes(&self, bytes: &[u8]) -> &'ll Value { + bytes_in_context(self.llcx, bytes) + } + + pub fn const_get_elt(&self, v: &'ll Value, idx: u64) -> &'ll Value { + unsafe { + assert_eq!(idx as c_uint as u64, idx); + let r = llvm::LLVMGetAggregateElement(v, idx as c_uint).unwrap(); + + debug!("const_get_elt(v={:?}, idx={}, r={:?})", v, idx, r); + + r + } + } +} + +impl<'ll, 'tcx> ConstMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn const_null(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMConstNull(t) } + } + + fn const_undef(&self, t: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMGetUndef(t) } + } + + fn const_int(&self, t: &'ll Type, i: i64) -> &'ll Value { + unsafe { llvm::LLVMConstInt(t, i as u64, True) } + } + + fn const_uint(&self, t: &'ll Type, i: u64) -> &'ll Value { + unsafe { llvm::LLVMConstInt(t, i, False) } + } + + fn const_uint_big(&self, t: &'ll Type, u: u128) -> &'ll Value { + unsafe { + let words = [u as u64, (u >> 64) as u64]; + llvm::LLVMConstIntOfArbitraryPrecision(t, 2, words.as_ptr()) + } + } + + fn const_bool(&self, val: bool) -> &'ll Value { + self.const_uint(self.type_i1(), val as u64) + } + + fn const_i16(&self, i: i16) -> &'ll Value { + self.const_int(self.type_i16(), i as i64) + } + + fn const_i32(&self, i: i32) -> &'ll Value { + self.const_int(self.type_i32(), i as i64) + } + + fn const_u32(&self, i: u32) -> &'ll Value { + self.const_uint(self.type_i32(), i as u64) + } + + fn const_u64(&self, i: u64) -> &'ll Value { + self.const_uint(self.type_i64(), i) + } + + fn const_usize(&self, i: u64) -> &'ll Value { + let bit_size = self.data_layout().pointer_size.bits(); + if bit_size < 64 { + // make sure it doesn't overflow + assert!(i < (1 << bit_size)); + } + + self.const_uint(self.isize_ty, i) + } + + fn const_u8(&self, i: u8) -> &'ll Value { + self.const_uint(self.type_i8(), i as u64) + } + + fn const_real(&self, t: &'ll Type, val: f64) -> &'ll Value { + unsafe { llvm::LLVMConstReal(t, val) } + } + + fn const_str(&self, s: &str) -> (&'ll Value, &'ll Value) { + let str_global = *self + .const_str_cache + .borrow_mut() + .raw_entry_mut() + .from_key(s) + .or_insert_with(|| { + let sc = self.const_bytes(s.as_bytes()); + let sym = self.generate_local_symbol_name("str"); + let g = self.define_global(&sym, self.val_ty(sc)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", sym); + }); + unsafe { + llvm::LLVMSetInitializer(g, sc); + llvm::LLVMSetGlobalConstant(g, True); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::InternalLinkage); + } + (s.to_owned(), g) + }) + .1; + let len = s.len(); + let cs = consts::ptrcast( + str_global, + self.type_ptr_to(self.layout_of(self.tcx.types.str_).llvm_type(self)), + ); + (cs, self.const_usize(len as u64)) + } + + fn const_struct(&self, elts: &[&'ll Value], packed: bool) -> &'ll Value { + struct_in_context(self.llcx, elts, packed) + } + + fn const_to_opt_uint(&self, v: &'ll Value) -> Option { + try_as_const_integral(v).map(|v| unsafe { llvm::LLVMConstIntGetZExtValue(v) }) + } + + fn const_to_opt_u128(&self, v: &'ll Value, sign_ext: bool) -> Option { + try_as_const_integral(v).and_then(|v| unsafe { + let (mut lo, mut hi) = (0u64, 0u64); + let success = llvm::LLVMRustConstInt128Get(v, sign_ext, &mut hi, &mut lo); + success.then_some(hi_lo_to_u128(lo, hi)) + }) + } + + fn zst_to_backend(&self, _llty: &'ll Type) -> &'ll Value { + self.const_undef(self.type_ix(0)) + } + + fn scalar_to_backend(&self, cv: Scalar, layout: abi::Scalar, llty: &'ll Type) -> &'ll Value { + let bitsize = if layout.is_bool() { 1 } else { layout.size(self).bits() }; + match cv { + Scalar::Int(int) => { + let data = int.assert_bits(layout.size(self)); + let llval = self.const_uint_big(self.type_ix(bitsize), data); + if layout.primitive() == Pointer { + unsafe { llvm::LLVMConstIntToPtr(llval, llty) } + } else { + self.const_bitcast(llval, llty) + } + } + Scalar::Ptr(ptr, _size) => { + let (alloc_id, offset) = ptr.into_parts(); + let (base_addr, base_addr_space) = match self.tcx.global_alloc(alloc_id) { + GlobalAlloc::Memory(alloc) => { + let init = const_alloc_to_llvm(self, alloc); + let alloc = alloc.inner(); + let value = match alloc.mutability { + Mutability::Mut => self.static_addr_of_mut(init, alloc.align, None), + _ => self.static_addr_of(init, alloc.align, None), + }; + if !self.sess().fewer_names() { + llvm::set_value_name(value, format!("{:?}", alloc_id).as_bytes()); + } + (value, AddressSpace::DATA) + } + GlobalAlloc::Function(fn_instance) => ( + self.get_fn_addr(fn_instance.polymorphize(self.tcx)), + self.data_layout().instruction_address_space, + ), + GlobalAlloc::VTable(ty, trait_ref) => { + let alloc = self + .tcx + .global_alloc(self.tcx.vtable_allocation((ty, trait_ref))) + .unwrap_memory(); + let init = const_alloc_to_llvm(self, alloc); + let value = self.static_addr_of(init, alloc.inner().align, None); + (value, AddressSpace::DATA) + } + GlobalAlloc::Static(def_id) => { + assert!(self.tcx.is_static(def_id)); + assert!(!self.tcx.is_thread_local_static(def_id)); + (self.get_static(def_id), AddressSpace::DATA) + } + }; + let llval = unsafe { + llvm::LLVMRustConstInBoundsGEP2( + self.type_i8(), + self.const_bitcast(base_addr, self.type_i8p_ext(base_addr_space)), + &self.const_usize(offset.bytes()), + 1, + ) + }; + if layout.primitive() != Pointer { + unsafe { llvm::LLVMConstPtrToInt(llval, llty) } + } else { + self.const_bitcast(llval, llty) + } + } + } + } + + fn const_data_from_alloc(&self, alloc: ConstAllocation<'tcx>) -> Self::Value { + const_alloc_to_llvm(self, alloc) + } + + fn from_const_alloc( + &self, + layout: TyAndLayout<'tcx>, + alloc: ConstAllocation<'tcx>, + offset: Size, + ) -> PlaceRef<'tcx, &'ll Value> { + let alloc_align = alloc.inner().align; + assert_eq!(alloc_align, layout.align.abi); + let llty = self.type_ptr_to(layout.llvm_type(self)); + let llval = if layout.size == Size::ZERO { + let llval = self.const_usize(alloc_align.bytes()); + unsafe { llvm::LLVMConstIntToPtr(llval, llty) } + } else { + let init = const_alloc_to_llvm(self, alloc); + let base_addr = self.static_addr_of(init, alloc_align, None); + + let llval = unsafe { + llvm::LLVMRustConstInBoundsGEP2( + self.type_i8(), + self.const_bitcast(base_addr, self.type_i8p()), + &self.const_usize(offset.bytes()), + 1, + ) + }; + self.const_bitcast(llval, llty) + }; + PlaceRef::new_sized(llval, layout) + } + + fn const_ptrcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + consts::ptrcast(val, ty) + } +} + +/// Get the [LLVM type][Type] of a [`Value`]. +pub fn val_ty(v: &Value) -> &Type { + unsafe { llvm::LLVMTypeOf(v) } +} + +pub fn bytes_in_context<'ll>(llcx: &'ll llvm::Context, bytes: &[u8]) -> &'ll Value { + unsafe { + let ptr = bytes.as_ptr() as *const c_char; + llvm::LLVMConstStringInContext(llcx, ptr, bytes.len() as c_uint, True) + } +} + +pub fn struct_in_context<'ll>( + llcx: &'ll llvm::Context, + elts: &[&'ll Value], + packed: bool, +) -> &'ll Value { + unsafe { + llvm::LLVMConstStructInContext(llcx, elts.as_ptr(), elts.len() as c_uint, packed as Bool) + } +} + +#[inline] +fn hi_lo_to_u128(lo: u64, hi: u64) -> u128 { + ((hi as u128) << 64) | (lo as u128) +} + +fn try_as_const_integral(v: &Value) -> Option<&ConstantInt> { + unsafe { llvm::LLVMIsAConstantInt(v) } +} diff --git a/compiler/rustc_codegen_llvm/src/consts.rs b/compiler/rustc_codegen_llvm/src/consts.rs new file mode 100644 index 000000000..18467e370 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/consts.rs @@ -0,0 +1,577 @@ +use crate::base; +use crate::common::CodegenCx; +use crate::debuginfo; +use crate::llvm::{self, True}; +use crate::llvm_util; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; +use cstr::cstr; +use libc::c_uint; +use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::DefId; +use rustc_middle::middle::codegen_fn_attrs::{CodegenFnAttrFlags, CodegenFnAttrs}; +use rustc_middle::mir::interpret::{ + read_target_uint, Allocation, ConstAllocation, ErrorHandled, GlobalAlloc, InitChunk, Pointer, + Scalar as InterpScalar, +}; +use rustc_middle::mir::mono::MonoItem; +use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::{self, Instance, Ty}; +use rustc_middle::{bug, span_bug}; +use rustc_target::abi::{ + AddressSpace, Align, HasDataLayout, Primitive, Scalar, Size, WrappingRange, +}; +use std::ops::Range; +use tracing::debug; + +pub fn const_alloc_to_llvm<'ll>(cx: &CodegenCx<'ll, '_>, alloc: ConstAllocation<'_>) -> &'ll Value { + let alloc = alloc.inner(); + let mut llvals = Vec::with_capacity(alloc.relocations().len() + 1); + let dl = cx.data_layout(); + let pointer_size = dl.pointer_size.bytes() as usize; + + // Note: this function may call `inspect_with_uninit_and_ptr_outside_interpreter`, + // so `range` must be within the bounds of `alloc` and not contain or overlap a relocation. + fn append_chunks_of_init_and_uninit_bytes<'ll, 'a, 'b>( + llvals: &mut Vec<&'ll Value>, + cx: &'a CodegenCx<'ll, 'b>, + alloc: &'a Allocation, + range: Range, + ) { + let chunks = alloc + .init_mask() + .range_as_init_chunks(Size::from_bytes(range.start), Size::from_bytes(range.end)); + + let chunk_to_llval = move |chunk| match chunk { + InitChunk::Init(range) => { + let range = (range.start.bytes() as usize)..(range.end.bytes() as usize); + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + cx.const_bytes(bytes) + } + InitChunk::Uninit(range) => { + let len = range.end.bytes() - range.start.bytes(); + cx.const_undef(cx.type_array(cx.type_i8(), len)) + } + }; + + // Generating partially-uninit consts is limited to small numbers of chunks, + // to avoid the cost of generating large complex const expressions. + // For example, `[(u32, u8); 1024 * 1024]` contains uninit padding in each element, + // and would result in `{ [5 x i8] zeroinitializer, [3 x i8] undef, ...repeat 1M times... }`. + let max = if llvm_util::get_version() < (14, 0, 0) { + // Generating partially-uninit consts inhibits optimizations in LLVM < 14. + // See https://github.com/rust-lang/rust/issues/84565. + 1 + } else { + cx.sess().opts.unstable_opts.uninit_const_chunk_threshold + }; + let allow_uninit_chunks = chunks.clone().take(max.saturating_add(1)).count() <= max; + + if allow_uninit_chunks { + llvals.extend(chunks.map(chunk_to_llval)); + } else { + // If this allocation contains any uninit bytes, codegen as if it was initialized + // (using some arbitrary value for uninit bytes). + let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(range); + llvals.push(cx.const_bytes(bytes)); + } + } + + let mut next_offset = 0; + for &(offset, alloc_id) in alloc.relocations().iter() { + let offset = offset.bytes(); + assert_eq!(offset as usize as u64, offset); + let offset = offset as usize; + if offset > next_offset { + // This `inspect` is okay since we have checked that it is not within a relocation, it + // is within the bounds of the allocation, and it doesn't affect interpreter execution + // (we inspect the result after interpreter execution). + append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, next_offset..offset); + } + let ptr_offset = read_target_uint( + dl.endian, + // This `inspect` is okay since it is within the bounds of the allocation, it doesn't + // affect interpreter execution (we inspect the result after interpreter execution), + // and we properly interpret the relocation as a relocation pointer offset. + alloc.inspect_with_uninit_and_ptr_outside_interpreter(offset..(offset + pointer_size)), + ) + .expect("const_alloc_to_llvm: could not read relocation pointer") + as u64; + + let address_space = match cx.tcx.global_alloc(alloc_id) { + GlobalAlloc::Function(..) => cx.data_layout().instruction_address_space, + GlobalAlloc::Static(..) | GlobalAlloc::Memory(..) | GlobalAlloc::VTable(..) => { + AddressSpace::DATA + } + }; + + llvals.push(cx.scalar_to_backend( + InterpScalar::from_pointer( + Pointer::new(alloc_id, Size::from_bytes(ptr_offset)), + &cx.tcx, + ), + Scalar::Initialized { + value: Primitive::Pointer, + valid_range: WrappingRange::full(dl.pointer_size), + }, + cx.type_i8p_ext(address_space), + )); + next_offset = offset + pointer_size; + } + if alloc.len() >= next_offset { + let range = next_offset..alloc.len(); + // This `inspect` is okay since we have check that it is after all relocations, it is + // within the bounds of the allocation, and it doesn't affect interpreter execution (we + // inspect the result after interpreter execution). + append_chunks_of_init_and_uninit_bytes(&mut llvals, cx, alloc, range); + } + + cx.const_struct(&llvals, true) +} + +pub fn codegen_static_initializer<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + def_id: DefId, +) -> Result<(&'ll Value, ConstAllocation<'tcx>), ErrorHandled> { + let alloc = cx.tcx.eval_static_initializer(def_id)?; + Ok((const_alloc_to_llvm(cx, alloc), alloc)) +} + +fn set_global_alignment<'ll>(cx: &CodegenCx<'ll, '_>, gv: &'ll Value, mut align: Align) { + // The target may require greater alignment for globals than the type does. + // Note: GCC and Clang also allow `__attribute__((aligned))` on variables, + // which can force it to be smaller. Rust doesn't support this yet. + if let Some(min) = cx.sess().target.min_global_align { + match Align::from_bits(min) { + Ok(min) => align = align.max(min), + Err(err) => { + cx.sess().err(&format!("invalid minimum global alignment: {}", err)); + } + } + } + unsafe { + llvm::LLVMSetAlignment(gv, align.bytes() as u32); + } +} + +fn check_and_apply_linkage<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + attrs: &CodegenFnAttrs, + ty: Ty<'tcx>, + sym: &str, + span_def_id: DefId, +) -> &'ll Value { + let llty = cx.layout_of(ty).llvm_type(cx); + if let Some(linkage) = attrs.linkage { + debug!("get_static: sym={} linkage={:?}", sym, linkage); + + // If this is a static with a linkage specified, then we need to handle + // it a little specially. The typesystem prevents things like &T and + // extern "C" fn() from being non-null, so we can't just declare a + // static and call it a day. Some linkages (like weak) will make it such + // that the static actually has a null value. + let llty2 = if let ty::RawPtr(ref mt) = ty.kind() { + cx.layout_of(mt.ty).llvm_type(cx) + } else { + cx.sess().span_fatal( + cx.tcx.def_span(span_def_id), + "must have type `*const T` or `*mut T` due to `#[linkage]` attribute", + ) + }; + unsafe { + // Declare a symbol `foo` with the desired linkage. + let g1 = cx.declare_global(sym, llty2); + llvm::LLVMRustSetLinkage(g1, base::linkage_to_llvm(linkage)); + + // Declare an internal global `extern_with_linkage_foo` which + // is initialized with the address of `foo`. If `foo` is + // discarded during linking (for example, if `foo` has weak + // linkage and there are no definitions), then + // `extern_with_linkage_foo` will instead be initialized to + // zero. + let mut real_name = "_rust_extern_with_linkage_".to_string(); + real_name.push_str(sym); + let g2 = cx.define_global(&real_name, llty).unwrap_or_else(|| { + cx.sess().span_fatal( + cx.tcx.def_span(span_def_id), + &format!("symbol `{}` is already defined", &sym), + ) + }); + llvm::LLVMRustSetLinkage(g2, llvm::Linkage::InternalLinkage); + llvm::LLVMSetInitializer(g2, g1); + g2 + } + } else { + // Generate an external declaration. + // FIXME(nagisa): investigate whether it can be changed into define_global + cx.declare_global(sym, llty) + } +} + +pub fn ptrcast<'ll>(val: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMConstPointerCast(val, ty) } +} + +impl<'ll> CodegenCx<'ll, '_> { + pub(crate) fn const_bitcast(&self, val: &'ll Value, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMConstBitCast(val, ty) } + } + + pub(crate) fn static_addr_of_mut( + &self, + cv: &'ll Value, + align: Align, + kind: Option<&str>, + ) -> &'ll Value { + unsafe { + let gv = match kind { + Some(kind) if !self.tcx.sess.fewer_names() => { + let name = self.generate_local_symbol_name(kind); + let gv = self.define_global(&name, self.val_ty(cv)).unwrap_or_else(|| { + bug!("symbol `{}` is already defined", name); + }); + llvm::LLVMRustSetLinkage(gv, llvm::Linkage::PrivateLinkage); + gv + } + _ => self.define_private_global(self.val_ty(cv)), + }; + llvm::LLVMSetInitializer(gv, cv); + set_global_alignment(self, gv, align); + llvm::SetUnnamedAddress(gv, llvm::UnnamedAddr::Global); + gv + } + } + + pub(crate) fn get_static(&self, def_id: DefId) -> &'ll Value { + let instance = Instance::mono(self.tcx, def_id); + if let Some(&g) = self.instances.borrow().get(&instance) { + return g; + } + + let defined_in_current_codegen_unit = + self.codegen_unit.items().contains_key(&MonoItem::Static(def_id)); + assert!( + !defined_in_current_codegen_unit, + "consts::get_static() should always hit the cache for \ + statics defined in the same CGU, but did not for `{:?}`", + def_id + ); + + let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let sym = self.tcx.symbol_name(instance).name; + let fn_attrs = self.tcx.codegen_fn_attrs(def_id); + + debug!("get_static: sym={} instance={:?} fn_attrs={:?}", sym, instance, fn_attrs); + + let g = if def_id.is_local() && !self.tcx.is_foreign_item(def_id) { + let llty = self.layout_of(ty).llvm_type(self); + if let Some(g) = self.get_declared_value(sym) { + if self.val_ty(g) != self.type_ptr_to(llty) { + span_bug!(self.tcx.def_span(def_id), "Conflicting types for static"); + } + } + + let g = self.declare_global(sym, llty); + + if !self.tcx.is_reachable_non_generic(def_id) { + unsafe { + llvm::LLVMRustSetVisibility(g, llvm::Visibility::Hidden); + } + } + + g + } else { + check_and_apply_linkage(self, fn_attrs, ty, sym, def_id) + }; + + // Thread-local statics in some other crate need to *always* be linked + // against in a thread-local fashion, so we need to be sure to apply the + // thread-local attribute locally if it was present remotely. If we + // don't do this then linker errors can be generated where the linker + // complains that one object files has a thread local version of the + // symbol and another one doesn't. + if fn_attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { + llvm::set_thread_local_mode(g, self.tls_model); + } + + if !def_id.is_local() { + let needs_dll_storage_attr = self.use_dll_storage_attrs && !self.tcx.is_foreign_item(def_id) && + // ThinLTO can't handle this workaround in all cases, so we don't + // emit the attrs. Instead we make them unnecessary by disallowing + // dynamic linking when linker plugin based LTO is enabled. + !self.tcx.sess.opts.cg.linker_plugin_lto.enabled(); + + // If this assertion triggers, there's something wrong with commandline + // argument validation. + debug_assert!( + !(self.tcx.sess.opts.cg.linker_plugin_lto.enabled() + && self.tcx.sess.target.is_like_windows + && self.tcx.sess.opts.cg.prefer_dynamic) + ); + + if needs_dll_storage_attr { + // This item is external but not foreign, i.e., it originates from an external Rust + // crate. Since we don't know whether this crate will be linked dynamically or + // statically in the final application, we always mark such symbols as 'dllimport'. + // If final linkage happens to be static, we rely on compiler-emitted __imp_ stubs + // to make things work. + // + // However, in some scenarios we defer emission of statics to downstream + // crates, so there are cases where a static with an upstream DefId + // is actually present in the current crate. We can find out via the + // is_codegened_item query. + if !self.tcx.is_codegened_item(def_id) { + unsafe { + llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); + } + } + } + } + + if self.use_dll_storage_attrs && self.tcx.is_dllimport_foreign_item(def_id) { + // For foreign (native) libs we know the exact storage type to use. + unsafe { + llvm::LLVMSetDLLStorageClass(g, llvm::DLLStorageClass::DllImport); + } + } + + unsafe { + if self.should_assume_dso_local(g, true) { + llvm::LLVMRustSetDSOLocal(g, true); + } + } + + self.instances.borrow_mut().insert(instance, g); + g + } +} + +impl<'ll> StaticMethods for CodegenCx<'ll, '_> { + fn static_addr_of(&self, cv: &'ll Value, align: Align, kind: Option<&str>) -> &'ll Value { + if let Some(&gv) = self.const_globals.borrow().get(&cv) { + unsafe { + // Upgrade the alignment in cases where the same constant is used with different + // alignment requirements + let llalign = align.bytes() as u32; + if llalign > llvm::LLVMGetAlignment(gv) { + llvm::LLVMSetAlignment(gv, llalign); + } + } + return gv; + } + let gv = self.static_addr_of_mut(cv, align, kind); + unsafe { + llvm::LLVMSetGlobalConstant(gv, True); + } + self.const_globals.borrow_mut().insert(cv, gv); + gv + } + + fn codegen_static(&self, def_id: DefId, is_mutable: bool) { + unsafe { + let attrs = self.tcx.codegen_fn_attrs(def_id); + + let Ok((v, alloc)) = codegen_static_initializer(self, def_id) else { + // Error has already been reported + return; + }; + let alloc = alloc.inner(); + + let g = self.get_static(def_id); + + // boolean SSA values are i1, but they have to be stored in i8 slots, + // otherwise some LLVM optimization passes don't work as expected + let mut val_llty = self.val_ty(v); + let v = if val_llty == self.type_i1() { + val_llty = self.type_i8(); + llvm::LLVMConstZExt(v, val_llty) + } else { + v + }; + + let instance = Instance::mono(self.tcx, def_id); + let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let llty = self.layout_of(ty).llvm_type(self); + let g = if val_llty == llty { + g + } else { + // If we created the global with the wrong type, + // correct the type. + let name = llvm::get_value_name(g).to_vec(); + llvm::set_value_name(g, b""); + + let linkage = llvm::LLVMRustGetLinkage(g); + let visibility = llvm::LLVMRustGetVisibility(g); + + let new_g = llvm::LLVMRustGetOrInsertGlobal( + self.llmod, + name.as_ptr().cast(), + name.len(), + val_llty, + ); + + llvm::LLVMRustSetLinkage(new_g, linkage); + llvm::LLVMRustSetVisibility(new_g, visibility); + + // The old global has had its name removed but is returned by + // get_static since it is in the instance cache. Provide an + // alternative lookup that points to the new global so that + // global_asm! can compute the correct mangled symbol name + // for the global. + self.renamed_statics.borrow_mut().insert(def_id, new_g); + + // To avoid breaking any invariants, we leave around the old + // global for the moment; we'll replace all references to it + // with the new global later. (See base::codegen_backend.) + self.statics_to_rauw.borrow_mut().push((g, new_g)); + new_g + }; + set_global_alignment(self, g, self.align_of(ty)); + llvm::LLVMSetInitializer(g, v); + + if self.should_assume_dso_local(g, true) { + llvm::LLVMRustSetDSOLocal(g, true); + } + + // As an optimization, all shared statics which do not have interior + // mutability are placed into read-only memory. + if !is_mutable && self.type_is_freeze(ty) { + llvm::LLVMSetGlobalConstant(g, llvm::True); + } + + debuginfo::build_global_var_di_node(self, def_id, g); + + if attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { + llvm::set_thread_local_mode(g, self.tls_model); + + // Do not allow LLVM to change the alignment of a TLS on macOS. + // + // By default a global's alignment can be freely increased. + // This allows LLVM to generate more performant instructions + // e.g., using load-aligned into a SIMD register. + // + // However, on macOS 10.10 or below, the dynamic linker does not + // respect any alignment given on the TLS (radar 24221680). + // This will violate the alignment assumption, and causing segfault at runtime. + // + // This bug is very easy to trigger. In `println!` and `panic!`, + // the `LOCAL_STDOUT`/`LOCAL_STDERR` handles are stored in a TLS, + // which the values would be `mem::replace`d on initialization. + // The implementation of `mem::replace` will use SIMD + // whenever the size is 32 bytes or higher. LLVM notices SIMD is used + // and tries to align `LOCAL_STDOUT`/`LOCAL_STDERR` to a 32-byte boundary, + // which macOS's dyld disregarded and causing crashes + // (see issues #51794, #51758, #50867, #48866 and #44056). + // + // To workaround the bug, we trick LLVM into not increasing + // the global's alignment by explicitly assigning a section to it + // (equivalent to automatically generating a `#[link_section]` attribute). + // See the comment in the `GlobalValue::canIncreaseAlignment()` function + // of `lib/IR/Globals.cpp` for why this works. + // + // When the alignment is not increased, the optimized `mem::replace` + // will use load-unaligned instructions instead, and thus avoiding the crash. + // + // We could remove this hack whenever we decide to drop macOS 10.10 support. + if self.tcx.sess.target.is_like_osx { + // The `inspect` method is okay here because we checked relocations, and + // because we are doing this access to inspect the final interpreter state + // (not as part of the interpreter execution). + // + // FIXME: This check requires that the (arbitrary) value of undefined bytes + // happens to be zero. Instead, we should only check the value of defined bytes + // and set all undefined bytes to zero if this allocation is headed for the + // BSS. + let all_bytes_are_zero = alloc.relocations().is_empty() + && alloc + .inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()) + .iter() + .all(|&byte| byte == 0); + + let sect_name = if all_bytes_are_zero { + cstr!("__DATA,__thread_bss") + } else { + cstr!("__DATA,__thread_data") + }; + llvm::LLVMSetSection(g, sect_name.as_ptr()); + } + } + + // Wasm statics with custom link sections get special treatment as they + // go into custom sections of the wasm executable. + if self.tcx.sess.target.is_like_wasm { + if let Some(section) = attrs.link_section { + let section = llvm::LLVMMDStringInContext( + self.llcx, + section.as_str().as_ptr().cast(), + section.as_str().len() as c_uint, + ); + assert!(alloc.relocations().is_empty()); + + // The `inspect` method is okay here because we checked relocations, and + // because we are doing this access to inspect the final interpreter state (not + // as part of the interpreter execution). + let bytes = + alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()); + let alloc = llvm::LLVMMDStringInContext( + self.llcx, + bytes.as_ptr().cast(), + bytes.len() as c_uint, + ); + let data = [section, alloc]; + let meta = llvm::LLVMMDNodeInContext(self.llcx, data.as_ptr(), 2); + llvm::LLVMAddNamedMetadataOperand( + self.llmod, + "wasm.custom_sections\0".as_ptr().cast(), + meta, + ); + } + } else { + base::set_link_section(g, attrs); + } + + if attrs.flags.contains(CodegenFnAttrFlags::USED) { + // `USED` and `USED_LINKER` can't be used together. + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER)); + + // The semantics of #[used] in Rust only require the symbol to make it into the + // object file. It is explicitly allowed for the linker to strip the symbol if it + // is dead, which means we are allowed use `llvm.compiler.used` instead of + // `llvm.used` here. + // + // Additionally, https://reviews.llvm.org/D97448 in LLVM 13 started emitting unique + // sections with SHF_GNU_RETAIN flag for llvm.used symbols, which may trigger bugs + // in the handling of `.init_array` (the static constructor list) in versions of + // the gold linker (prior to the one released with binutils 2.36). + // + // That said, we only ever emit these when compiling for ELF targets, unless + // `#[used(compiler)]` is explicitly requested. This is to avoid similar breakage + // on other targets, in particular MachO targets have *their* static constructor + // lists broken if `llvm.compiler.used` is emitted rather than llvm.used. However, + // that check happens when assigning the `CodegenFnAttrFlags` in `rustc_typeck`, + // so we don't need to take care of it here. + self.add_compiler_used_global(g); + } + if attrs.flags.contains(CodegenFnAttrFlags::USED_LINKER) { + // `USED` and `USED_LINKER` can't be used together. + assert!(!attrs.flags.contains(CodegenFnAttrFlags::USED)); + + self.add_used_global(g); + } + } + } + + /// Add a global value to a list to be stored in the `llvm.used` variable, an array of i8*. + fn add_used_global(&self, global: &'ll Value) { + let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; + self.used_statics.borrow_mut().push(cast); + } + + /// Add a global value to a list to be stored in the `llvm.compiler.used` variable, + /// an array of i8*. + fn add_compiler_used_global(&self, global: &'ll Value) { + let cast = unsafe { llvm::LLVMConstPointerCast(global, self.type_i8p()) }; + self.compiler_used_statics.borrow_mut().push(cast); + } +} diff --git a/compiler/rustc_codegen_llvm/src/context.rs b/compiler/rustc_codegen_llvm/src/context.rs new file mode 100644 index 000000000..5857b83f6 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/context.rs @@ -0,0 +1,1014 @@ +use crate::attributes; +use crate::back::write::to_llvm_code_model; +use crate::callee::get_fn; +use crate::coverageinfo; +use crate::debuginfo; +use crate::llvm; +use crate::llvm_util; +use crate::type_::Type; +use crate::value::Value; + +use cstr::cstr; +use rustc_codegen_ssa::base::wants_msvc_seh; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::base_n; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_hir::def_id::DefId; +use rustc_middle::mir::mono::CodegenUnit; +use rustc_middle::ty::layout::{ + FnAbiError, FnAbiOfHelpers, FnAbiRequest, HasParamEnv, LayoutError, LayoutOfHelpers, + TyAndLayout, +}; +use rustc_middle::ty::{self, Instance, Ty, TyCtxt}; +use rustc_middle::{bug, span_bug}; +use rustc_session::config::{BranchProtection, CFGuard, CFProtection}; +use rustc_session::config::{CrateType, DebugInfo, PAuthKey, PacRet}; +use rustc_session::Session; +use rustc_span::source_map::Span; +use rustc_target::abi::{ + call::FnAbi, HasDataLayout, PointeeInfo, Size, TargetDataLayout, VariantIdx, +}; +use rustc_target::spec::{HasTargetSpec, RelocModel, Target, TlsModel}; +use smallvec::SmallVec; + +use std::cell::{Cell, RefCell}; +use std::ffi::CStr; +use std::str; + +/// There is one `CodegenCx` per compilation unit. Each one has its own LLVM +/// `llvm::Context` so that several compilation units may be optimized in parallel. +/// All other LLVM data structures in the `CodegenCx` are tied to that `llvm::Context`. +pub struct CodegenCx<'ll, 'tcx> { + pub tcx: TyCtxt<'tcx>, + pub check_overflow: bool, + pub use_dll_storage_attrs: bool, + pub tls_model: llvm::ThreadLocalMode, + + pub llmod: &'ll llvm::Module, + pub llcx: &'ll llvm::Context, + pub codegen_unit: &'tcx CodegenUnit<'tcx>, + + /// Cache instances of monomorphic and polymorphic items + pub instances: RefCell, &'ll Value>>, + /// Cache generated vtables + pub vtables: + RefCell, Option>), &'ll Value>>, + /// Cache of constant strings, + pub const_str_cache: RefCell>, + + /// Reverse-direction for const ptrs cast from globals. + /// + /// Key is a Value holding a `*T`, + /// Val is a Value holding a `*[T]`. + /// + /// Needed because LLVM loses pointer->pointee association + /// when we ptrcast, and we have to ptrcast during codegen + /// of a `[T]` const because we form a slice, a `(*T,usize)` pair, not + /// a pointer to an LLVM array type. Similar for trait objects. + pub const_unsized: RefCell>, + + /// Cache of emitted const globals (value -> global) + pub const_globals: RefCell>, + + /// List of globals for static variables which need to be passed to the + /// LLVM function ReplaceAllUsesWith (RAUW) when codegen is complete. + /// (We have to make sure we don't invalidate any Values referring + /// to constants.) + pub statics_to_rauw: RefCell>, + + /// Statics that will be placed in the llvm.used variable + /// See for details + pub used_statics: RefCell>, + + /// Statics that will be placed in the llvm.compiler.used variable + /// See for details + pub compiler_used_statics: RefCell>, + + /// Mapping of non-scalar types to llvm types and field remapping if needed. + pub type_lowering: RefCell, Option), TypeLowering<'ll>>>, + + /// Mapping of scalar types to llvm types. + pub scalar_lltypes: RefCell, &'ll Type>>, + + pub pointee_infos: RefCell, Size), Option>>, + pub isize_ty: &'ll Type, + + pub coverage_cx: Option>, + pub dbg_cx: Option>, + + eh_personality: Cell>, + eh_catch_typeinfo: Cell>, + pub rust_try_fn: Cell>, + + intrinsics: RefCell>, + + /// A counter that is used for generating local symbol names + local_gen_sym_counter: Cell, + + /// `codegen_static` will sometimes create a second global variable with a + /// different type and clear the symbol name of the original global. + /// `global_asm!` needs to be able to find this new global so that it can + /// compute the correct mangled symbol name to insert into the asm. + pub renamed_statics: RefCell>, +} + +pub struct TypeLowering<'ll> { + /// Associated LLVM type + pub lltype: &'ll Type, + + /// If padding is used the slice maps fields from source order + /// to llvm order. + pub field_remapping: Option>, +} + +fn to_llvm_tls_model(tls_model: TlsModel) -> llvm::ThreadLocalMode { + match tls_model { + TlsModel::GeneralDynamic => llvm::ThreadLocalMode::GeneralDynamic, + TlsModel::LocalDynamic => llvm::ThreadLocalMode::LocalDynamic, + TlsModel::InitialExec => llvm::ThreadLocalMode::InitialExec, + TlsModel::LocalExec => llvm::ThreadLocalMode::LocalExec, + } +} + +pub unsafe fn create_module<'ll>( + tcx: TyCtxt<'_>, + llcx: &'ll llvm::Context, + mod_name: &str, +) -> &'ll llvm::Module { + let sess = tcx.sess; + let mod_name = SmallCStr::new(mod_name); + let llmod = llvm::LLVMModuleCreateWithNameInContext(mod_name.as_ptr(), llcx); + + let mut target_data_layout = sess.target.data_layout.to_string(); + let llvm_version = llvm_util::get_version(); + if llvm_version < (13, 0, 0) { + if sess.target.arch == "powerpc64" { + target_data_layout = target_data_layout.replace("-S128", ""); + } + if sess.target.arch == "wasm32" { + target_data_layout = "e-m:e-p:32:32-i64:64-n32:64-S128".to_string(); + } + if sess.target.arch == "wasm64" { + target_data_layout = "e-m:e-p:64:64-i64:64-n32:64-S128".to_string(); + } + } + if llvm_version < (14, 0, 0) { + if sess.target.llvm_target == "i686-pc-windows-msvc" + || sess.target.llvm_target == "i586-pc-windows-msvc" + { + target_data_layout = + "e-m:x-p:32:32-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:32-n8:16:32-a:0:32-S32" + .to_string(); + } + if sess.target.arch == "wasm32" { + target_data_layout = target_data_layout.replace("-p10:8:8-p20:8:8", ""); + } + } + + // Ensure the data-layout values hardcoded remain the defaults. + if sess.target.is_builtin { + let tm = crate::back::write::create_informational_target_machine(tcx.sess); + llvm::LLVMRustSetDataLayoutFromTargetMachine(llmod, tm); + llvm::LLVMRustDisposeTargetMachine(tm); + + let llvm_data_layout = llvm::LLVMGetDataLayoutStr(llmod); + let llvm_data_layout = str::from_utf8(CStr::from_ptr(llvm_data_layout).to_bytes()) + .expect("got a non-UTF8 data-layout from LLVM"); + + // Unfortunately LLVM target specs change over time, and right now we + // don't have proper support to work with any more than one + // `data_layout` than the one that is in the rust-lang/rust repo. If + // this compiler is configured against a custom LLVM, we may have a + // differing data layout, even though we should update our own to use + // that one. + // + // As an interim hack, if CFG_LLVM_ROOT is not an empty string then we + // disable this check entirely as we may be configured with something + // that has a different target layout. + // + // Unsure if this will actually cause breakage when rustc is configured + // as such. + // + // FIXME(#34960) + let cfg_llvm_root = option_env!("CFG_LLVM_ROOT").unwrap_or(""); + let custom_llvm_used = cfg_llvm_root.trim() != ""; + + if !custom_llvm_used && target_data_layout != llvm_data_layout { + bug!( + "data-layout for target `{rustc_target}`, `{rustc_layout}`, \ + differs from LLVM target's `{llvm_target}` default layout, `{llvm_layout}`", + rustc_target = sess.opts.target_triple, + rustc_layout = target_data_layout, + llvm_target = sess.target.llvm_target, + llvm_layout = llvm_data_layout + ); + } + } + + let data_layout = SmallCStr::new(&target_data_layout); + llvm::LLVMSetDataLayout(llmod, data_layout.as_ptr()); + + let llvm_target = SmallCStr::new(&sess.target.llvm_target); + llvm::LLVMRustSetNormalizedTarget(llmod, llvm_target.as_ptr()); + + let reloc_model = sess.relocation_model(); + if matches!(reloc_model, RelocModel::Pic | RelocModel::Pie) { + llvm::LLVMRustSetModulePICLevel(llmod); + // PIE is potentially more effective than PIC, but can only be used in executables. + // If all our outputs are executables, then we can relax PIC to PIE. + if reloc_model == RelocModel::Pie + || sess.crate_types().iter().all(|ty| *ty == CrateType::Executable) + { + llvm::LLVMRustSetModulePIELevel(llmod); + } + } + + // Linking object files with different code models is undefined behavior + // because the compiler would have to generate additional code (to span + // longer jumps) if a larger code model is used with a smaller one. + // + // See https://reviews.llvm.org/D52322 and https://reviews.llvm.org/D52323. + llvm::LLVMRustSetModuleCodeModel(llmod, to_llvm_code_model(sess.code_model())); + + // If skipping the PLT is enabled, we need to add some module metadata + // to ensure intrinsic calls don't use it. + if !sess.needs_plt() { + let avoid_plt = "RtLibUseGOT\0".as_ptr().cast(); + llvm::LLVMRustAddModuleFlag(llmod, llvm::LLVMModFlagBehavior::Warning, avoid_plt, 1); + } + + if sess.is_sanitizer_cfi_enabled() { + // FIXME(rcvalle): Add support for non canonical jump tables. + let canonical_jump_tables = "CFI Canonical Jump Tables\0".as_ptr().cast(); + // FIXME(rcvalle): Add it with Override behavior flag. + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + canonical_jump_tables, + 1, + ); + } + + // Control Flow Guard is currently only supported by the MSVC linker on Windows. + if sess.target.is_like_msvc { + match sess.opts.cg.control_flow_guard { + CFGuard::Disabled => {} + CFGuard::NoChecks => { + // Set `cfguard=1` module flag to emit metadata only. + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + "cfguard\0".as_ptr() as *const _, + 1, + ) + } + CFGuard::Checks => { + // Set `cfguard=2` module flag to emit metadata and checks. + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Warning, + "cfguard\0".as_ptr() as *const _, + 2, + ) + } + } + } + + if let Some(BranchProtection { bti, pac_ret }) = sess.opts.unstable_opts.branch_protection { + if sess.target.arch != "aarch64" { + sess.err("-Zbranch-protection is only supported on aarch64"); + } else { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "branch-target-enforcement\0".as_ptr().cast(), + bti.into(), + ); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "sign-return-address\0".as_ptr().cast(), + pac_ret.is_some().into(), + ); + let pac_opts = pac_ret.unwrap_or(PacRet { leaf: false, key: PAuthKey::A }); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "sign-return-address-all\0".as_ptr().cast(), + pac_opts.leaf.into(), + ); + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "sign-return-address-with-bkey\0".as_ptr().cast(), + u32::from(pac_opts.key == PAuthKey::B), + ); + } + } + + // Pass on the control-flow protection flags to LLVM (equivalent to `-fcf-protection` in Clang). + if let CFProtection::Branch | CFProtection::Full = sess.opts.unstable_opts.cf_protection { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + "cf-protection-branch\0".as_ptr().cast(), + 1, + ) + } + if let CFProtection::Return | CFProtection::Full = sess.opts.unstable_opts.cf_protection { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Override, + "cf-protection-return\0".as_ptr().cast(), + 1, + ) + } + + if sess.opts.unstable_opts.virtual_function_elimination { + llvm::LLVMRustAddModuleFlag( + llmod, + llvm::LLVMModFlagBehavior::Error, + "Virtual Function Elim\0".as_ptr().cast(), + 1, + ); + } + + llmod +} + +impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { + pub(crate) fn new( + tcx: TyCtxt<'tcx>, + codegen_unit: &'tcx CodegenUnit<'tcx>, + llvm_module: &'ll crate::ModuleLlvm, + ) -> Self { + // An interesting part of Windows which MSVC forces our hand on (and + // apparently MinGW didn't) is the usage of `dllimport` and `dllexport` + // attributes in LLVM IR as well as native dependencies (in C these + // correspond to `__declspec(dllimport)`). + // + // LD (BFD) in MinGW mode can often correctly guess `dllexport` but + // relying on that can result in issues like #50176. + // LLD won't support that and expects symbols with proper attributes. + // Because of that we make MinGW target emit dllexport just like MSVC. + // When it comes to dllimport we use it for constants but for functions + // rely on the linker to do the right thing. Opposed to dllexport this + // task is easy for them (both LD and LLD) and allows us to easily use + // symbols from static libraries in shared libraries. + // + // Whenever a dynamic library is built on Windows it must have its public + // interface specified by functions tagged with `dllexport` or otherwise + // they're not available to be linked against. This poses a few problems + // for the compiler, some of which are somewhat fundamental, but we use + // the `use_dll_storage_attrs` variable below to attach the `dllexport` + // attribute to all LLVM functions that are exported e.g., they're + // already tagged with external linkage). This is suboptimal for a few + // reasons: + // + // * If an object file will never be included in a dynamic library, + // there's no need to attach the dllexport attribute. Most object + // files in Rust are not destined to become part of a dll as binaries + // are statically linked by default. + // * If the compiler is emitting both an rlib and a dylib, the same + // source object file is currently used but with MSVC this may be less + // feasible. The compiler may be able to get around this, but it may + // involve some invasive changes to deal with this. + // + // The flip side of this situation is that whenever you link to a dll and + // you import a function from it, the import should be tagged with + // `dllimport`. At this time, however, the compiler does not emit + // `dllimport` for any declarations other than constants (where it is + // required), which is again suboptimal for even more reasons! + // + // * Calling a function imported from another dll without using + // `dllimport` causes the linker/compiler to have extra overhead (one + // `jmp` instruction on x86) when calling the function. + // * The same object file may be used in different circumstances, so a + // function may be imported from a dll if the object is linked into a + // dll, but it may be just linked against if linked into an rlib. + // * The compiler has no knowledge about whether native functions should + // be tagged dllimport or not. + // + // For now the compiler takes the perf hit (I do not have any numbers to + // this effect) by marking very little as `dllimport` and praying the + // linker will take care of everything. Fixing this problem will likely + // require adding a few attributes to Rust itself (feature gated at the + // start) and then strongly recommending static linkage on Windows! + let use_dll_storage_attrs = tcx.sess.target.is_like_windows; + + let check_overflow = tcx.sess.overflow_checks(); + + let tls_model = to_llvm_tls_model(tcx.sess.tls_model()); + + let (llcx, llmod) = (&*llvm_module.llcx, llvm_module.llmod()); + + let coverage_cx = if tcx.sess.instrument_coverage() { + let covctx = coverageinfo::CrateCoverageContext::new(); + Some(covctx) + } else { + None + }; + + let dbg_cx = if tcx.sess.opts.debuginfo != DebugInfo::None { + let dctx = debuginfo::CodegenUnitDebugContext::new(llmod); + debuginfo::metadata::build_compile_unit_di_node( + tcx, + codegen_unit.name().as_str(), + &dctx, + ); + Some(dctx) + } else { + None + }; + + let isize_ty = Type::ix_llcx(llcx, tcx.data_layout.pointer_size.bits()); + + CodegenCx { + tcx, + check_overflow, + use_dll_storage_attrs, + tls_model, + llmod, + llcx, + codegen_unit, + instances: Default::default(), + vtables: Default::default(), + const_str_cache: Default::default(), + const_unsized: Default::default(), + const_globals: Default::default(), + statics_to_rauw: RefCell::new(Vec::new()), + used_statics: RefCell::new(Vec::new()), + compiler_used_statics: RefCell::new(Vec::new()), + type_lowering: Default::default(), + scalar_lltypes: Default::default(), + pointee_infos: Default::default(), + isize_ty, + coverage_cx, + dbg_cx, + eh_personality: Cell::new(None), + eh_catch_typeinfo: Cell::new(None), + rust_try_fn: Cell::new(None), + intrinsics: Default::default(), + local_gen_sym_counter: Cell::new(0), + renamed_statics: Default::default(), + } + } + + pub(crate) fn statics_to_rauw(&self) -> &RefCell> { + &self.statics_to_rauw + } + + #[inline] + pub fn coverage_context(&self) -> Option<&coverageinfo::CrateCoverageContext<'ll, 'tcx>> { + self.coverage_cx.as_ref() + } + + fn create_used_variable_impl(&self, name: &'static CStr, values: &[&'ll Value]) { + let section = cstr!("llvm.metadata"); + let array = self.const_array(self.type_ptr_to(self.type_i8()), values); + + unsafe { + let g = llvm::LLVMAddGlobal(self.llmod, self.val_ty(array), name.as_ptr()); + llvm::LLVMSetInitializer(g, array); + llvm::LLVMRustSetLinkage(g, llvm::Linkage::AppendingLinkage); + llvm::LLVMSetSection(g, section.as_ptr()); + } + } +} + +impl<'ll, 'tcx> MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn vtables( + &self, + ) -> &RefCell, Option>), &'ll Value>> + { + &self.vtables + } + + fn get_fn(&self, instance: Instance<'tcx>) -> &'ll Value { + get_fn(self, instance) + } + + fn get_fn_addr(&self, instance: Instance<'tcx>) -> &'ll Value { + get_fn(self, instance) + } + + fn eh_personality(&self) -> &'ll Value { + // The exception handling personality function. + // + // If our compilation unit has the `eh_personality` lang item somewhere + // within it, then we just need to codegen that. Otherwise, we're + // building an rlib which will depend on some upstream implementation of + // this function, so we just codegen a generic reference to it. We don't + // specify any of the types for the function, we just make it a symbol + // that LLVM can later use. + // + // Note that MSVC is a little special here in that we don't use the + // `eh_personality` lang item at all. Currently LLVM has support for + // both Dwarf and SEH unwind mechanisms for MSVC targets and uses the + // *name of the personality function* to decide what kind of unwind side + // tables/landing pads to emit. It looks like Dwarf is used by default, + // injecting a dependency on the `_Unwind_Resume` symbol for resuming + // an "exception", but for MSVC we want to force SEH. This means that we + // can't actually have the personality function be our standard + // `rust_eh_personality` function, but rather we wired it up to the + // CRT's custom personality function, which forces LLVM to consider + // landing pads as "landing pads for SEH". + if let Some(llpersonality) = self.eh_personality.get() { + return llpersonality; + } + let tcx = self.tcx; + let llfn = match tcx.lang_items().eh_personality() { + Some(def_id) if !wants_msvc_seh(self.sess()) => self.get_fn_addr( + ty::Instance::resolve( + tcx, + ty::ParamEnv::reveal_all(), + def_id, + tcx.intern_substs(&[]), + ) + .unwrap() + .unwrap(), + ), + _ => { + let name = if wants_msvc_seh(self.sess()) { + "__CxxFrameHandler3" + } else { + "rust_eh_personality" + }; + if let Some(llfn) = self.get_declared_value(name) { + llfn + } else { + let fty = self.type_variadic_func(&[], self.type_i32()); + let llfn = self.declare_cfn(name, llvm::UnnamedAddr::Global, fty); + let target_cpu = attributes::target_cpu_attr(self); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[target_cpu]); + llfn + } + } + }; + self.eh_personality.set(Some(llfn)); + llfn + } + + fn sess(&self) -> &Session { + self.tcx.sess + } + + fn check_overflow(&self) -> bool { + self.check_overflow + } + + fn codegen_unit(&self) -> &'tcx CodegenUnit<'tcx> { + self.codegen_unit + } + + fn used_statics(&self) -> &RefCell> { + &self.used_statics + } + + fn compiler_used_statics(&self) -> &RefCell> { + &self.compiler_used_statics + } + + fn set_frame_pointer_type(&self, llfn: &'ll Value) { + if let Some(attr) = attributes::frame_pointer_type_attr(self) { + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &[attr]); + } + } + + fn apply_target_cpu_attr(&self, llfn: &'ll Value) { + let mut attrs = SmallVec::<[_; 2]>::new(); + attrs.push(attributes::target_cpu_attr(self)); + attrs.extend(attributes::tune_cpu_attr(self)); + attributes::apply_to_llfn(llfn, llvm::AttributePlace::Function, &attrs); + } + + fn create_used_variable(&self) { + self.create_used_variable_impl(cstr!("llvm.used"), &*self.used_statics.borrow()); + } + + fn create_compiler_used_variable(&self) { + self.create_used_variable_impl( + cstr!("llvm.compiler.used"), + &*self.compiler_used_statics.borrow(), + ); + } + + fn declare_c_main(&self, fn_type: Self::Type) -> Option { + if self.get_declared_value("main").is_none() { + Some(self.declare_cfn("main", llvm::UnnamedAddr::Global, fn_type)) + } else { + // If the symbol already exists, it is an error: for example, the user wrote + // #[no_mangle] extern "C" fn main(..) {..} + // instead of #[start] + None + } + } +} + +impl<'ll> CodegenCx<'ll, '_> { + pub(crate) fn get_intrinsic(&self, key: &str) -> (&'ll Type, &'ll Value) { + if let Some(v) = self.intrinsics.borrow().get(key).cloned() { + return v; + } + + self.declare_intrinsic(key).unwrap_or_else(|| bug!("unknown intrinsic '{}'", key)) + } + + fn insert_intrinsic( + &self, + name: &'static str, + args: Option<&[&'ll llvm::Type]>, + ret: &'ll llvm::Type, + ) -> (&'ll llvm::Type, &'ll llvm::Value) { + let fn_ty = if let Some(args) = args { + self.type_func(args, ret) + } else { + self.type_variadic_func(&[], ret) + }; + let f = self.declare_cfn(name, llvm::UnnamedAddr::No, fn_ty); + self.intrinsics.borrow_mut().insert(name, (fn_ty, f)); + (fn_ty, f) + } + + fn declare_intrinsic(&self, key: &str) -> Option<(&'ll Type, &'ll Value)> { + macro_rules! ifn { + ($name:expr, fn() -> $ret:expr) => ( + if key == $name { + return Some(self.insert_intrinsic($name, Some(&[]), $ret)); + } + ); + ($name:expr, fn(...) -> $ret:expr) => ( + if key == $name { + return Some(self.insert_intrinsic($name, None, $ret)); + } + ); + ($name:expr, fn($($arg:expr),*) -> $ret:expr) => ( + if key == $name { + return Some(self.insert_intrinsic($name, Some(&[$($arg),*]), $ret)); + } + ); + } + macro_rules! mk_struct { + ($($field_ty:expr),*) => (self.type_struct( &[$($field_ty),*], false)) + } + + let i8p = self.type_i8p(); + let void = self.type_void(); + let i1 = self.type_i1(); + let t_i8 = self.type_i8(); + let t_i16 = self.type_i16(); + let t_i32 = self.type_i32(); + let t_i64 = self.type_i64(); + let t_i128 = self.type_i128(); + let t_isize = self.type_isize(); + let t_f32 = self.type_f32(); + let t_f64 = self.type_f64(); + let t_metadata = self.type_metadata(); + + ifn!("llvm.wasm.trunc.unsigned.i32.f32", fn(t_f32) -> t_i32); + ifn!("llvm.wasm.trunc.unsigned.i32.f64", fn(t_f64) -> t_i32); + ifn!("llvm.wasm.trunc.unsigned.i64.f32", fn(t_f32) -> t_i64); + ifn!("llvm.wasm.trunc.unsigned.i64.f64", fn(t_f64) -> t_i64); + ifn!("llvm.wasm.trunc.signed.i32.f32", fn(t_f32) -> t_i32); + ifn!("llvm.wasm.trunc.signed.i32.f64", fn(t_f64) -> t_i32); + ifn!("llvm.wasm.trunc.signed.i64.f32", fn(t_f32) -> t_i64); + ifn!("llvm.wasm.trunc.signed.i64.f64", fn(t_f64) -> t_i64); + + ifn!("llvm.fptosi.sat.i8.f32", fn(t_f32) -> t_i8); + ifn!("llvm.fptosi.sat.i16.f32", fn(t_f32) -> t_i16); + ifn!("llvm.fptosi.sat.i32.f32", fn(t_f32) -> t_i32); + ifn!("llvm.fptosi.sat.i64.f32", fn(t_f32) -> t_i64); + ifn!("llvm.fptosi.sat.i128.f32", fn(t_f32) -> t_i128); + ifn!("llvm.fptosi.sat.i8.f64", fn(t_f64) -> t_i8); + ifn!("llvm.fptosi.sat.i16.f64", fn(t_f64) -> t_i16); + ifn!("llvm.fptosi.sat.i32.f64", fn(t_f64) -> t_i32); + ifn!("llvm.fptosi.sat.i64.f64", fn(t_f64) -> t_i64); + ifn!("llvm.fptosi.sat.i128.f64", fn(t_f64) -> t_i128); + + ifn!("llvm.fptoui.sat.i8.f32", fn(t_f32) -> t_i8); + ifn!("llvm.fptoui.sat.i16.f32", fn(t_f32) -> t_i16); + ifn!("llvm.fptoui.sat.i32.f32", fn(t_f32) -> t_i32); + ifn!("llvm.fptoui.sat.i64.f32", fn(t_f32) -> t_i64); + ifn!("llvm.fptoui.sat.i128.f32", fn(t_f32) -> t_i128); + ifn!("llvm.fptoui.sat.i8.f64", fn(t_f64) -> t_i8); + ifn!("llvm.fptoui.sat.i16.f64", fn(t_f64) -> t_i16); + ifn!("llvm.fptoui.sat.i32.f64", fn(t_f64) -> t_i32); + ifn!("llvm.fptoui.sat.i64.f64", fn(t_f64) -> t_i64); + ifn!("llvm.fptoui.sat.i128.f64", fn(t_f64) -> t_i128); + + ifn!("llvm.trap", fn() -> void); + ifn!("llvm.debugtrap", fn() -> void); + ifn!("llvm.frameaddress", fn(t_i32) -> i8p); + + ifn!("llvm.powi.f32", fn(t_f32, t_i32) -> t_f32); + ifn!("llvm.powi.f64", fn(t_f64, t_i32) -> t_f64); + + ifn!("llvm.pow.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.pow.f64", fn(t_f64, t_f64) -> t_f64); + + ifn!("llvm.sqrt.f32", fn(t_f32) -> t_f32); + ifn!("llvm.sqrt.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.sin.f32", fn(t_f32) -> t_f32); + ifn!("llvm.sin.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.cos.f32", fn(t_f32) -> t_f32); + ifn!("llvm.cos.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.exp.f32", fn(t_f32) -> t_f32); + ifn!("llvm.exp.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.exp2.f32", fn(t_f32) -> t_f32); + ifn!("llvm.exp2.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.log.f32", fn(t_f32) -> t_f32); + ifn!("llvm.log.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.log10.f32", fn(t_f32) -> t_f32); + ifn!("llvm.log10.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.log2.f32", fn(t_f32) -> t_f32); + ifn!("llvm.log2.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.fma.f32", fn(t_f32, t_f32, t_f32) -> t_f32); + ifn!("llvm.fma.f64", fn(t_f64, t_f64, t_f64) -> t_f64); + + ifn!("llvm.fabs.f32", fn(t_f32) -> t_f32); + ifn!("llvm.fabs.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.minnum.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.minnum.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.maxnum.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.maxnum.f64", fn(t_f64, t_f64) -> t_f64); + + ifn!("llvm.floor.f32", fn(t_f32) -> t_f32); + ifn!("llvm.floor.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.ceil.f32", fn(t_f32) -> t_f32); + ifn!("llvm.ceil.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.trunc.f32", fn(t_f32) -> t_f32); + ifn!("llvm.trunc.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.copysign.f32", fn(t_f32, t_f32) -> t_f32); + ifn!("llvm.copysign.f64", fn(t_f64, t_f64) -> t_f64); + ifn!("llvm.round.f32", fn(t_f32) -> t_f32); + ifn!("llvm.round.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.rint.f32", fn(t_f32) -> t_f32); + ifn!("llvm.rint.f64", fn(t_f64) -> t_f64); + ifn!("llvm.nearbyint.f32", fn(t_f32) -> t_f32); + ifn!("llvm.nearbyint.f64", fn(t_f64) -> t_f64); + + ifn!("llvm.ctpop.i8", fn(t_i8) -> t_i8); + ifn!("llvm.ctpop.i16", fn(t_i16) -> t_i16); + ifn!("llvm.ctpop.i32", fn(t_i32) -> t_i32); + ifn!("llvm.ctpop.i64", fn(t_i64) -> t_i64); + ifn!("llvm.ctpop.i128", fn(t_i128) -> t_i128); + + ifn!("llvm.ctlz.i8", fn(t_i8, i1) -> t_i8); + ifn!("llvm.ctlz.i16", fn(t_i16, i1) -> t_i16); + ifn!("llvm.ctlz.i32", fn(t_i32, i1) -> t_i32); + ifn!("llvm.ctlz.i64", fn(t_i64, i1) -> t_i64); + ifn!("llvm.ctlz.i128", fn(t_i128, i1) -> t_i128); + + ifn!("llvm.cttz.i8", fn(t_i8, i1) -> t_i8); + ifn!("llvm.cttz.i16", fn(t_i16, i1) -> t_i16); + ifn!("llvm.cttz.i32", fn(t_i32, i1) -> t_i32); + ifn!("llvm.cttz.i64", fn(t_i64, i1) -> t_i64); + ifn!("llvm.cttz.i128", fn(t_i128, i1) -> t_i128); + + ifn!("llvm.bswap.i16", fn(t_i16) -> t_i16); + ifn!("llvm.bswap.i32", fn(t_i32) -> t_i32); + ifn!("llvm.bswap.i64", fn(t_i64) -> t_i64); + ifn!("llvm.bswap.i128", fn(t_i128) -> t_i128); + + ifn!("llvm.bitreverse.i8", fn(t_i8) -> t_i8); + ifn!("llvm.bitreverse.i16", fn(t_i16) -> t_i16); + ifn!("llvm.bitreverse.i32", fn(t_i32) -> t_i32); + ifn!("llvm.bitreverse.i64", fn(t_i64) -> t_i64); + ifn!("llvm.bitreverse.i128", fn(t_i128) -> t_i128); + + ifn!("llvm.fshl.i8", fn(t_i8, t_i8, t_i8) -> t_i8); + ifn!("llvm.fshl.i16", fn(t_i16, t_i16, t_i16) -> t_i16); + ifn!("llvm.fshl.i32", fn(t_i32, t_i32, t_i32) -> t_i32); + ifn!("llvm.fshl.i64", fn(t_i64, t_i64, t_i64) -> t_i64); + ifn!("llvm.fshl.i128", fn(t_i128, t_i128, t_i128) -> t_i128); + + ifn!("llvm.fshr.i8", fn(t_i8, t_i8, t_i8) -> t_i8); + ifn!("llvm.fshr.i16", fn(t_i16, t_i16, t_i16) -> t_i16); + ifn!("llvm.fshr.i32", fn(t_i32, t_i32, t_i32) -> t_i32); + ifn!("llvm.fshr.i64", fn(t_i64, t_i64, t_i64) -> t_i64); + ifn!("llvm.fshr.i128", fn(t_i128, t_i128, t_i128) -> t_i128); + + ifn!("llvm.sadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.sadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.sadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.sadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.sadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.uadd.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.uadd.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.uadd.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.uadd.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.uadd.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.ssub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.ssub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.ssub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.ssub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.ssub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.usub.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.usub.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.usub.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.usub.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.usub.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.smul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.smul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.smul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.smul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.smul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.umul.with.overflow.i8", fn(t_i8, t_i8) -> mk_struct! {t_i8, i1}); + ifn!("llvm.umul.with.overflow.i16", fn(t_i16, t_i16) -> mk_struct! {t_i16, i1}); + ifn!("llvm.umul.with.overflow.i32", fn(t_i32, t_i32) -> mk_struct! {t_i32, i1}); + ifn!("llvm.umul.with.overflow.i64", fn(t_i64, t_i64) -> mk_struct! {t_i64, i1}); + ifn!("llvm.umul.with.overflow.i128", fn(t_i128, t_i128) -> mk_struct! {t_i128, i1}); + + ifn!("llvm.sadd.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.sadd.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!("llvm.sadd.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!("llvm.sadd.sat.i64", fn(t_i64, t_i64) -> t_i64); + ifn!("llvm.sadd.sat.i128", fn(t_i128, t_i128) -> t_i128); + + ifn!("llvm.uadd.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.uadd.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!("llvm.uadd.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!("llvm.uadd.sat.i64", fn(t_i64, t_i64) -> t_i64); + ifn!("llvm.uadd.sat.i128", fn(t_i128, t_i128) -> t_i128); + + ifn!("llvm.ssub.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.ssub.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!("llvm.ssub.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!("llvm.ssub.sat.i64", fn(t_i64, t_i64) -> t_i64); + ifn!("llvm.ssub.sat.i128", fn(t_i128, t_i128) -> t_i128); + + ifn!("llvm.usub.sat.i8", fn(t_i8, t_i8) -> t_i8); + ifn!("llvm.usub.sat.i16", fn(t_i16, t_i16) -> t_i16); + ifn!("llvm.usub.sat.i32", fn(t_i32, t_i32) -> t_i32); + ifn!("llvm.usub.sat.i64", fn(t_i64, t_i64) -> t_i64); + ifn!("llvm.usub.sat.i128", fn(t_i128, t_i128) -> t_i128); + + ifn!("llvm.lifetime.start.p0i8", fn(t_i64, i8p) -> void); + ifn!("llvm.lifetime.end.p0i8", fn(t_i64, i8p) -> void); + + ifn!("llvm.expect.i1", fn(i1, i1) -> i1); + ifn!("llvm.eh.typeid.for", fn(i8p) -> t_i32); + ifn!("llvm.localescape", fn(...) -> void); + ifn!("llvm.localrecover", fn(i8p, i8p, t_i32) -> i8p); + ifn!("llvm.x86.seh.recoverfp", fn(i8p, i8p) -> i8p); + + ifn!("llvm.assume", fn(i1) -> void); + ifn!("llvm.prefetch", fn(i8p, t_i32, t_i32, t_i32) -> void); + + // This isn't an "LLVM intrinsic", but LLVM's optimization passes + // recognize it like one and we assume it exists in `core::slice::cmp` + match self.sess().target.arch.as_ref() { + "avr" | "msp430" => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i16), + _ => ifn!("memcmp", fn(i8p, i8p, t_isize) -> t_i32), + } + + // variadic intrinsics + ifn!("llvm.va_start", fn(i8p) -> void); + ifn!("llvm.va_end", fn(i8p) -> void); + ifn!("llvm.va_copy", fn(i8p, i8p) -> void); + + if self.sess().instrument_coverage() { + ifn!("llvm.instrprof.increment", fn(i8p, t_i64, t_i32, t_i32) -> void); + } + + ifn!("llvm.type.test", fn(i8p, t_metadata) -> i1); + ifn!("llvm.type.checked.load", fn(i8p, t_i32, t_metadata) -> mk_struct! {i8p, i1}); + + if self.sess().opts.debuginfo != DebugInfo::None { + ifn!("llvm.dbg.declare", fn(t_metadata, t_metadata) -> void); + ifn!("llvm.dbg.value", fn(t_metadata, t_i64, t_metadata) -> void); + } + None + } + + pub(crate) fn eh_catch_typeinfo(&self) -> &'ll Value { + if let Some(eh_catch_typeinfo) = self.eh_catch_typeinfo.get() { + return eh_catch_typeinfo; + } + let tcx = self.tcx; + assert!(self.sess().target.os == "emscripten"); + let eh_catch_typeinfo = match tcx.lang_items().eh_catch_typeinfo() { + Some(def_id) => self.get_static(def_id), + _ => { + let ty = self + .type_struct(&[self.type_ptr_to(self.type_isize()), self.type_i8p()], false); + self.declare_global("rust_eh_catch_typeinfo", ty) + } + }; + let eh_catch_typeinfo = self.const_bitcast(eh_catch_typeinfo, self.type_i8p()); + self.eh_catch_typeinfo.set(Some(eh_catch_typeinfo)); + eh_catch_typeinfo + } +} + +impl CodegenCx<'_, '_> { + /// Generates a new symbol name with the given prefix. This symbol name must + /// only be used for definitions with `internal` or `private` linkage. + pub fn generate_local_symbol_name(&self, prefix: &str) -> String { + let idx = self.local_gen_sym_counter.get(); + self.local_gen_sym_counter.set(idx + 1); + // Include a '.' character, so there can be no accidental conflicts with + // user defined names + let mut name = String::with_capacity(prefix.len() + 6); + name.push_str(prefix); + name.push('.'); + base_n::push_str(idx as u128, base_n::ALPHANUMERIC_ONLY, &mut name); + name + } +} + +impl HasDataLayout for CodegenCx<'_, '_> { + #[inline] + fn data_layout(&self) -> &TargetDataLayout { + &self.tcx.data_layout + } +} + +impl HasTargetSpec for CodegenCx<'_, '_> { + #[inline] + fn target_spec(&self) -> &Target { + &self.tcx.sess.target + } +} + +impl<'tcx> ty::layout::HasTyCtxt<'tcx> for CodegenCx<'_, 'tcx> { + #[inline] + fn tcx(&self) -> TyCtxt<'tcx> { + self.tcx + } +} + +impl<'tcx, 'll> HasParamEnv<'tcx> for CodegenCx<'ll, 'tcx> { + fn param_env(&self) -> ty::ParamEnv<'tcx> { + ty::ParamEnv::reveal_all() + } +} + +impl<'tcx> LayoutOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { + type LayoutOfResult = TyAndLayout<'tcx>; + + #[inline] + fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { + if let LayoutError::SizeOverflow(_) = err { + self.sess().span_fatal(span, &err.to_string()) + } else { + span_bug!(span, "failed to get layout for `{}`: {}", ty, err) + } + } +} + +impl<'tcx> FnAbiOfHelpers<'tcx> for CodegenCx<'_, 'tcx> { + type FnAbiOfResult = &'tcx FnAbi<'tcx, Ty<'tcx>>; + + #[inline] + fn handle_fn_abi_err( + &self, + err: FnAbiError<'tcx>, + span: Span, + fn_abi_request: FnAbiRequest<'tcx>, + ) -> ! { + if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { + self.sess().span_fatal(span, &err.to_string()) + } else { + match fn_abi_request { + FnAbiRequest::OfFnPtr { sig, extra_args } => { + span_bug!( + span, + "`fn_abi_of_fn_ptr({}, {:?})` failed: {}", + sig, + extra_args, + err + ); + } + FnAbiRequest::OfInstance { instance, extra_args } => { + span_bug!( + span, + "`fn_abi_of_instance({}, {:?})` failed: {}", + instance, + extra_args, + err + ); + } + } + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs new file mode 100644 index 000000000..58f391692 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mapgen.rs @@ -0,0 +1,334 @@ +use crate::common::CodegenCx; +use crate::coverageinfo; +use crate::llvm; + +use llvm::coverageinfo::CounterMappingRegion; +use rustc_codegen_ssa::coverageinfo::map::{Counter, CounterExpression}; +use rustc_codegen_ssa::traits::{ConstMethods, CoverageInfoMethods}; +use rustc_data_structures::fx::FxIndexSet; +use rustc_hir::def::DefKind; +use rustc_hir::def_id::DefIdSet; +use rustc_llvm::RustString; +use rustc_middle::bug; +use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; +use rustc_middle::mir::coverage::CodeRegion; +use rustc_middle::ty::TyCtxt; + +use std::ffi::CString; + +use tracing::debug; + +/// Generates and exports the Coverage Map. +/// +/// Rust Coverage Map generation supports LLVM Coverage Mapping Format versions +/// 5 (LLVM 12, only) and 6 (zero-based encoded as 4 and 5, respectively), as defined at +/// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). +/// These versions are supported by the LLVM coverage tools (`llvm-profdata` and `llvm-cov`) +/// bundled with Rust's fork of LLVM. +/// +/// Consequently, Rust's bundled version of Clang also generates Coverage Maps compliant with +/// the same version. Clang's implementation of Coverage Map generation was referenced when +/// implementing this Rust version, and though the format documentation is very explicit and +/// detailed, some undocumented details in Clang's implementation (that may or may not be important) +/// were also replicated for Rust's Coverage Map. +pub fn finalize<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { + let tcx = cx.tcx; + + // Ensure the installed version of LLVM supports at least Coverage Map + // Version 5 (encoded as a zero-based value: 4), which was introduced with + // LLVM 12. + let version = coverageinfo::mapping_version(); + if version < 4 { + tcx.sess.fatal("rustc option `-C instrument-coverage` requires LLVM 12 or higher."); + } + + debug!("Generating coverage map for CodegenUnit: `{}`", cx.codegen_unit.name()); + + // In order to show that unused functions have coverage counts of zero (0), LLVM requires the + // functions exist. Generate synthetic functions with a (required) single counter, and add the + // MIR `Coverage` code regions to the `function_coverage_map`, before calling + // `ctx.take_function_coverage_map()`. + if cx.codegen_unit.is_code_coverage_dead_code_cgu() { + add_unused_functions(cx); + } + + let function_coverage_map = match cx.coverage_context() { + Some(ctx) => ctx.take_function_coverage_map(), + None => return, + }; + + if function_coverage_map.is_empty() { + // This module has no functions with coverage instrumentation + return; + } + + let mut mapgen = CoverageMapGenerator::new(tcx, version); + + // Encode coverage mappings and generate function records + let mut function_data = Vec::new(); + for (instance, function_coverage) in function_coverage_map { + debug!("Generate function coverage for {}, {:?}", cx.codegen_unit.name(), instance); + let mangled_function_name = tcx.symbol_name(instance).to_string(); + let source_hash = function_coverage.source_hash(); + let is_used = function_coverage.is_used(); + let (expressions, counter_regions) = + function_coverage.get_expressions_and_counter_regions(); + + let coverage_mapping_buffer = llvm::build_byte_buffer(|coverage_mapping_buffer| { + mapgen.write_coverage_mapping(expressions, counter_regions, coverage_mapping_buffer); + }); + + if coverage_mapping_buffer.is_empty() { + if function_coverage.is_used() { + bug!( + "A used function should have had coverage mapping data but did not: {}", + mangled_function_name + ); + } else { + debug!("unused function had no coverage mapping data: {}", mangled_function_name); + continue; + } + } + + function_data.push((mangled_function_name, source_hash, is_used, coverage_mapping_buffer)); + } + + // Encode all filenames referenced by counters/expressions in this module + let filenames_buffer = llvm::build_byte_buffer(|filenames_buffer| { + coverageinfo::write_filenames_section_to_buffer(&mapgen.filenames, filenames_buffer); + }); + + let filenames_size = filenames_buffer.len(); + let filenames_val = cx.const_bytes(&filenames_buffer); + let filenames_ref = coverageinfo::hash_bytes(filenames_buffer); + + // Generate the LLVM IR representation of the coverage map and store it in a well-known global + let cov_data_val = mapgen.generate_coverage_map(cx, version, filenames_size, filenames_val); + + for (mangled_function_name, source_hash, is_used, coverage_mapping_buffer) in function_data { + save_function_record( + cx, + mangled_function_name, + source_hash, + filenames_ref, + coverage_mapping_buffer, + is_used, + ); + } + + // Save the coverage data value to LLVM IR + coverageinfo::save_cov_data_to_mod(cx, cov_data_val); +} + +struct CoverageMapGenerator { + filenames: FxIndexSet, +} + +impl CoverageMapGenerator { + fn new(tcx: TyCtxt<'_>, version: u32) -> Self { + let mut filenames = FxIndexSet::default(); + if version >= 5 { + // LLVM Coverage Mapping Format version 6 (zero-based encoded as 5) + // requires setting the first filename to the compilation directory. + // Since rustc generates coverage maps with relative paths, the + // compilation directory can be combined with the the relative paths + // to get absolute paths, if needed. + let working_dir = tcx + .sess + .opts + .working_dir + .remapped_path_if_available() + .to_string_lossy() + .to_string(); + let c_filename = + CString::new(working_dir).expect("null error converting filename to C string"); + filenames.insert(c_filename); + } + Self { filenames } + } + + /// Using the `expressions` and `counter_regions` collected for the current function, generate + /// the `mapping_regions` and `virtual_file_mapping`, and capture any new filenames. Then use + /// LLVM APIs to encode the `virtual_file_mapping`, `expressions`, and `mapping_regions` into + /// the given `coverage_mapping` byte buffer, compliant with the LLVM Coverage Mapping format. + fn write_coverage_mapping<'a>( + &mut self, + expressions: Vec, + counter_regions: impl Iterator, + coverage_mapping_buffer: &RustString, + ) { + let mut counter_regions = counter_regions.collect::>(); + if counter_regions.is_empty() { + return; + } + + let mut virtual_file_mapping = Vec::new(); + let mut mapping_regions = Vec::new(); + let mut current_file_name = None; + let mut current_file_id = 0; + + // Convert the list of (Counter, CodeRegion) pairs to an array of `CounterMappingRegion`, sorted + // by filename and position. Capture any new files to compute the `CounterMappingRegion`s + // `file_id` (indexing files referenced by the current function), and construct the + // function-specific `virtual_file_mapping` from `file_id` to its index in the module's + // `filenames` array. + counter_regions.sort_unstable_by_key(|(_counter, region)| *region); + for (counter, region) in counter_regions { + let CodeRegion { file_name, start_line, start_col, end_line, end_col } = *region; + let same_file = current_file_name.as_ref().map_or(false, |p| *p == file_name); + if !same_file { + if current_file_name.is_some() { + current_file_id += 1; + } + current_file_name = Some(file_name); + let c_filename = CString::new(file_name.to_string()) + .expect("null error converting filename to C string"); + debug!(" file_id: {} = '{:?}'", current_file_id, c_filename); + let (filenames_index, _) = self.filenames.insert_full(c_filename); + virtual_file_mapping.push(filenames_index as u32); + } + debug!("Adding counter {:?} to map for {:?}", counter, region); + mapping_regions.push(CounterMappingRegion::code_region( + counter, + current_file_id, + start_line, + start_col, + end_line, + end_col, + )); + } + + // Encode and append the current function's coverage mapping data + coverageinfo::write_mapping_to_buffer( + virtual_file_mapping, + expressions, + mapping_regions, + coverage_mapping_buffer, + ); + } + + /// Construct coverage map header and the array of function records, and combine them into the + /// coverage map. Save the coverage map data into the LLVM IR as a static global using a + /// specific, well-known section and name. + fn generate_coverage_map<'ll>( + self, + cx: &CodegenCx<'ll, '_>, + version: u32, + filenames_size: usize, + filenames_val: &'ll llvm::Value, + ) -> &'ll llvm::Value { + debug!("cov map: filenames_size = {}, 0-based version = {}", filenames_size, version); + + // Create the coverage data header (Note, fields 0 and 2 are now always zero, + // as of `llvm::coverage::CovMapVersion::Version4`.) + let zero_was_n_records_val = cx.const_u32(0); + let filenames_size_val = cx.const_u32(filenames_size as u32); + let zero_was_coverage_size_val = cx.const_u32(0); + let version_val = cx.const_u32(version); + let cov_data_header_val = cx.const_struct( + &[zero_was_n_records_val, filenames_size_val, zero_was_coverage_size_val, version_val], + /*packed=*/ false, + ); + + // Create the complete LLVM coverage data value to add to the LLVM IR + cx.const_struct(&[cov_data_header_val, filenames_val], /*packed=*/ false) + } +} + +/// Construct a function record and combine it with the function's coverage mapping data. +/// Save the function record into the LLVM IR as a static global using a +/// specific, well-known section and name. +fn save_function_record( + cx: &CodegenCx<'_, '_>, + mangled_function_name: String, + source_hash: u64, + filenames_ref: u64, + coverage_mapping_buffer: Vec, + is_used: bool, +) { + // Concatenate the encoded coverage mappings + let coverage_mapping_size = coverage_mapping_buffer.len(); + let coverage_mapping_val = cx.const_bytes(&coverage_mapping_buffer); + + let func_name_hash = coverageinfo::hash_str(&mangled_function_name); + let func_name_hash_val = cx.const_u64(func_name_hash); + let coverage_mapping_size_val = cx.const_u32(coverage_mapping_size as u32); + let source_hash_val = cx.const_u64(source_hash); + let filenames_ref_val = cx.const_u64(filenames_ref); + let func_record_val = cx.const_struct( + &[ + func_name_hash_val, + coverage_mapping_size_val, + source_hash_val, + filenames_ref_val, + coverage_mapping_val, + ], + /*packed=*/ true, + ); + + coverageinfo::save_func_record_to_mod(cx, func_name_hash, func_record_val, is_used); +} + +/// When finalizing the coverage map, `FunctionCoverage` only has the `CodeRegion`s and counters for +/// the functions that went through codegen; such as public functions and "used" functions +/// (functions referenced by other "used" or public items). Any other functions considered unused, +/// or "Unreachable", were still parsed and processed through the MIR stage, but were not +/// codegenned. (Note that `-Clink-dead-code` can force some unused code to be codegenned, but +/// that flag is known to cause other errors, when combined with `-C instrument-coverage`; and +/// `-Clink-dead-code` will not generate code for unused generic functions.) +/// +/// We can find the unused functions (including generic functions) by the set difference of all MIR +/// `DefId`s (`tcx` query `mir_keys`) minus the codegenned `DefId`s (`tcx` query +/// `codegened_and_inlined_items`). +/// +/// These unused functions are then codegen'd in one of the CGUs which is marked as the +/// "code coverage dead code cgu" during the partitioning process. This prevents us from generating +/// code regions for the same function more than once which can lead to linker errors regarding +/// duplicate symbols. +fn add_unused_functions<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) { + assert!(cx.codegen_unit.is_code_coverage_dead_code_cgu()); + + let tcx = cx.tcx; + + let ignore_unused_generics = tcx.sess.instrument_coverage_except_unused_generics(); + + let eligible_def_ids: DefIdSet = tcx + .mir_keys(()) + .iter() + .filter_map(|local_def_id| { + let def_id = local_def_id.to_def_id(); + let kind = tcx.def_kind(def_id); + // `mir_keys` will give us `DefId`s for all kinds of things, not + // just "functions", like consts, statics, etc. Filter those out. + // If `ignore_unused_generics` was specified, filter out any + // generic functions from consideration as well. + if !matches!( + kind, + DefKind::Fn | DefKind::AssocFn | DefKind::Closure | DefKind::Generator + ) { + return None; + } else if ignore_unused_generics + && tcx.generics_of(def_id).requires_monomorphization(tcx) + { + return None; + } + Some(local_def_id.to_def_id()) + }) + .collect(); + + let codegenned_def_ids = tcx.codegened_and_inlined_items(()); + + for &non_codegenned_def_id in eligible_def_ids.difference(codegenned_def_ids) { + let codegen_fn_attrs = tcx.codegen_fn_attrs(non_codegenned_def_id); + + // If a function is marked `#[no_coverage]`, then skip generating a + // dead code stub for it. + if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NO_COVERAGE) { + debug!("skipping unused fn marked #[no_coverage]: {:?}", non_codegenned_def_id); + continue; + } + + debug!("generating unused fn: {:?}", non_codegenned_def_id); + cx.define_unused_fn(non_codegenned_def_id); + } +} diff --git a/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs new file mode 100644 index 000000000..98ba38356 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/coverageinfo/mod.rs @@ -0,0 +1,385 @@ +use crate::llvm; + +use crate::abi::Abi; +use crate::builder::Builder; +use crate::common::CodegenCx; + +use libc::c_uint; +use llvm::coverageinfo::CounterMappingRegion; +use rustc_codegen_ssa::coverageinfo::map::{CounterExpression, FunctionCoverage}; +use rustc_codegen_ssa::traits::{ + BaseTypeMethods, BuilderMethods, ConstMethods, CoverageInfoBuilderMethods, CoverageInfoMethods, + MiscMethods, StaticMethods, +}; +use rustc_data_structures::fx::FxHashMap; +use rustc_hir as hir; +use rustc_hir::def_id::DefId; +use rustc_llvm::RustString; +use rustc_middle::bug; +use rustc_middle::mir::coverage::{ + CodeRegion, CounterValueReference, ExpressionOperandId, InjectedExpressionId, Op, +}; +use rustc_middle::ty; +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::subst::InternalSubsts; +use rustc_middle::ty::Instance; + +use std::cell::RefCell; +use std::ffi::CString; + +use std::iter; +use tracing::debug; + +pub mod mapgen; + +const UNUSED_FUNCTION_COUNTER_ID: CounterValueReference = CounterValueReference::START; + +const VAR_ALIGN_BYTES: usize = 8; + +/// A context object for maintaining all state needed by the coverageinfo module. +pub struct CrateCoverageContext<'ll, 'tcx> { + // Coverage data for each instrumented function identified by DefId. + pub(crate) function_coverage_map: RefCell, FunctionCoverage<'tcx>>>, + pub(crate) pgo_func_name_var_map: RefCell, &'ll llvm::Value>>, +} + +impl<'ll, 'tcx> CrateCoverageContext<'ll, 'tcx> { + pub fn new() -> Self { + Self { + function_coverage_map: Default::default(), + pgo_func_name_var_map: Default::default(), + } + } + + pub fn take_function_coverage_map(&self) -> FxHashMap, FunctionCoverage<'tcx>> { + self.function_coverage_map.replace(FxHashMap::default()) + } +} + +impl<'ll, 'tcx> CoverageInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn coverageinfo_finalize(&self) { + mapgen::finalize(self) + } + + fn get_pgo_func_name_var(&self, instance: Instance<'tcx>) -> &'ll llvm::Value { + if let Some(coverage_context) = self.coverage_context() { + debug!("getting pgo_func_name_var for instance={:?}", instance); + let mut pgo_func_name_var_map = coverage_context.pgo_func_name_var_map.borrow_mut(); + pgo_func_name_var_map + .entry(instance) + .or_insert_with(|| create_pgo_func_name_var(self, instance)) + } else { + bug!("Could not get the `coverage_context`"); + } + } + + /// Functions with MIR-based coverage are normally codegenned _only_ if + /// called. LLVM coverage tools typically expect every function to be + /// defined (even if unused), with at least one call to LLVM intrinsic + /// `instrprof.increment`. + /// + /// Codegen a small function that will never be called, with one counter + /// that will never be incremented. + /// + /// For used/called functions, the coverageinfo was already added to the + /// `function_coverage_map` (keyed by function `Instance`) during codegen. + /// But in this case, since the unused function was _not_ previously + /// codegenned, collect the coverage `CodeRegion`s from the MIR and add + /// them. The first `CodeRegion` is used to add a single counter, with the + /// same counter ID used in the injected `instrprof.increment` intrinsic + /// call. Since the function is never called, all other `CodeRegion`s can be + /// added as `unreachable_region`s. + fn define_unused_fn(&self, def_id: DefId) { + let instance = declare_unused_fn(self, def_id); + codegen_unused_fn_and_counter(self, instance); + add_unused_function_coverage(self, instance, def_id); + } +} + +impl<'tcx> CoverageInfoBuilderMethods<'tcx> for Builder<'_, '_, 'tcx> { + fn set_function_source_hash( + &mut self, + instance: Instance<'tcx>, + function_source_hash: u64, + ) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "ensuring function source hash is set for instance={:?}; function_source_hash={}", + instance, function_source_hash, + ); + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .set_function_source_hash(function_source_hash); + true + } else { + false + } + } + + fn add_coverage_counter( + &mut self, + instance: Instance<'tcx>, + id: CounterValueReference, + region: CodeRegion, + ) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "adding counter to coverage_map: instance={:?}, id={:?}, region={:?}", + instance, id, region, + ); + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .add_counter(id, region); + true + } else { + false + } + } + + fn add_coverage_counter_expression( + &mut self, + instance: Instance<'tcx>, + id: InjectedExpressionId, + lhs: ExpressionOperandId, + op: Op, + rhs: ExpressionOperandId, + region: Option, + ) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "adding counter expression to coverage_map: instance={:?}, id={:?}, {:?} {:?} {:?}; \ + region: {:?}", + instance, id, lhs, op, rhs, region, + ); + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .add_counter_expression(id, lhs, op, rhs, region); + true + } else { + false + } + } + + fn add_coverage_unreachable(&mut self, instance: Instance<'tcx>, region: CodeRegion) -> bool { + if let Some(coverage_context) = self.coverage_context() { + debug!( + "adding unreachable code to coverage_map: instance={:?}, at {:?}", + instance, region, + ); + let mut coverage_map = coverage_context.function_coverage_map.borrow_mut(); + coverage_map + .entry(instance) + .or_insert_with(|| FunctionCoverage::new(self.tcx, instance)) + .add_unreachable_region(region); + true + } else { + false + } + } +} + +fn declare_unused_fn<'tcx>(cx: &CodegenCx<'_, 'tcx>, def_id: DefId) -> Instance<'tcx> { + let tcx = cx.tcx; + + let instance = Instance::new( + def_id, + InternalSubsts::for_item(tcx, def_id, |param, _| { + if let ty::GenericParamDefKind::Lifetime = param.kind { + tcx.lifetimes.re_erased.into() + } else { + tcx.mk_param_from_def(param) + } + }), + ); + + let llfn = cx.declare_fn( + tcx.symbol_name(instance).name, + cx.fn_abi_of_fn_ptr( + ty::Binder::dummy(tcx.mk_fn_sig( + iter::once(tcx.mk_unit()), + tcx.mk_unit(), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + )), + ty::List::empty(), + ), + ); + + llvm::set_linkage(llfn, llvm::Linkage::PrivateLinkage); + llvm::set_visibility(llfn, llvm::Visibility::Default); + + assert!(cx.instances.borrow_mut().insert(instance, llfn).is_none()); + + instance +} + +fn codegen_unused_fn_and_counter<'tcx>(cx: &CodegenCx<'_, 'tcx>, instance: Instance<'tcx>) { + let llfn = cx.get_fn(instance); + let llbb = Builder::append_block(cx, llfn, "unused_function"); + let mut bx = Builder::build(cx, llbb); + let fn_name = bx.get_pgo_func_name_var(instance); + let hash = bx.const_u64(0); + let num_counters = bx.const_u32(1); + let index = bx.const_u32(u32::from(UNUSED_FUNCTION_COUNTER_ID)); + debug!( + "codegen intrinsic instrprof.increment(fn_name={:?}, hash={:?}, num_counters={:?}, + index={:?}) for unused function: {:?}", + fn_name, hash, num_counters, index, instance + ); + bx.instrprof_increment(fn_name, hash, num_counters, index); + bx.ret_void(); +} + +fn add_unused_function_coverage<'tcx>( + cx: &CodegenCx<'_, 'tcx>, + instance: Instance<'tcx>, + def_id: DefId, +) { + let tcx = cx.tcx; + + let mut function_coverage = FunctionCoverage::unused(tcx, instance); + for (index, &code_region) in tcx.covered_code_regions(def_id).iter().enumerate() { + if index == 0 { + // Insert at least one real counter so the LLVM CoverageMappingReader will find expected + // definitions. + function_coverage.add_counter(UNUSED_FUNCTION_COUNTER_ID, code_region.clone()); + } else { + function_coverage.add_unreachable_region(code_region.clone()); + } + } + + if let Some(coverage_context) = cx.coverage_context() { + coverage_context.function_coverage_map.borrow_mut().insert(instance, function_coverage); + } else { + bug!("Could not get the `coverage_context`"); + } +} + +/// Calls llvm::createPGOFuncNameVar() with the given function instance's +/// mangled function name. The LLVM API returns an llvm::GlobalVariable +/// containing the function name, with the specific variable name and linkage +/// required by LLVM InstrProf source-based coverage instrumentation. Use +/// `bx.get_pgo_func_name_var()` to ensure the variable is only created once per +/// `Instance`. +fn create_pgo_func_name_var<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, +) -> &'ll llvm::Value { + let mangled_fn_name = CString::new(cx.tcx.symbol_name(instance).name) + .expect("error converting function name to C string"); + let llfn = cx.get_fn(instance); + unsafe { llvm::LLVMRustCoverageCreatePGOFuncNameVar(llfn, mangled_fn_name.as_ptr()) } +} + +pub(crate) fn write_filenames_section_to_buffer<'a>( + filenames: impl IntoIterator, + buffer: &RustString, +) { + let c_str_vec = filenames.into_iter().map(|cstring| cstring.as_ptr()).collect::>(); + unsafe { + llvm::LLVMRustCoverageWriteFilenamesSectionToBuffer( + c_str_vec.as_ptr(), + c_str_vec.len(), + buffer, + ); + } +} + +pub(crate) fn write_mapping_to_buffer( + virtual_file_mapping: Vec, + expressions: Vec, + mapping_regions: Vec, + buffer: &RustString, +) { + unsafe { + llvm::LLVMRustCoverageWriteMappingToBuffer( + virtual_file_mapping.as_ptr(), + virtual_file_mapping.len() as c_uint, + expressions.as_ptr(), + expressions.len() as c_uint, + mapping_regions.as_ptr(), + mapping_regions.len() as c_uint, + buffer, + ); + } +} + +pub(crate) fn hash_str(strval: &str) -> u64 { + let strval = CString::new(strval).expect("null error converting hashable str to C string"); + unsafe { llvm::LLVMRustCoverageHashCString(strval.as_ptr()) } +} + +pub(crate) fn hash_bytes(bytes: Vec) -> u64 { + unsafe { llvm::LLVMRustCoverageHashByteArray(bytes.as_ptr().cast(), bytes.len()) } +} + +pub(crate) fn mapping_version() -> u32 { + unsafe { llvm::LLVMRustCoverageMappingVersion() } +} + +pub(crate) fn save_cov_data_to_mod<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + cov_data_val: &'ll llvm::Value, +) { + let covmap_var_name = llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteMappingVarNameToString(s); + }) + .expect("Rust Coverage Mapping var name failed UTF-8 conversion"); + debug!("covmap var name: {:?}", covmap_var_name); + + let covmap_section_name = llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteMapSectionNameToString(cx.llmod, s); + }) + .expect("Rust Coverage section name failed UTF-8 conversion"); + debug!("covmap section name: {:?}", covmap_section_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(cov_data_val), &covmap_var_name); + llvm::set_initializer(llglobal, cov_data_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::PrivateLinkage); + llvm::set_section(llglobal, &covmap_section_name); + llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); + cx.add_used_global(llglobal); +} + +pub(crate) fn save_func_record_to_mod<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + func_name_hash: u64, + func_record_val: &'ll llvm::Value, + is_used: bool, +) { + // Assign a name to the function record. This is used to merge duplicates. + // + // In LLVM, a "translation unit" (effectively, a `Crate` in Rust) can describe functions that + // are included-but-not-used. If (or when) Rust generates functions that are + // included-but-not-used, note that a dummy description for a function included-but-not-used + // in a Crate can be replaced by full description provided by a different Crate. The two kinds + // of descriptions play distinct roles in LLVM IR; therefore, assign them different names (by + // appending "u" to the end of the function record var name, to prevent `linkonce_odr` merging. + let func_record_var_name = + format!("__covrec_{:X}{}", func_name_hash, if is_used { "u" } else { "" }); + debug!("function record var name: {:?}", func_record_var_name); + + let func_record_section_name = llvm::build_string(|s| unsafe { + llvm::LLVMRustCoverageWriteFuncSectionNameToString(cx.llmod, s); + }) + .expect("Rust Coverage function record section name failed UTF-8 conversion"); + debug!("function record section name: {:?}", func_record_section_name); + + let llglobal = llvm::add_global(cx.llmod, cx.val_ty(func_record_val), &func_record_var_name); + llvm::set_initializer(llglobal, func_record_val); + llvm::set_global_constant(llglobal, true); + llvm::set_linkage(llglobal, llvm::Linkage::LinkOnceODRLinkage); + llvm::set_visibility(llglobal, llvm::Visibility::Hidden); + llvm::set_section(llglobal, &func_record_section_name); + llvm::set_alignment(llglobal, VAR_ALIGN_BYTES); + llvm::set_comdat(cx.llmod, llglobal, &func_record_var_name); + cx.add_used_global(llglobal); +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs new file mode 100644 index 000000000..99e4ded62 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/create_scope_map.rs @@ -0,0 +1,126 @@ +use super::metadata::file_metadata; +use super::utils::DIB; +use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext}; +use rustc_codegen_ssa::traits::*; + +use crate::common::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::{DILocation, DIScope}; +use rustc_middle::mir::{Body, SourceScope}; +use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::{self, Instance}; +use rustc_session::config::DebugInfo; + +use rustc_index::bit_set::BitSet; +use rustc_index::vec::Idx; + +/// Produces DIScope DIEs for each MIR Scope which has variables defined in it. +// FIXME(eddyb) almost all of this should be in `rustc_codegen_ssa::mir::debuginfo`. +pub fn compute_mir_scopes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, +) { + // Find all scopes with variables defined in them. + let variables = if cx.sess().opts.debuginfo == DebugInfo::Full { + let mut vars = BitSet::new_empty(mir.source_scopes.len()); + // FIXME(eddyb) take into account that arguments always have debuginfo, + // irrespective of their name (assuming full debuginfo is enabled). + // NOTE(eddyb) actually, on second thought, those are always in the + // function scope, which always exists. + for var_debug_info in &mir.var_debug_info { + vars.insert(var_debug_info.source_info.scope); + } + Some(vars) + } else { + // Nothing to emit, of course. + None + }; + let mut instantiated = BitSet::new_empty(mir.source_scopes.len()); + // Instantiate all scopes. + for idx in 0..mir.source_scopes.len() { + let scope = SourceScope::new(idx); + make_mir_scope(cx, instance, mir, &variables, debug_context, &mut instantiated, scope); + } + assert!(instantiated.count() == mir.source_scopes.len()); +} + +fn make_mir_scope<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + mir: &Body<'tcx>, + variables: &Option>, + debug_context: &mut FunctionDebugContext<&'ll DIScope, &'ll DILocation>, + instantiated: &mut BitSet, + scope: SourceScope, +) { + if instantiated.contains(scope) { + return; + } + + let scope_data = &mir.source_scopes[scope]; + let parent_scope = if let Some(parent) = scope_data.parent_scope { + make_mir_scope(cx, instance, mir, variables, debug_context, instantiated, parent); + debug_context.scopes[parent] + } else { + // The root is the function itself. + let loc = cx.lookup_debug_loc(mir.span.lo()); + debug_context.scopes[scope] = DebugScope { + file_start_pos: loc.file.start_pos, + file_end_pos: loc.file.end_pos, + ..debug_context.scopes[scope] + }; + instantiated.insert(scope); + return; + }; + + if let Some(vars) = variables && !vars.contains(scope) && scope_data.inlined.is_none() { + // Do not create a DIScope if there are no variables defined in this + // MIR `SourceScope`, and it's not `inlined`, to avoid debuginfo bloat. + debug_context.scopes[scope] = parent_scope; + instantiated.insert(scope); + return; + } + + let loc = cx.lookup_debug_loc(scope_data.span.lo()); + let file_metadata = file_metadata(cx, &loc.file); + + let dbg_scope = match scope_data.inlined { + Some((callee, _)) => { + // FIXME(eddyb) this would be `self.monomorphize(&callee)` + // if this is moved to `rustc_codegen_ssa::mir::debuginfo`. + let callee = cx.tcx.subst_and_normalize_erasing_regions( + instance.substs, + ty::ParamEnv::reveal_all(), + callee, + ); + let callee_fn_abi = cx.fn_abi_of_instance(callee, ty::List::empty()); + cx.dbg_scope_fn(callee, callee_fn_abi, None) + } + None => unsafe { + llvm::LLVMRustDIBuilderCreateLexicalBlock( + DIB(cx), + parent_scope.dbg_scope, + file_metadata, + loc.line, + loc.col, + ) + }, + }; + + let inlined_at = scope_data.inlined.map(|(_, callsite_span)| { + // FIXME(eddyb) this doesn't account for the macro-related + // `Span` fixups that `rustc_codegen_ssa::mir::debuginfo` does. + let callsite_scope = parent_scope.adjust_dbg_scope_for_span(cx, callsite_span); + cx.dbg_loc(callsite_scope, parent_scope.inlined_at, callsite_span) + }); + + debug_context.scopes[scope] = DebugScope { + dbg_scope, + inlined_at: inlined_at.or(parent_scope.inlined_at), + file_start_pos: loc.file.start_pos, + file_end_pos: loc.file.end_pos, + }; + instantiated.insert(scope); +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/doc.md b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md new file mode 100644 index 000000000..aaec4e68c --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/doc.md @@ -0,0 +1,131 @@ +# Debug Info Module + +This module serves the purpose of generating debug symbols. We use LLVM's +[source level debugging](https://llvm.org/docs/SourceLevelDebugging.html) +features for generating the debug information. The general principle is +this: + +Given the right metadata in the LLVM IR, the LLVM code generator is able to +create DWARF debug symbols for the given code. The +[metadata](https://llvm.org/docs/LangRef.html#metadata-type) is structured +much like DWARF *debugging information entries* (DIE), representing type +information such as datatype layout, function signatures, block layout, +variable location and scope information, etc. It is the purpose of this +module to generate correct metadata and insert it into the LLVM IR. + +As the exact format of metadata trees may change between different LLVM +versions, we now use LLVM +[DIBuilder](https://llvm.org/docs/doxygen/html/classllvm_1_1DIBuilder.html) +to create metadata where possible. This will hopefully ease the adaption of +this module to future LLVM versions. + +The public API of the module is a set of functions that will insert the +correct metadata into the LLVM IR when called with the right parameters. +The module is thus driven from an outside client with functions like +`debuginfo::create_local_var_metadata(bx: block, local: &ast::local)`. + +Internally the module will try to reuse already created metadata by +utilizing a cache. The way to get a shared metadata node when needed is +thus to just call the corresponding function in this module: +```ignore (illustrative) +let file_metadata = file_metadata(cx, file); +``` +The function will take care of probing the cache for an existing node for +that exact file path. + +All private state used by the module is stored within either the +CodegenUnitDebugContext struct (owned by the CodegenCx) or the +FunctionDebugContext (owned by the FunctionCx). + +This file consists of three conceptual sections: +1. The public interface of the module +2. Module-internal metadata creation functions +3. Minor utility functions + + +## Recursive Types + +Some kinds of types, such as structs and enums can be recursive. That means +that the type definition of some type X refers to some other type which in +turn (transitively) refers to X. This introduces cycles into the type +referral graph. A naive algorithm doing an on-demand, depth-first traversal +of this graph when describing types, can get trapped in an endless loop +when it reaches such a cycle. + +For example, the following simple type for a singly-linked list... + +``` +struct List { + value: i32, + tail: Option>, +} +``` + +will generate the following callstack with a naive DFS algorithm: + +```ignore (illustrative) +describe(t = List) + describe(t = i32) + describe(t = Option>) + describe(t = Box) + describe(t = List) // at the beginning again... + ... +``` + +To break cycles like these, we use "stubs". That is, when +the algorithm encounters a possibly recursive type (any struct or enum), it +immediately creates a type description node and inserts it into the cache +*before* describing the members of the type. This type description is just +a stub (as type members are not described and added to it yet) but it +allows the algorithm to already refer to the type. After the stub is +inserted into the cache, the algorithm continues as before. If it now +encounters a recursive reference, it will hit the cache and does not try to +describe the type anew. This behavior is encapsulated in the +`type_map::build_type_with_children()` function. + + +## Source Locations and Line Information + +In addition to data type descriptions the debugging information must also +allow to map machine code locations back to source code locations in order +to be useful. This functionality is also handled in this module. The +following functions allow to control source mappings: + ++ `set_source_location()` ++ `clear_source_location()` ++ `start_emitting_source_locations()` + +`set_source_location()` allows to set the current source location. All IR +instructions created after a call to this function will be linked to the +given source location, until another location is specified with +`set_source_location()` or the source location is cleared with +`clear_source_location()`. In the later case, subsequent IR instruction +will not be linked to any source location. As you can see, this is a +stateful API (mimicking the one in LLVM), so be careful with source +locations set by previous calls. It's probably best to not rely on any +specific state being present at a given point in code. + +One topic that deserves some extra attention is *function prologues*. At +the beginning of a function's machine code there are typically a few +instructions for loading argument values into allocas and checking if +there's enough stack space for the function to execute. This *prologue* is +not visible in the source code and LLVM puts a special PROLOGUE END marker +into the line table at the first non-prologue instruction of the function. +In order to find out where the prologue ends, LLVM looks for the first +instruction in the function body that is linked to a source location. So, +when generating prologue instructions we have to make sure that we don't +emit source location information until the 'real' function body begins. For +this reason, source location emission is disabled by default for any new +function being codegened and is only activated after a call to the third +function from the list above, `start_emitting_source_locations()`. This +function should be called right before regularly starting to codegen the +top-level block of the given function. + +There is one exception to the above rule: `llvm.dbg.declare` instruction +must be linked to the source location of the variable being declared. For +function parameters these `llvm.dbg.declare` instructions typically occur +in the middle of the prologue, however, they are ignored by LLVM's prologue +detection. The `create_argument_metadata()` and related functions take care +of linking the `llvm.dbg.declare` instructions to the correct source +locations even while source location emission is still disabled, so there +is no need to do anything special with source location handling here. diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs new file mode 100644 index 000000000..80fd9726f --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/gdb.rs @@ -0,0 +1,120 @@ +// .debug_gdb_scripts binary section. + +use crate::llvm; + +use crate::builder::Builder; +use crate::common::CodegenCx; +use crate::value::Value; +use rustc_codegen_ssa::base::collect_debugger_visualizers_transitive; +use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::LOCAL_CRATE; +use rustc_middle::bug; +use rustc_session::config::{CrateType, DebugInfo}; + +use rustc_span::symbol::sym; +use rustc_span::DebuggerVisualizerType; + +/// Inserts a side-effect free instruction sequence that makes sure that the +/// .debug_gdb_scripts global is referenced, so it isn't removed by the linker. +pub fn insert_reference_to_gdb_debug_scripts_section_global(bx: &mut Builder<'_, '_, '_>) { + if needs_gdb_debug_scripts_section(bx) { + let gdb_debug_scripts_section = + bx.const_bitcast(get_or_insert_gdb_debug_scripts_section_global(bx), bx.type_i8p()); + // Load just the first byte as that's all that's necessary to force + // LLVM to keep around the reference to the global. + let volative_load_instruction = bx.volatile_load(bx.type_i8(), gdb_debug_scripts_section); + unsafe { + llvm::LLVMSetAlignment(volative_load_instruction, 1); + } + } +} + +/// Allocates the global variable responsible for the .debug_gdb_scripts binary +/// section. +pub fn get_or_insert_gdb_debug_scripts_section_global<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll Value { + let c_section_var_name = "__rustc_debug_gdb_scripts_section__\0"; + let section_var_name = &c_section_var_name[..c_section_var_name.len() - 1]; + + let section_var = + unsafe { llvm::LLVMGetNamedGlobal(cx.llmod, c_section_var_name.as_ptr().cast()) }; + + section_var.unwrap_or_else(|| { + let section_name = b".debug_gdb_scripts\0"; + let mut section_contents = Vec::new(); + + // Add the pretty printers for the standard library first. + section_contents.extend_from_slice(b"\x01gdb_load_rust_pretty_printers.py\0"); + + // Next, add the pretty printers that were specified via the `#[debugger_visualizer]` attribute. + let visualizers = collect_debugger_visualizers_transitive( + cx.tcx, + DebuggerVisualizerType::GdbPrettyPrinter, + ); + let crate_name = cx.tcx.crate_name(LOCAL_CRATE); + for (index, visualizer) in visualizers.iter().enumerate() { + // The initial byte `4` instructs GDB that the following pretty printer + // is defined inline as opposed to in a standalone file. + section_contents.extend_from_slice(b"\x04"); + let vis_name = format!("pretty-printer-{}-{}\n", crate_name, index); + section_contents.extend_from_slice(vis_name.as_bytes()); + section_contents.extend_from_slice(&visualizer.src); + + // The final byte `0` tells GDB that the pretty printer has been + // fully defined and can continue searching for additional + // pretty printers. + section_contents.extend_from_slice(b"\0"); + } + + unsafe { + let section_contents = section_contents.as_slice(); + let llvm_type = cx.type_array(cx.type_i8(), section_contents.len() as u64); + + let section_var = cx + .define_global(section_var_name, llvm_type) + .unwrap_or_else(|| bug!("symbol `{}` is already defined", section_var_name)); + llvm::LLVMSetSection(section_var, section_name.as_ptr().cast()); + llvm::LLVMSetInitializer(section_var, cx.const_bytes(section_contents)); + llvm::LLVMSetGlobalConstant(section_var, llvm::True); + llvm::LLVMSetUnnamedAddress(section_var, llvm::UnnamedAddr::Global); + llvm::LLVMRustSetLinkage(section_var, llvm::Linkage::LinkOnceODRLinkage); + // This should make sure that the whole section is not larger than + // the string it contains. Otherwise we get a warning from GDB. + llvm::LLVMSetAlignment(section_var, 1); + section_var + } + }) +} + +pub fn needs_gdb_debug_scripts_section(cx: &CodegenCx<'_, '_>) -> bool { + let omit_gdb_pretty_printer_section = + cx.tcx.sess.contains_name(cx.tcx.hir().krate_attrs(), sym::omit_gdb_pretty_printer_section); + + // To ensure the section `__rustc_debug_gdb_scripts_section__` will not create + // ODR violations at link time, this section will not be emitted for rlibs since + // each rlib could produce a different set of visualizers that would be embedded + // in the `.debug_gdb_scripts` section. For that reason, we make sure that the + // section is only emitted for leaf crates. + let embed_visualizers = cx.sess().crate_types().iter().any(|&crate_type| match crate_type { + CrateType::Executable | CrateType::Dylib | CrateType::Cdylib | CrateType::Staticlib => { + // These are crate types for which we will embed pretty printers since they + // are treated as leaf crates. + true + } + CrateType::ProcMacro => { + // We could embed pretty printers for proc macro crates too but it does not + // seem like a good default, since this is a rare use case and we don't + // want to slow down the common case. + false + } + CrateType::Rlib => { + // As per the above description, embedding pretty printers for rlibs could + // lead to ODR violations so we skip this crate type as well. + false + } + }); + + !omit_gdb_pretty_printer_section + && cx.sess().opts.debuginfo != DebugInfo::None + && cx.sess().target.emit_debug_gdb_scripts + && embed_visualizers +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs new file mode 100644 index 000000000..bd84100e0 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata.rs @@ -0,0 +1,1618 @@ +use self::type_map::DINodeCreationResult; +use self::type_map::Stub; +use self::type_map::UniqueTypeId; + +use super::namespace::mangled_name_of_instance; +use super::type_names::{compute_debuginfo_type_name, compute_debuginfo_vtable_name}; +use super::utils::{ + create_DIArray, debug_context, get_namespace_for_item, is_node_local_to_unit, DIB, +}; +use super::CodegenUnitDebugContext; + +use crate::abi; +use crate::common::CodegenCx; +use crate::debuginfo::metadata::type_map::build_type_with_children; +use crate::debuginfo::utils::fat_pointer_kind; +use crate::debuginfo::utils::FatPtrKind; +use crate::llvm; +use crate::llvm::debuginfo::{ + DIDescriptor, DIFile, DIFlags, DILexicalBlock, DIScope, DIType, DebugEmissionKind, +}; +use crate::value::Value; + +use cstr::cstr; +use rustc_codegen_ssa::debuginfo::type_names::cpp_like_debuginfo; +use rustc_codegen_ssa::debuginfo::type_names::VTableNameKind; +use rustc_codegen_ssa::traits::*; +use rustc_fs_util::path_to_c_string; +use rustc_hir::def::CtorKind; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +use rustc_index::vec::{Idx, IndexVec}; +use rustc_middle::bug; +use rustc_middle::mir::{self, GeneratorLayout}; +use rustc_middle::ty::layout::{LayoutOf, TyAndLayout}; +use rustc_middle::ty::subst::GenericArgKind; +use rustc_middle::ty::{ + self, AdtKind, Instance, ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt, Visibility, +}; +use rustc_session::config::{self, DebugInfo, Lto}; +use rustc_span::symbol::Symbol; +use rustc_span::FileName; +use rustc_span::{self, FileNameDisplayPreference, SourceFile}; +use rustc_symbol_mangling::typeid_for_trait_ref; +use rustc_target::abi::{Align, Size}; +use smallvec::smallvec; +use tracing::debug; + +use libc::{c_char, c_longlong, c_uint}; +use std::borrow::Cow; +use std::fmt::{self, Write}; +use std::hash::{Hash, Hasher}; +use std::iter; +use std::path::{Path, PathBuf}; +use std::ptr; +use tracing::instrument; + +impl PartialEq for llvm::Metadata { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self, other) + } +} + +impl Eq for llvm::Metadata {} + +impl Hash for llvm::Metadata { + fn hash(&self, hasher: &mut H) { + (self as *const Self).hash(hasher); + } +} + +impl fmt::Debug for llvm::Metadata { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + (self as *const Self).fmt(f) + } +} + +// From DWARF 5. +// See http://www.dwarfstd.org/ShowIssue.php?issue=140129.1. +const DW_LANG_RUST: c_uint = 0x1c; +#[allow(non_upper_case_globals)] +const DW_ATE_boolean: c_uint = 0x02; +#[allow(non_upper_case_globals)] +const DW_ATE_float: c_uint = 0x04; +#[allow(non_upper_case_globals)] +const DW_ATE_signed: c_uint = 0x05; +#[allow(non_upper_case_globals)] +const DW_ATE_unsigned: c_uint = 0x07; +#[allow(non_upper_case_globals)] +const DW_ATE_UTF: c_uint = 0x10; + +pub(super) const UNKNOWN_LINE_NUMBER: c_uint = 0; +pub(super) const UNKNOWN_COLUMN_NUMBER: c_uint = 0; + +const NO_SCOPE_METADATA: Option<&DIScope> = None; +/// A function that returns an empty list of generic parameter debuginfo nodes. +const NO_GENERICS: for<'ll> fn(&CodegenCx<'ll, '_>) -> SmallVec<&'ll DIType> = |_| SmallVec::new(); + +// SmallVec is used quite a bit in this module, so create a shorthand. +// The actual number of elements is not so important. +pub type SmallVec = smallvec::SmallVec<[T; 16]>; + +mod enums; +mod type_map; + +pub(crate) use type_map::TypeMap; + +/// Returns from the enclosing function if the type debuginfo node with the given +/// unique ID can be found in the type map. +macro_rules! return_if_di_node_created_in_meantime { + ($cx: expr, $unique_type_id: expr) => { + if let Some(di_node) = debug_context($cx).type_map.di_node_for_unique_id($unique_type_id) { + return DINodeCreationResult::new(di_node, true); + } + }; +} + +/// Extract size and alignment from a TyAndLayout. +fn size_and_align_of<'tcx>(ty_and_layout: TyAndLayout<'tcx>) -> (Size, Align) { + (ty_and_layout.size, ty_and_layout.align.abi) +} + +/// Creates debuginfo for a fixed size array (e.g. `[u64; 123]`). +/// For slices (that is, "arrays" of unknown size) use [build_slice_type_di_node]. +fn build_fixed_size_array_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, + array_type: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + let ty::Array(element_type, len) = array_type.kind() else { + bug!("build_fixed_size_array_di_node() called with non-ty::Array type `{:?}`", array_type) + }; + + let element_type_di_node = type_di_node(cx, *element_type); + + return_if_di_node_created_in_meantime!(cx, unique_type_id); + + let (size, align) = cx.size_and_align_of(array_type); + + let upper_bound = len.eval_usize(cx.tcx, ty::ParamEnv::reveal_all()) as c_longlong; + + let subrange = + unsafe { Some(llvm::LLVMRustDIBuilderGetOrCreateSubrange(DIB(cx), 0, upper_bound)) }; + + let subscripts = create_DIArray(DIB(cx), &[subrange]); + let di_node = unsafe { + llvm::LLVMRustDIBuilderCreateArrayType( + DIB(cx), + size.bits(), + align.bits() as u32, + element_type_di_node, + subscripts, + ) + }; + + DINodeCreationResult::new(di_node, false) +} + +/// Creates debuginfo for built-in pointer-like things: +/// +/// - ty::Ref +/// - ty::RawPtr +/// - ty::Adt in the case it's Box +/// +/// At some point we might want to remove the special handling of Box +/// and treat it the same as other smart pointers (like Rc, Arc, ...). +fn build_pointer_or_reference_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ptr_type: Ty<'tcx>, + pointee_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + // The debuginfo generated by this function is only valid if `ptr_type` is really just + // a (fat) pointer. Make sure it is not called for e.g. `Box`. + debug_assert_eq!( + cx.size_and_align_of(ptr_type), + cx.size_and_align_of(cx.tcx.mk_mut_ptr(pointee_type)) + ); + + let pointee_type_di_node = type_di_node(cx, pointee_type); + + return_if_di_node_created_in_meantime!(cx, unique_type_id); + + let (thin_pointer_size, thin_pointer_align) = + cx.size_and_align_of(cx.tcx.mk_imm_ptr(cx.tcx.types.unit)); + let ptr_type_debuginfo_name = compute_debuginfo_type_name(cx.tcx, ptr_type, true); + + match fat_pointer_kind(cx, pointee_type) { + None => { + // This is a thin pointer. Create a regular pointer type and give it the correct name. + debug_assert_eq!( + (thin_pointer_size, thin_pointer_align), + cx.size_and_align_of(ptr_type), + "ptr_type={}, pointee_type={}", + ptr_type, + pointee_type, + ); + + let di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + pointee_type_di_node, + thin_pointer_size.bits(), + thin_pointer_align.bits() as u32, + 0, // Ignore DWARF address space. + ptr_type_debuginfo_name.as_ptr().cast(), + ptr_type_debuginfo_name.len(), + ) + }; + + DINodeCreationResult { di_node, already_stored_in_typemap: false } + } + Some(fat_pointer_kind) => { + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &ptr_type_debuginfo_name, + cx.size_and_align_of(ptr_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, owner| { + // FIXME: If this fat pointer is a `Box` then we don't want to use its + // type layout and instead use the layout of the raw pointer inside + // of it. + // The proper way to handle this is to not treat Box as a pointer + // at all and instead emit regular struct debuginfo for it. We just + // need to make sure that we don't break existing debuginfo consumers + // by doing that (at least not without a warning period). + let layout_type = + if ptr_type.is_box() { cx.tcx.mk_mut_ptr(pointee_type) } else { ptr_type }; + + let layout = cx.layout_of(layout_type); + let addr_field = layout.field(cx, abi::FAT_PTR_ADDR); + let extra_field = layout.field(cx, abi::FAT_PTR_EXTRA); + + let (addr_field_name, extra_field_name) = match fat_pointer_kind { + FatPtrKind::Dyn => ("pointer", "vtable"), + FatPtrKind::Slice => ("data_ptr", "length"), + }; + + debug_assert_eq!(abi::FAT_PTR_ADDR, 0); + debug_assert_eq!(abi::FAT_PTR_EXTRA, 1); + + // The data pointer type is a regular, thin pointer, regardless of whether this + // is a slice or a trait object. + let data_ptr_type_di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + pointee_type_di_node, + addr_field.size.bits(), + addr_field.align.abi.bits() as u32, + 0, // Ignore DWARF address space. + std::ptr::null(), + 0, + ) + }; + + smallvec![ + build_field_di_node( + cx, + owner, + addr_field_name, + (addr_field.size, addr_field.align.abi), + layout.fields.offset(abi::FAT_PTR_ADDR), + DIFlags::FlagZero, + data_ptr_type_di_node, + ), + build_field_di_node( + cx, + owner, + extra_field_name, + (extra_field.size, extra_field.align.abi), + layout.fields.offset(abi::FAT_PTR_EXTRA), + DIFlags::FlagZero, + type_di_node(cx, extra_field.ty), + ), + ] + }, + NO_GENERICS, + ) + } + } +} + +fn build_subroutine_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + // It's possible to create a self-referential + // type in Rust by using 'impl trait': + // + // fn foo() -> impl Copy { foo } + // + // Unfortunately LLVM's API does not allow us to create recursive subroutine types. + // In order to work around that restriction we place a marker type in the type map, + // before creating the actual type. If the actual type is recursive, it will hit the + // marker type. So we end up with a type that looks like + // + // fn foo() -> + // + // Once that is created, we replace the marker in the typemap with the actual type. + debug_context(cx) + .type_map + .unique_id_to_di_node + .borrow_mut() + .insert(unique_type_id, recursion_marker_type_di_node(cx)); + + let fn_ty = unique_type_id.expect_ty(); + let signature = cx + .tcx + .normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), fn_ty.fn_sig(cx.tcx)); + + let signature_di_nodes: SmallVec<_> = iter::once( + // return type + match signature.output().kind() { + ty::Tuple(tys) if tys.is_empty() => { + // this is a "void" function + None + } + _ => Some(type_di_node(cx, signature.output())), + }, + ) + .chain( + // regular arguments + signature.inputs().iter().map(|&argument_type| Some(type_di_node(cx, argument_type))), + ) + .collect(); + + debug_context(cx).type_map.unique_id_to_di_node.borrow_mut().remove(&unique_type_id); + + let fn_di_node = unsafe { + llvm::LLVMRustDIBuilderCreateSubroutineType( + DIB(cx), + create_DIArray(DIB(cx), &signature_di_nodes[..]), + ) + }; + + // This is actually a function pointer, so wrap it in pointer DI. + let name = compute_debuginfo_type_name(cx.tcx, fn_ty, false); + let di_node = unsafe { + llvm::LLVMRustDIBuilderCreatePointerType( + DIB(cx), + fn_di_node, + cx.tcx.data_layout.pointer_size.bits(), + cx.tcx.data_layout.pointer_align.abi.bits() as u32, + 0, // Ignore DWARF address space. + name.as_ptr().cast(), + name.len(), + ) + }; + + DINodeCreationResult::new(di_node, false) +} + +/// Create debuginfo for `dyn SomeTrait` types. Currently these are empty structs +/// we with the correct type name (e.g. "dyn SomeTrait + Sync"). +fn build_dyn_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + dyn_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + if let ty::Dynamic(..) = dyn_type.kind() { + let type_name = compute_debuginfo_type_name(cx.tcx, dyn_type, true); + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + cx.size_and_align_of(dyn_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |_, _| smallvec![], + NO_GENERICS, + ) + } else { + bug!( + "Only ty::Dynamic is valid for build_dyn_type_di_node(). Found {:?} instead.", + dyn_type + ) + } +} + +/// Create debuginfo for `[T]` and `str`. These are unsized. +/// +/// NOTE: We currently emit just emit the debuginfo for the element type here +/// (i.e. `T` for slices and `u8` for `str`), so that we end up with +/// `*const T` for the `data_ptr` field of the corresponding fat-pointer +/// debuginfo of `&[T]`. +/// +/// It would be preferable and more accurate if we emitted a DIArray of T +/// without an upper bound instead. That is, LLVM already supports emitting +/// debuginfo of arrays of unknown size. But GDB currently seems to end up +/// in an infinite loop when confronted with such a type. +/// +/// As a side effect of the current encoding every instance of a type like +/// `struct Foo { unsized_field: [u8] }` will look like +/// `struct Foo { unsized_field: u8 }` in debuginfo. If the length of the +/// slice is zero, then accessing `unsized_field` in the debugger would +/// result in an out-of-bounds access. +fn build_slice_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + slice_type: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let element_type = match slice_type.kind() { + ty::Slice(element_type) => *element_type, + ty::Str => cx.tcx.types.u8, + _ => { + bug!( + "Only ty::Slice is valid for build_slice_type_di_node(). Found {:?} instead.", + slice_type + ) + } + }; + + let element_type_di_node = type_di_node(cx, element_type); + return_if_di_node_created_in_meantime!(cx, unique_type_id); + DINodeCreationResult { di_node: element_type_di_node, already_stored_in_typemap: false } +} + +/// Get the debuginfo node for the given type. +/// +/// This function will look up the debuginfo node in the TypeMap. If it can't find it, it +/// will create the node by dispatching to the corresponding `build_*_di_node()` function. +pub fn type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>, t: Ty<'tcx>) -> &'ll DIType { + let unique_type_id = UniqueTypeId::for_ty(cx.tcx, t); + + if let Some(existing_di_node) = debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) + { + return existing_di_node; + } + + debug!("type_di_node: {:?}", t); + + let DINodeCreationResult { di_node, already_stored_in_typemap } = match *t.kind() { + ty::Never | ty::Bool | ty::Char | ty::Int(_) | ty::Uint(_) | ty::Float(_) => { + build_basic_type_di_node(cx, t) + } + ty::Tuple(elements) if elements.is_empty() => build_basic_type_di_node(cx, t), + ty::Array(..) => build_fixed_size_array_di_node(cx, unique_type_id, t), + ty::Slice(_) | ty::Str => build_slice_type_di_node(cx, t, unique_type_id), + ty::Dynamic(..) => build_dyn_type_di_node(cx, t, unique_type_id), + ty::Foreign(..) => build_foreign_type_di_node(cx, t, unique_type_id), + ty::RawPtr(ty::TypeAndMut { ty: pointee_type, .. }) | ty::Ref(_, pointee_type, _) => { + build_pointer_or_reference_di_node(cx, t, pointee_type, unique_type_id) + } + // Box may have a non-ZST allocator A. In that case, we + // cannot treat Box as just an owned alias of `*mut T`. + ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { + build_pointer_or_reference_di_node(cx, t, t.boxed_ty(), unique_type_id) + } + ty::FnDef(..) | ty::FnPtr(_) => build_subroutine_type_di_node(cx, unique_type_id), + ty::Closure(..) => build_closure_env_di_node(cx, unique_type_id), + ty::Generator(..) => enums::build_generator_di_node(cx, unique_type_id), + ty::Adt(def, ..) => match def.adt_kind() { + AdtKind::Struct => build_struct_type_di_node(cx, unique_type_id), + AdtKind::Union => build_union_type_di_node(cx, unique_type_id), + AdtKind::Enum => enums::build_enum_type_di_node(cx, unique_type_id), + }, + ty::Tuple(_) => build_tuple_type_di_node(cx, unique_type_id), + // Type parameters from polymorphized functions. + ty::Param(_) => build_param_type_di_node(cx, t), + _ => bug!("debuginfo: unexpected type in type_di_node(): {:?}", t), + }; + + { + if already_stored_in_typemap { + // Make sure that we really do have a `TypeMap` entry for the unique type ID. + let di_node_for_uid = + match debug_context(cx).type_map.di_node_for_unique_id(unique_type_id) { + Some(di_node) => di_node, + None => { + bug!( + "expected type debuginfo node for unique \ + type ID '{:?}' to already be in \ + the `debuginfo::TypeMap` but it \ + was not.", + unique_type_id, + ); + } + }; + + debug_assert_eq!(di_node_for_uid as *const _, di_node as *const _); + } else { + debug_context(cx).type_map.insert(unique_type_id, di_node); + } + } + + di_node +} + +// FIXME(mw): Cache this via a regular UniqueTypeId instead of an extra field in the debug context. +fn recursion_marker_type_di_node<'ll, 'tcx>(cx: &CodegenCx<'ll, 'tcx>) -> &'ll DIType { + *debug_context(cx).recursion_marker_type.get_or_init(move || { + unsafe { + // The choice of type here is pretty arbitrary - + // anything reading the debuginfo for a recursive + // type is going to see *something* weird - the only + // question is what exactly it will see. + // + // FIXME: the name `` does not fit the naming scheme + // of other types. + // + // FIXME: it might make sense to use an actual pointer type here + // so that debuggers can show the address. + let name = ""; + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_ptr().cast(), + name.len(), + cx.tcx.data_layout.pointer_size.bits(), + DW_ATE_unsigned, + ) + } + }) +} + +fn hex_encode(data: &[u8]) -> String { + let mut hex_string = String::with_capacity(data.len() * 2); + for byte in data.iter() { + write!(&mut hex_string, "{:02x}", byte).unwrap(); + } + hex_string +} + +pub fn file_metadata<'ll>(cx: &CodegenCx<'ll, '_>, source_file: &SourceFile) -> &'ll DIFile { + let cache_key = Some((source_file.name_hash, source_file.src_hash)); + return debug_context(cx) + .created_files + .borrow_mut() + .entry(cache_key) + .or_insert_with(|| alloc_new_file_metadata(cx, source_file)); + + #[instrument(skip(cx, source_file), level = "debug")] + fn alloc_new_file_metadata<'ll>( + cx: &CodegenCx<'ll, '_>, + source_file: &SourceFile, + ) -> &'ll DIFile { + debug!(?source_file.name); + + let (directory, file_name) = match &source_file.name { + FileName::Real(filename) => { + let working_directory = &cx.sess().opts.working_dir; + debug!(?working_directory); + + let filename = cx + .sess() + .source_map() + .path_mapping() + .to_embeddable_absolute_path(filename.clone(), working_directory); + + // Construct the absolute path of the file + let abs_path = filename.remapped_path_if_available(); + debug!(?abs_path); + + if let Ok(rel_path) = + abs_path.strip_prefix(working_directory.remapped_path_if_available()) + { + // If the compiler's working directory (which also is the DW_AT_comp_dir of + // the compilation unit) is a prefix of the path we are about to emit, then + // only emit the part relative to the working directory. + // Because of path remapping we sometimes see strange things here: `abs_path` + // might actually look like a relative path + // (e.g. `/src/lib.rs`), so if we emit it without + // taking the working directory into account, downstream tooling will + // interpret it as `//src/lib.rs`, + // which makes no sense. Usually in such cases the working directory will also + // be remapped to `` or some other prefix of the path + // we are remapping, so we end up with + // `//src/lib.rs`. + // By moving the working directory portion into the `directory` part of the + // DIFile, we allow LLVM to emit just the relative path for DWARF, while + // still emitting the correct absolute path for CodeView. + ( + working_directory.to_string_lossy(FileNameDisplayPreference::Remapped), + rel_path.to_string_lossy().into_owned(), + ) + } else { + ("".into(), abs_path.to_string_lossy().into_owned()) + } + } + other => ("".into(), other.prefer_remapped().to_string_lossy().into_owned()), + }; + + let hash_kind = match source_file.src_hash.kind { + rustc_span::SourceFileHashAlgorithm::Md5 => llvm::ChecksumKind::MD5, + rustc_span::SourceFileHashAlgorithm::Sha1 => llvm::ChecksumKind::SHA1, + rustc_span::SourceFileHashAlgorithm::Sha256 => llvm::ChecksumKind::SHA256, + }; + let hash_value = hex_encode(source_file.src_hash.hash_bytes()); + + unsafe { + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_ptr().cast(), + file_name.len(), + directory.as_ptr().cast(), + directory.len(), + hash_kind, + hash_value.as_ptr().cast(), + hash_value.len(), + ) + } + } +} + +pub fn unknown_file_metadata<'ll>(cx: &CodegenCx<'ll, '_>) -> &'ll DIFile { + debug_context(cx).created_files.borrow_mut().entry(None).or_insert_with(|| unsafe { + let file_name = ""; + let directory = ""; + let hash_value = ""; + + llvm::LLVMRustDIBuilderCreateFile( + DIB(cx), + file_name.as_ptr().cast(), + file_name.len(), + directory.as_ptr().cast(), + directory.len(), + llvm::ChecksumKind::None, + hash_value.as_ptr().cast(), + hash_value.len(), + ) + }) +} + +trait MsvcBasicName { + fn msvc_basic_name(self) -> &'static str; +} + +impl MsvcBasicName for ty::IntTy { + fn msvc_basic_name(self) -> &'static str { + match self { + ty::IntTy::Isize => "ptrdiff_t", + ty::IntTy::I8 => "__int8", + ty::IntTy::I16 => "__int16", + ty::IntTy::I32 => "__int32", + ty::IntTy::I64 => "__int64", + ty::IntTy::I128 => "__int128", + } + } +} + +impl MsvcBasicName for ty::UintTy { + fn msvc_basic_name(self) -> &'static str { + match self { + ty::UintTy::Usize => "size_t", + ty::UintTy::U8 => "unsigned __int8", + ty::UintTy::U16 => "unsigned __int16", + ty::UintTy::U32 => "unsigned __int32", + ty::UintTy::U64 => "unsigned __int64", + ty::UintTy::U128 => "unsigned __int128", + } + } +} + +impl MsvcBasicName for ty::FloatTy { + fn msvc_basic_name(self) -> &'static str { + match self { + ty::FloatTy::F32 => "float", + ty::FloatTy::F64 => "double", + } + } +} + +fn build_basic_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_basic_type_di_node: {:?}", t); + + // When targeting MSVC, emit MSVC style type names for compatibility with + // .natvis visualizers (and perhaps other existing native debuggers?) + let cpp_like_debuginfo = cpp_like_debuginfo(cx.tcx); + + let (name, encoding) = match t.kind() { + ty::Never => ("!", DW_ATE_unsigned), + ty::Tuple(elements) if elements.is_empty() => { + if cpp_like_debuginfo { + return build_tuple_type_di_node(cx, UniqueTypeId::for_ty(cx.tcx, t)); + } else { + ("()", DW_ATE_unsigned) + } + } + ty::Bool => ("bool", DW_ATE_boolean), + ty::Char => ("char", DW_ATE_UTF), + ty::Int(int_ty) if cpp_like_debuginfo => (int_ty.msvc_basic_name(), DW_ATE_signed), + ty::Uint(uint_ty) if cpp_like_debuginfo => (uint_ty.msvc_basic_name(), DW_ATE_unsigned), + ty::Float(float_ty) if cpp_like_debuginfo => (float_ty.msvc_basic_name(), DW_ATE_float), + ty::Int(int_ty) => (int_ty.name_str(), DW_ATE_signed), + ty::Uint(uint_ty) => (uint_ty.name_str(), DW_ATE_unsigned), + ty::Float(float_ty) => (float_ty.name_str(), DW_ATE_float), + _ => bug!("debuginfo::build_basic_type_di_node - `t` is invalid type"), + }; + + let ty_di_node = unsafe { + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_ptr().cast(), + name.len(), + cx.size_of(t).bits(), + encoding, + ) + }; + + if !cpp_like_debuginfo { + return DINodeCreationResult::new(ty_di_node, false); + } + + let typedef_name = match t.kind() { + ty::Int(int_ty) => int_ty.name_str(), + ty::Uint(uint_ty) => uint_ty.name_str(), + ty::Float(float_ty) => float_ty.name_str(), + _ => return DINodeCreationResult::new(ty_di_node, false), + }; + + let typedef_di_node = unsafe { + llvm::LLVMRustDIBuilderCreateTypedef( + DIB(cx), + ty_di_node, + typedef_name.as_ptr().cast(), + typedef_name.len(), + unknown_file_metadata(cx), + 0, + None, + ) + }; + + DINodeCreationResult::new(typedef_di_node, false) +} + +fn build_foreign_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_foreign_type_di_node: {:?}", t); + + let &ty::Foreign(def_id) = unique_type_id.expect_ty().kind() else { + bug!("build_foreign_type_di_node() called with unexpected type: {:?}", unique_type_id.expect_ty()); + }; + + build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &compute_debuginfo_type_name(cx.tcx, t, false), + cx.size_and_align_of(t), + Some(get_namespace_for_item(cx, def_id)), + DIFlags::FlagZero, + ), + |_, _| smallvec![], + NO_GENERICS, + ) +} + +fn build_param_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + t: Ty<'tcx>, +) -> DINodeCreationResult<'ll> { + debug!("build_param_type_di_node: {:?}", t); + let name = format!("{:?}", t); + DINodeCreationResult { + di_node: unsafe { + llvm::LLVMRustDIBuilderCreateBasicType( + DIB(cx), + name.as_ptr().cast(), + name.len(), + Size::ZERO.bits(), + DW_ATE_unsigned, + ) + }, + already_stored_in_typemap: false, + } +} + +pub fn build_compile_unit_di_node<'ll, 'tcx>( + tcx: TyCtxt<'tcx>, + codegen_unit_name: &str, + debug_context: &CodegenUnitDebugContext<'ll, 'tcx>, +) -> &'ll DIDescriptor { + let mut name_in_debuginfo = match tcx.sess.local_crate_source_file { + Some(ref path) => path.clone(), + None => PathBuf::from(tcx.crate_name(LOCAL_CRATE).as_str()), + }; + + // To avoid breaking split DWARF, we need to ensure that each codegen unit + // has a unique `DW_AT_name`. This is because there's a remote chance that + // different codegen units for the same module will have entirely + // identical DWARF entries for the purpose of the DWO ID, which would + // violate Appendix F ("Split Dwarf Object Files") of the DWARF 5 + // specification. LLVM uses the algorithm specified in section 7.32 "Type + // Signature Computation" to compute the DWO ID, which does not include + // any fields that would distinguish compilation units. So we must embed + // the codegen unit name into the `DW_AT_name`. (Issue #88521.) + // + // Additionally, the OSX linker has an idiosyncrasy where it will ignore + // some debuginfo if multiple object files with the same `DW_AT_name` are + // linked together. + // + // As a workaround for these two issues, we generate unique names for each + // object file. Those do not correspond to an actual source file but that + // is harmless. + name_in_debuginfo.push("@"); + name_in_debuginfo.push(codegen_unit_name); + + debug!("build_compile_unit_di_node: {:?}", name_in_debuginfo); + let rustc_producer = + format!("rustc version {}", option_env!("CFG_VERSION").expect("CFG_VERSION"),); + // FIXME(#41252) Remove "clang LLVM" if we can get GDB and LLVM to play nice. + let producer = format!("clang LLVM ({})", rustc_producer); + + let name_in_debuginfo = name_in_debuginfo.to_string_lossy(); + let work_dir = tcx.sess.opts.working_dir.to_string_lossy(FileNameDisplayPreference::Remapped); + let flags = "\0"; + let output_filenames = tcx.output_filenames(()); + let split_name = if tcx.sess.target_can_use_split_dwarf() { + output_filenames + .split_dwarf_path( + tcx.sess.split_debuginfo(), + tcx.sess.opts.unstable_opts.split_dwarf_kind, + Some(codegen_unit_name), + ) + // We get a path relative to the working directory from split_dwarf_path + .map(|f| tcx.sess.source_map().path_mapping().map_prefix(f).0) + } else { + None + } + .unwrap_or_default(); + let split_name = split_name.to_str().unwrap(); + + // FIXME(#60020): + // + // This should actually be + // + // let kind = DebugEmissionKind::from_generic(tcx.sess.opts.debuginfo); + // + // That is, we should set LLVM's emission kind to `LineTablesOnly` if + // we are compiling with "limited" debuginfo. However, some of the + // existing tools relied on slightly more debuginfo being generated than + // would be the case with `LineTablesOnly`, and we did not want to break + // these tools in a "drive-by fix", without a good idea or plan about + // what limited debuginfo should exactly look like. So for now we keep + // the emission kind as `FullDebug`. + // + // See https://github.com/rust-lang/rust/issues/60020 for details. + let kind = DebugEmissionKind::FullDebug; + assert!(tcx.sess.opts.debuginfo != DebugInfo::None); + + unsafe { + let compile_unit_file = llvm::LLVMRustDIBuilderCreateFile( + debug_context.builder, + name_in_debuginfo.as_ptr().cast(), + name_in_debuginfo.len(), + work_dir.as_ptr().cast(), + work_dir.len(), + llvm::ChecksumKind::None, + ptr::null(), + 0, + ); + + let unit_metadata = llvm::LLVMRustDIBuilderCreateCompileUnit( + debug_context.builder, + DW_LANG_RUST, + compile_unit_file, + producer.as_ptr().cast(), + producer.len(), + tcx.sess.opts.optimize != config::OptLevel::No, + flags.as_ptr().cast(), + 0, + // NB: this doesn't actually have any perceptible effect, it seems. LLVM will instead + // put the path supplied to `MCSplitDwarfFile` into the debug info of the final + // output(s). + split_name.as_ptr().cast(), + split_name.len(), + kind, + 0, + tcx.sess.opts.unstable_opts.split_dwarf_inlining, + ); + + if tcx.sess.opts.unstable_opts.profile { + let cu_desc_metadata = + llvm::LLVMRustMetadataAsValue(debug_context.llcontext, unit_metadata); + let default_gcda_path = &output_filenames.with_extension("gcda"); + let gcda_path = + tcx.sess.opts.unstable_opts.profile_emit.as_ref().unwrap_or(default_gcda_path); + + let gcov_cu_info = [ + path_to_mdstring(debug_context.llcontext, &output_filenames.with_extension("gcno")), + path_to_mdstring(debug_context.llcontext, gcda_path), + cu_desc_metadata, + ]; + let gcov_metadata = llvm::LLVMMDNodeInContext( + debug_context.llcontext, + gcov_cu_info.as_ptr(), + gcov_cu_info.len() as c_uint, + ); + + let llvm_gcov_ident = cstr!("llvm.gcov"); + llvm::LLVMAddNamedMetadataOperand( + debug_context.llmod, + llvm_gcov_ident.as_ptr(), + gcov_metadata, + ); + } + + // Insert `llvm.ident` metadata on the wasm targets since that will + // get hooked up to the "producer" sections `processed-by` information. + if tcx.sess.target.is_like_wasm { + let name_metadata = llvm::LLVMMDStringInContext( + debug_context.llcontext, + rustc_producer.as_ptr().cast(), + rustc_producer.as_bytes().len() as c_uint, + ); + llvm::LLVMAddNamedMetadataOperand( + debug_context.llmod, + cstr!("llvm.ident").as_ptr(), + llvm::LLVMMDNodeInContext(debug_context.llcontext, &name_metadata, 1), + ); + } + + return unit_metadata; + }; + + fn path_to_mdstring<'ll>(llcx: &'ll llvm::Context, path: &Path) -> &'ll Value { + let path_str = path_to_c_string(path); + unsafe { + llvm::LLVMMDStringInContext( + llcx, + path_str.as_ptr(), + path_str.as_bytes().len() as c_uint, + ) + } + } +} + +/// Creates a `DW_TAG_member` entry inside the DIE represented by the given `type_di_node`. +fn build_field_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + owner: &'ll DIScope, + name: &str, + size_and_align: (Size, Align), + offset: Size, + flags: DIFlags, + type_di_node: &'ll DIType, +) -> &'ll DIType { + unsafe { + llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + owner, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size_and_align.0.bits(), + size_and_align.1.bits() as u32, + offset.bits(), + flags, + type_di_node, + ) + } +} + +/// Creates the debuginfo node for a Rust struct type. Maybe be a regular struct or a tuple-struct. +fn build_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let struct_type = unique_type_id.expect_ty(); + let ty::Adt(adt_def, _) = struct_type.kind() else { + bug!("build_struct_type_di_node() called with non-struct-type: {:?}", struct_type); + }; + debug_assert!(adt_def.is_struct()); + let containing_scope = get_namespace_for_item(cx, adt_def.did()); + let struct_type_and_layout = cx.layout_of(struct_type); + let variant_def = adt_def.non_enum_variant(); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &compute_debuginfo_type_name(cx.tcx, struct_type, false), + size_and_align_of(struct_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| { + variant_def + .fields + .iter() + .enumerate() + .map(|(i, f)| { + let field_name = if variant_def.ctor_kind == CtorKind::Fn { + // This is a tuple struct + tuple_field_name(i) + } else { + // This is struct with named fields + Cow::Borrowed(f.name.as_str()) + }; + let field_layout = struct_type_and_layout.field(cx, i); + build_field_di_node( + cx, + owner, + &field_name[..], + (field_layout.size, field_layout.align.abi), + struct_type_and_layout.fields.offset(i), + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + ) + }) + .collect() + }, + |cx| build_generic_type_param_di_nodes(cx, struct_type), + ) +} + +//=----------------------------------------------------------------------------- +// Tuples +//=----------------------------------------------------------------------------- + +/// Returns names of captured upvars for closures and generators. +/// +/// Here are some examples: +/// - `name__field1__field2` when the upvar is captured by value. +/// - `_ref__name__field` when the upvar is captured by reference. +/// +/// For generators this only contains upvars that are shared by all states. +fn closure_saved_names_of_captured_variables(tcx: TyCtxt<'_>, def_id: DefId) -> SmallVec { + let body = tcx.optimized_mir(def_id); + + body.var_debug_info + .iter() + .filter_map(|var| { + let is_ref = match var.value { + mir::VarDebugInfoContents::Place(place) if place.local == mir::Local::new(1) => { + // The projection is either `[.., Field, Deref]` or `[.., Field]`. It + // implies whether the variable is captured by value or by reference. + matches!(place.projection.last().unwrap(), mir::ProjectionElem::Deref) + } + _ => return None, + }; + let prefix = if is_ref { "_ref__" } else { "" }; + Some(prefix.to_owned() + var.name.as_str()) + }) + .collect() +} + +/// Builds the DW_TAG_member debuginfo nodes for the upvars of a closure or generator. +/// For a generator, this will handle upvars shared by all states. +fn build_upvar_field_di_nodes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + closure_or_generator_ty: Ty<'tcx>, + closure_or_generator_di_node: &'ll DIType, +) -> SmallVec<&'ll DIType> { + let (&def_id, up_var_tys) = match closure_or_generator_ty.kind() { + ty::Generator(def_id, substs, _) => { + let upvar_tys: SmallVec<_> = substs.as_generator().prefix_tys().collect(); + (def_id, upvar_tys) + } + ty::Closure(def_id, substs) => { + let upvar_tys: SmallVec<_> = substs.as_closure().upvar_tys().collect(); + (def_id, upvar_tys) + } + _ => { + bug!( + "build_upvar_field_di_nodes() called with non-closure-or-generator-type: {:?}", + closure_or_generator_ty + ) + } + }; + + debug_assert!( + up_var_tys + .iter() + .all(|&t| t == cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)) + ); + + let capture_names = closure_saved_names_of_captured_variables(cx.tcx, def_id); + let layout = cx.layout_of(closure_or_generator_ty); + + up_var_tys + .into_iter() + .zip(capture_names.iter()) + .enumerate() + .map(|(index, (up_var_ty, capture_name))| { + build_field_di_node( + cx, + closure_or_generator_di_node, + capture_name, + cx.size_and_align_of(up_var_ty), + layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, up_var_ty), + ) + }) + .collect() +} + +/// Builds the DW_TAG_structure_type debuginfo node for a Rust tuple type. +fn build_tuple_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let tuple_type = unique_type_id.expect_ty(); + let &ty::Tuple(component_types) = tuple_type.kind() else { + bug!("build_tuple_type_di_node() called with non-tuple-type: {:?}", tuple_type) + }; + + let tuple_type_and_layout = cx.layout_of(tuple_type); + let type_name = compute_debuginfo_type_name(cx.tcx, tuple_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + size_and_align_of(tuple_type_and_layout), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + // Fields: + |cx, tuple_di_node| { + component_types + .into_iter() + .enumerate() + .map(|(index, component_type)| { + build_field_di_node( + cx, + tuple_di_node, + &tuple_field_name(index), + cx.size_and_align_of(component_type), + tuple_type_and_layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, component_type), + ) + }) + .collect() + }, + NO_GENERICS, + ) +} + +/// Builds the debuginfo node for a closure environment. +fn build_closure_env_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let closure_env_type = unique_type_id.expect_ty(); + let &ty::Closure(def_id, _substs) = closure_env_type.kind() else { + bug!("build_closure_env_di_node() called with non-closure-type: {:?}", closure_env_type) + }; + let containing_scope = get_namespace_for_item(cx, def_id); + let type_name = compute_debuginfo_type_name(cx.tcx, closure_env_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &type_name, + cx.size_and_align_of(closure_env_type), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| build_upvar_field_di_nodes(cx, closure_env_type, owner), + NO_GENERICS, + ) +} + +/// Build the debuginfo node for a Rust `union` type. +fn build_union_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let union_type = unique_type_id.expect_ty(); + let (union_def_id, variant_def) = match union_type.kind() { + ty::Adt(def, _) => (def.did(), def.non_enum_variant()), + _ => bug!("build_union_type_di_node on a non-ADT"), + }; + let containing_scope = get_namespace_for_item(cx, union_def_id); + let union_ty_and_layout = cx.layout_of(union_type); + let type_name = compute_debuginfo_type_name(cx.tcx, union_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Union, + unique_type_id, + &type_name, + size_and_align_of(union_ty_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + // Fields: + |cx, owner| { + variant_def + .fields + .iter() + .enumerate() + .map(|(i, f)| { + let field_layout = union_ty_and_layout.field(cx, i); + build_field_di_node( + cx, + owner, + f.name.as_str(), + size_and_align_of(field_layout), + Size::ZERO, + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + ) + }) + .collect() + }, + // Generics: + |cx| build_generic_type_param_di_nodes(cx, union_type), + ) +} + +// FIXME(eddyb) maybe precompute this? Right now it's computed once +// per generator monomorphization, but it doesn't depend on substs. +fn generator_layout_and_saved_local_names<'tcx>( + tcx: TyCtxt<'tcx>, + def_id: DefId, +) -> (&'tcx GeneratorLayout<'tcx>, IndexVec>) { + let body = tcx.optimized_mir(def_id); + let generator_layout = body.generator_layout().unwrap(); + let mut generator_saved_local_names = IndexVec::from_elem(None, &generator_layout.field_tys); + + let state_arg = mir::Local::new(1); + for var in &body.var_debug_info { + let mir::VarDebugInfoContents::Place(place) = &var.value else { continue }; + if place.local != state_arg { + continue; + } + match place.projection[..] { + [ + // Deref of the `Pin<&mut Self>` state argument. + mir::ProjectionElem::Field(..), + mir::ProjectionElem::Deref, + // Field of a variant of the state. + mir::ProjectionElem::Downcast(_, variant), + mir::ProjectionElem::Field(field, _), + ] => { + let name = &mut generator_saved_local_names + [generator_layout.variant_fields[variant][field]]; + if name.is_none() { + name.replace(var.name); + } + } + _ => {} + } + } + (generator_layout, generator_saved_local_names) +} + +/// Computes the type parameters for a type, if any, for the given metadata. +fn build_generic_type_param_di_nodes<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, +) -> SmallVec<&'ll DIType> { + if let ty::Adt(def, substs) = *ty.kind() { + if substs.types().next().is_some() { + let generics = cx.tcx.generics_of(def.did()); + let names = get_parameter_names(cx, generics); + let template_params: SmallVec<_> = iter::zip(substs, names) + .filter_map(|(kind, name)| { + if let GenericArgKind::Type(ty) = kind.unpack() { + let actual_type = + cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); + let actual_type_di_node = type_di_node(cx, actual_type); + let name = name.as_str(); + Some(unsafe { + llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( + DIB(cx), + None, + name.as_ptr().cast(), + name.len(), + actual_type_di_node, + ) + }) + } else { + None + } + }) + .collect(); + + return template_params; + } + } + + return smallvec![]; + + fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec { + let mut names = generics + .parent + .map_or_else(Vec::new, |def_id| get_parameter_names(cx, cx.tcx.generics_of(def_id))); + names.extend(generics.params.iter().map(|param| param.name)); + names + } +} + +/// Creates debug information for the given global variable. +/// +/// Adds the created debuginfo nodes directly to the crate's IR. +pub fn build_global_var_di_node<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId, global: &'ll Value) { + if cx.dbg_cx.is_none() { + return; + } + + // Only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo != DebugInfo::Full { + return; + } + + let tcx = cx.tcx; + + // We may want to remove the namespace scope if we're in an extern block (see + // https://github.com/rust-lang/rust/pull/46457#issuecomment-351750952). + let var_scope = get_namespace_for_item(cx, def_id); + let span = tcx.def_span(def_id); + + let (file_metadata, line_number) = if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + (file_metadata(cx, &loc.file), loc.line) + } else { + (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER) + }; + + let is_local_to_unit = is_node_local_to_unit(cx, def_id); + let variable_type = Instance::mono(cx.tcx, def_id).ty(cx.tcx, ty::ParamEnv::reveal_all()); + let type_di_node = type_di_node(cx, variable_type); + let var_name = tcx.item_name(def_id); + let var_name = var_name.as_str(); + let linkage_name = mangled_name_of_instance(cx, Instance::mono(tcx, def_id)).name; + // When empty, linkage_name field is omitted, + // which is what we want for no_mangle statics + let linkage_name = if var_name == linkage_name { "" } else { linkage_name }; + + let global_align = cx.align_of(variable_type); + + unsafe { + llvm::LLVMRustDIBuilderCreateStaticVariable( + DIB(cx), + Some(var_scope), + var_name.as_ptr().cast(), + var_name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + file_metadata, + line_number, + type_di_node, + is_local_to_unit, + global, + None, + global_align.bits() as u32, + ); + } +} + +/// Generates LLVM debuginfo for a vtable. +/// +/// The vtable type looks like a struct with a field for each function pointer and super-trait +/// pointer it contains (plus the `size` and `align` fields). +/// +/// Except for `size`, `align`, and `drop_in_place`, the field names don't try to mirror +/// the name of the method they implement. This can be implemented in the future once there +/// is a proper disambiguation scheme for dealing with methods from different traits that have +/// the same name. +fn build_vtable_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + poly_trait_ref: Option>, +) -> &'ll DIType { + let tcx = cx.tcx; + + let vtable_entries = if let Some(poly_trait_ref) = poly_trait_ref { + let trait_ref = poly_trait_ref.with_self_ty(tcx, ty); + let trait_ref = tcx.erase_regions(trait_ref); + + tcx.vtable_entries(trait_ref) + } else { + TyCtxt::COMMON_VTABLE_ENTRIES + }; + + // All function pointers are described as opaque pointers. This could be improved in the future + // by describing them as actual function pointers. + let void_pointer_ty = tcx.mk_imm_ptr(tcx.types.unit); + let void_pointer_type_di_node = type_di_node(cx, void_pointer_ty); + let usize_di_node = type_di_node(cx, tcx.types.usize); + let (pointer_size, pointer_align) = cx.size_and_align_of(void_pointer_ty); + // If `usize` is not pointer-sized and -aligned then the size and alignment computations + // for the vtable as a whole would be wrong. Let's make sure this holds even on weird + // platforms. + assert_eq!(cx.size_and_align_of(tcx.types.usize), (pointer_size, pointer_align)); + + let vtable_type_name = + compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::Type); + let unique_type_id = UniqueTypeId::for_vtable_ty(tcx, ty, poly_trait_ref); + let size = pointer_size * vtable_entries.len() as u64; + + // This gets mapped to a DW_AT_containing_type attribute which allows GDB to correlate + // the vtable to the type it is for. + let vtable_holder = type_di_node(cx, ty); + + build_type_with_children( + cx, + type_map::stub( + cx, + Stub::VTableTy { vtable_holder }, + unique_type_id, + &vtable_type_name, + (size, pointer_align), + NO_SCOPE_METADATA, + DIFlags::FlagArtificial, + ), + |cx, vtable_type_di_node| { + vtable_entries + .iter() + .enumerate() + .filter_map(|(index, vtable_entry)| { + let (field_name, field_type_di_node) = match vtable_entry { + ty::VtblEntry::MetadataDropInPlace => { + ("drop_in_place".to_string(), void_pointer_type_di_node) + } + ty::VtblEntry::Method(_) => { + // Note: This code does not try to give a proper name to each method + // because their might be multiple methods with the same name + // (coming from different traits). + (format!("__method{}", index), void_pointer_type_di_node) + } + ty::VtblEntry::TraitVPtr(_) => { + (format!("__super_trait_ptr{}", index), void_pointer_type_di_node) + } + ty::VtblEntry::MetadataAlign => ("align".to_string(), usize_di_node), + ty::VtblEntry::MetadataSize => ("size".to_string(), usize_di_node), + ty::VtblEntry::Vacant => return None, + }; + + let field_offset = pointer_size * index as u64; + + Some(build_field_di_node( + cx, + vtable_type_di_node, + &field_name, + (pointer_size, pointer_align), + field_offset, + DIFlags::FlagZero, + field_type_di_node, + )) + }) + .collect() + }, + NO_GENERICS, + ) + .di_node +} + +fn vcall_visibility_metadata<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + trait_ref: Option>, + vtable: &'ll Value, +) { + enum VCallVisibility { + Public = 0, + LinkageUnit = 1, + TranslationUnit = 2, + } + + let Some(trait_ref) = trait_ref else { return }; + + let trait_ref_self = trait_ref.with_self_ty(cx.tcx, ty); + let trait_ref_self = cx.tcx.erase_regions(trait_ref_self); + let trait_def_id = trait_ref_self.def_id(); + let trait_vis = cx.tcx.visibility(trait_def_id); + + let cgus = cx.sess().codegen_units(); + let single_cgu = cgus == 1; + + let lto = cx.sess().lto(); + + // Since LLVM requires full LTO for the virtual function elimination optimization to apply, + // only the `Lto::Fat` cases are relevant currently. + let vcall_visibility = match (lto, trait_vis, single_cgu) { + // If there is not LTO and the visibility in public, we have to assume that the vtable can + // be seen from anywhere. With multiple CGUs, the vtable is quasi-public. + (Lto::No | Lto::ThinLocal, Visibility::Public, _) + | (Lto::No, Visibility::Restricted(_) | Visibility::Invisible, false) => { + VCallVisibility::Public + } + // With LTO and a quasi-public visibility, the usages of the functions of the vtable are + // all known by the `LinkageUnit`. + // FIXME: LLVM only supports this optimization for `Lto::Fat` currently. Once it also + // supports `Lto::Thin` the `VCallVisibility` may have to be adjusted for those. + (Lto::Fat | Lto::Thin, Visibility::Public, _) + | ( + Lto::ThinLocal | Lto::Thin | Lto::Fat, + Visibility::Restricted(_) | Visibility::Invisible, + false, + ) => VCallVisibility::LinkageUnit, + // If there is only one CGU, private vtables can only be seen by that CGU/translation unit + // and therefore we know of all usages of functions in the vtable. + (_, Visibility::Restricted(_) | Visibility::Invisible, true) => { + VCallVisibility::TranslationUnit + } + }; + + let trait_ref_typeid = typeid_for_trait_ref(cx.tcx, trait_ref); + + unsafe { + let typeid = llvm::LLVMMDStringInContext( + cx.llcx, + trait_ref_typeid.as_ptr() as *const c_char, + trait_ref_typeid.as_bytes().len() as c_uint, + ); + let v = [cx.const_usize(0), typeid]; + llvm::LLVMRustGlobalAddMetadata( + vtable, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + cx.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ); + let vcall_visibility = llvm::LLVMValueAsMetadata(cx.const_u64(vcall_visibility as u64)); + let vcall_visibility_metadata = llvm::LLVMMDNodeInContext2(cx.llcx, &vcall_visibility, 1); + llvm::LLVMGlobalSetMetadata( + vtable, + llvm::MetadataType::MD_vcall_visibility as c_uint, + vcall_visibility_metadata, + ); + } +} + +/// Creates debug information for the given vtable, which is for the +/// given type. +/// +/// Adds the created metadata nodes directly to the crate's IR. +pub fn create_vtable_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + ty: Ty<'tcx>, + poly_trait_ref: Option>, + vtable: &'ll Value, +) { + // FIXME(flip1995): The virtual function elimination optimization only works with full LTO in + // LLVM at the moment. + if cx.sess().opts.unstable_opts.virtual_function_elimination && cx.sess().lto() == Lto::Fat { + vcall_visibility_metadata(cx, ty, poly_trait_ref, vtable); + } + + if cx.dbg_cx.is_none() { + return; + } + + // Only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo != DebugInfo::Full { + return; + } + + let vtable_name = + compute_debuginfo_vtable_name(cx.tcx, ty, poly_trait_ref, VTableNameKind::GlobalVariable); + let vtable_type_di_node = build_vtable_type_di_node(cx, ty, poly_trait_ref); + let linkage_name = ""; + + unsafe { + llvm::LLVMRustDIBuilderCreateStaticVariable( + DIB(cx), + NO_SCOPE_METADATA, + vtable_name.as_ptr().cast(), + vtable_name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + vtable_type_di_node, + true, + vtable, + None, + 0, + ); + } +} + +/// Creates an "extension" of an existing `DIScope` into another file. +pub fn extend_scope_to_file<'ll>( + cx: &CodegenCx<'ll, '_>, + scope_metadata: &'ll DIScope, + file: &SourceFile, +) -> &'ll DILexicalBlock { + let file_metadata = file_metadata(cx, file); + unsafe { llvm::LLVMRustDIBuilderCreateLexicalBlockFile(DIB(cx), scope_metadata, file_metadata) } +} + +pub fn tuple_field_name(field_index: usize) -> Cow<'static, str> { + const TUPLE_FIELD_NAMES: [&'static str; 16] = [ + "__0", "__1", "__2", "__3", "__4", "__5", "__6", "__7", "__8", "__9", "__10", "__11", + "__12", "__13", "__14", "__15", + ]; + TUPLE_FIELD_NAMES + .get(field_index) + .map(|s| Cow::from(*s)) + .unwrap_or_else(|| Cow::from(format!("__{}", field_index))) +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs new file mode 100644 index 000000000..d6e2c8ccd --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/cpp_like.rs @@ -0,0 +1,514 @@ +use std::borrow::Cow; + +use libc::c_uint; +use rustc_codegen_ssa::debuginfo::{ + type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo, +}; +use rustc_middle::{ + bug, + ty::{ + self, + layout::{LayoutOf, TyAndLayout}, + util::Discr, + AdtDef, GeneratorSubsts, + }, +}; +use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; +use smallvec::smallvec; + +use crate::{ + common::CodegenCx, + debuginfo::{ + metadata::{ + build_field_di_node, closure_saved_names_of_captured_variables, + enums::tag_base_type, + file_metadata, generator_layout_and_saved_local_names, size_and_align_of, + type_map::{self, UniqueTypeId}, + unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, NO_SCOPE_METADATA, + UNKNOWN_LINE_NUMBER, + }, + utils::DIB, + }, + llvm::{ + self, + debuginfo::{DIFile, DIFlags, DIType}, + }, +}; + +/// In CPP-like mode, we generate a union of structs for each variant and an +/// explicit discriminant field roughly equivalent to the following C/C++ code: +/// +/// ```c +/// union enum$<{fully-qualified-name}> { +/// struct {variant 0 name} { +/// +/// } variant0; +/// +/// {name} discriminant; +/// } +/// ``` +/// +/// As you can see, the type name is wrapped `enum$`. This way we can have a +/// single NatVis rule for handling all enums. +/// +/// At the LLVM IR level this looks like +/// +/// ```txt +/// DW_TAG_union_type (top-level type for enum) +/// DW_TAG_member (member for variant 1) +/// DW_TAG_member (member for variant 2) +/// DW_TAG_member (member for variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// DW_TAG_enumeration_type (type of tag) +/// ``` +/// +/// The above encoding applies for enums with a direct tag. For niche-tag we have to do things +/// differently in order to allow a NatVis visualizer to extract all the information needed: +/// We generate a union of two fields, one for the dataful variant +/// and one that just points to the discriminant (which is some field within the dataful variant). +/// We also create a DW_TAG_enumeration_type DIE that contains tag values for the non-dataful +/// variants and make the discriminant field that type. We then use NatVis to render the enum type +/// correctly in Windbg/VS. This will generate debuginfo roughly equivalent to the following C: +/// +/// ```c +/// union enum$<{name}, {min niche}, {max niche}, {dataful variant name}> { +/// struct { +/// +/// } dataful_variant; +/// enum Discriminant$ { +/// +/// } discriminant; +/// } +/// ``` +/// +/// The NatVis in `intrinsic.natvis` matches on the type name `enum$<*, *, *, *>` +/// and evaluates `this.discriminant`. If the value is between the min niche and max +/// niche, then the enum is in the dataful variant and `this.dataful_variant` is +/// rendered. Otherwise, the enum is in one of the non-dataful variants. In that +/// case, we just need to render the name of the `this.discriminant` enum. +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) + }; + + let enum_type_and_layout = cx.layout_of(enum_type); + let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); + + debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + type_map::Stub::Union, + unique_type_id, + &enum_type_name, + cx.size_and_align_of(enum_type), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, enum_type_di_node| { + match enum_type_and_layout.variants { + Variants::Single { index: variant_index } => { + if enum_adt_def.variants().is_empty() { + // Uninhabited enums have Variants::Single. We don't generate + // any members for them. + return smallvec![]; + } + + build_single_variant_union_fields( + cx, + enum_adt_def, + enum_type_and_layout, + enum_type_di_node, + variant_index, + ) + } + Variants::Multiple { + tag_encoding: TagEncoding::Direct, + ref variants, + tag_field, + .. + } => build_union_fields_for_direct_tag_enum( + cx, + enum_adt_def, + enum_type_and_layout, + enum_type_di_node, + &mut variants.indices(), + tag_field, + ), + Variants::Multiple { + tag_encoding: TagEncoding::Niche { dataful_variant, .. }, + ref variants, + tag_field, + .. + } => build_union_fields_for_niche_tag_enum( + cx, + enum_adt_def, + enum_type_and_layout, + enum_type_di_node, + dataful_variant, + &mut variants.indices(), + tag_field, + ), + } + }, + NO_GENERICS, + ) +} + +/// A generator debuginfo node looks the same as a that of an enum type. +/// +/// See [build_enum_type_di_node] for more information. +pub(super) fn build_generator_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let generator_type = unique_type_id.expect_ty(); + let generator_type_and_layout = cx.layout_of(generator_type); + let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + + debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + type_map::Stub::Union, + unique_type_id, + &generator_type_name, + size_and_align_of(generator_type_and_layout), + NO_SCOPE_METADATA, + DIFlags::FlagZero, + ), + |cx, generator_type_di_node| match generator_type_and_layout.variants { + Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => { + build_union_fields_for_direct_tag_generator( + cx, + generator_type_and_layout, + generator_type_di_node, + ) + } + Variants::Single { .. } + | Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, .. } => { + bug!( + "Encountered generator with non-direct-tag layout: {:?}", + generator_type_and_layout + ) + } + }, + NO_GENERICS, + ) +} + +fn build_single_variant_union_fields<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_index: VariantIdx, +) -> SmallVec<&'ll DIType> { + let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + let variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout.ty, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + variant_layout, + ); + + // NOTE: The field name of the union is the same as the variant name, not "variant0". + let variant_name = enum_adt_def.variant(variant_index).name.as_str(); + + smallvec![build_field_di_node( + cx, + enum_type_di_node, + variant_name, + // NOTE: We use the size and align of the entire type, not from variant_layout + // since the later is sometimes smaller (if it has fewer fields). + size_and_align_of(enum_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + variant_struct_type_di_node, + )] +} + +fn build_union_fields_for_direct_tag_enum<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_indices: &mut dyn Iterator, + tag_field: usize, +) -> SmallVec<&'ll DIType> { + let variant_field_infos: SmallVec> = variant_indices + .map(|variant_index| { + let variant_layout = enum_type_and_layout.for_variant(cx, variant_index); + + VariantFieldInfo { + variant_index, + variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout.ty, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + variant_layout, + ), + source_info: None, + } + }) + .collect(); + + let discr_type_name = cx.tcx.item_name(enum_adt_def.did()); + let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + let discr_type_di_node = super::build_enumeration_type_di_node( + cx, + discr_type_name.as_str(), + tag_base_type, + &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { + (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) + }), + enum_type_di_node, + ); + + build_union_fields_for_direct_tag_enum_or_generator( + cx, + enum_type_and_layout, + enum_type_di_node, + &variant_field_infos, + discr_type_di_node, + tag_field, + ) +} + +fn build_union_fields_for_niche_tag_enum<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + dataful_variant_index: VariantIdx, + variant_indices: &mut dyn Iterator, + tag_field: usize, +) -> SmallVec<&'ll DIType> { + let dataful_variant_struct_type_di_node = super::build_enum_variant_struct_type_di_node( + cx, + enum_type_and_layout.ty, + enum_type_di_node, + dataful_variant_index, + &enum_adt_def.variant(dataful_variant_index), + enum_type_and_layout.for_variant(cx, dataful_variant_index), + ); + + let tag_base_type = super::tag_base_type(cx, enum_type_and_layout); + // Create an DW_TAG_enumerator for each variant except the dataful one. + let discr_type_di_node = super::build_enumeration_type_di_node( + cx, + "Discriminant$", + tag_base_type, + &mut variant_indices.filter_map(|variant_index| { + if let Some(discr_val) = + super::compute_discriminant_value(cx, enum_type_and_layout, variant_index) + { + let discr = Discr { val: discr_val as u128, ty: tag_base_type }; + let variant_name = Cow::from(enum_adt_def.variant(variant_index).name.as_str()); + Some((discr, variant_name)) + } else { + debug_assert_eq!(variant_index, dataful_variant_index); + None + } + }), + enum_type_di_node, + ); + + smallvec![ + build_field_di_node( + cx, + enum_type_di_node, + "dataful_variant", + size_and_align_of(enum_type_and_layout), + Size::ZERO, + DIFlags::FlagZero, + dataful_variant_struct_type_di_node, + ), + build_field_di_node( + cx, + enum_type_di_node, + "discriminant", + cx.size_and_align_of(tag_base_type), + enum_type_and_layout.fields.offset(tag_field), + DIFlags::FlagZero, + discr_type_di_node, + ), + ] +} + +fn build_union_fields_for_direct_tag_generator<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + generator_type_and_layout: TyAndLayout<'tcx>, + generator_type_di_node: &'ll DIType, +) -> SmallVec<&'ll DIType> { + let Variants::Multiple { tag_encoding: TagEncoding::Direct, tag_field, .. } = generator_type_and_layout.variants else { + bug!("This function only supports layouts with directly encoded tags.") + }; + + let (generator_def_id, generator_substs) = match generator_type_and_layout.ty.kind() { + &ty::Generator(def_id, substs, _) => (def_id, substs.as_generator()), + _ => unreachable!(), + }; + + let (generator_layout, state_specific_upvar_names) = + generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + + let common_upvar_names = closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + let variant_range = generator_substs.variant_range(generator_def_id, cx.tcx); + + // Build the type node for each field. + let variant_field_infos: SmallVec> = variant_range + .map(|variant_index| { + let variant_struct_type_di_node = super::build_generator_variant_struct_type_di_node( + cx, + variant_index, + generator_type_and_layout, + generator_type_di_node, + generator_layout, + &state_specific_upvar_names, + &common_upvar_names, + ); + + let span = generator_layout.variant_source_info[variant_index].span; + let source_info = if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + Some((file_metadata(cx, &loc.file), loc.line as c_uint)) + } else { + None + }; + + VariantFieldInfo { variant_index, variant_struct_type_di_node, source_info } + }) + .collect(); + + let tag_base_type = tag_base_type(cx, generator_type_and_layout); + let discr_type_name = "Discriminant$"; + let discr_type_di_node = super::build_enumeration_type_di_node( + cx, + discr_type_name, + tag_base_type, + &mut generator_substs + .discriminants(generator_def_id, cx.tcx) + .map(|(variant_index, discr)| (discr, GeneratorSubsts::variant_name(variant_index))), + generator_type_di_node, + ); + + build_union_fields_for_direct_tag_enum_or_generator( + cx, + generator_type_and_layout, + generator_type_di_node, + &variant_field_infos[..], + discr_type_di_node, + tag_field, + ) +} + +/// This is a helper function shared between enums and generators that makes sure fields have the +/// expect names. +fn build_union_fields_for_direct_tag_enum_or_generator<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_field_infos: &[VariantFieldInfo<'ll>], + discr_type_di_node: &'ll DIType, + tag_field: usize, +) -> SmallVec<&'ll DIType> { + let mut unions_fields = SmallVec::with_capacity(variant_field_infos.len() + 1); + + // We create a field in the union for each variant ... + unions_fields.extend(variant_field_infos.into_iter().map(|variant_member_info| { + let (file_di_node, line_number) = variant_member_info + .source_info + .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); + + let field_name = variant_union_field_name(variant_member_info.variant_index); + let (size, align) = size_and_align_of(enum_type_and_layout); + + // We use LLVMRustDIBuilderCreateMemberType() member type directly because + // the build_field_di_node() function does not support specifying a source location, + // which is something that we don't do anywhere else. + unsafe { + llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + enum_type_di_node, + field_name.as_ptr().cast(), + field_name.len(), + file_di_node, + line_number, + // NOTE: We use the size and align of the entire type, not from variant_layout + // since the later is sometimes smaller (if it has fewer fields). + size.bits(), + align.bits() as u32, + // Union fields are always at offset zero + Size::ZERO.bits(), + DIFlags::FlagZero, + variant_member_info.variant_struct_type_di_node, + ) + } + })); + + debug_assert_eq!( + cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), + cx.size_and_align_of(super::tag_base_type(cx, enum_type_and_layout)) + ); + + // ... and a field for the discriminant. + unions_fields.push(build_field_di_node( + cx, + enum_type_di_node, + "discriminant", + cx.size_and_align_of(enum_type_and_layout.field(cx, tag_field).ty), + enum_type_and_layout.fields.offset(tag_field), + DIFlags::FlagZero, + discr_type_di_node, + )); + + unions_fields +} + +/// Information about a single field of the top-level DW_TAG_union_type. +struct VariantFieldInfo<'ll> { + variant_index: VariantIdx, + variant_struct_type_di_node: &'ll DIType, + source_info: Option<(&'ll DIFile, c_uint)>, +} + +fn variant_union_field_name(variant_index: VariantIdx) -> Cow<'static, str> { + const PRE_ALLOCATED: [&str; 16] = [ + "variant0", + "variant1", + "variant2", + "variant3", + "variant4", + "variant5", + "variant6", + "variant7", + "variant8", + "variant9", + "variant10", + "variant11", + "variant12", + "variant13", + "variant14", + "variant15", + ]; + + PRE_ALLOCATED + .get(variant_index.as_usize()) + .map(|&s| Cow::from(s)) + .unwrap_or_else(|| format!("variant{}", variant_index.as_usize()).into()) +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs new file mode 100644 index 000000000..73e01d045 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/mod.rs @@ -0,0 +1,437 @@ +use rustc_codegen_ssa::debuginfo::{ + type_names::{compute_debuginfo_type_name, cpp_like_debuginfo}, + wants_c_like_enum_debuginfo, +}; +use rustc_hir::def::CtorKind; +use rustc_index::vec::IndexVec; +use rustc_middle::{ + bug, + mir::{Field, GeneratorLayout, GeneratorSavedLocal}, + ty::{ + self, + layout::{IntegerExt, LayoutOf, PrimitiveExt, TyAndLayout}, + util::Discr, + AdtDef, GeneratorSubsts, Ty, VariantDef, + }, +}; +use rustc_span::Symbol; +use rustc_target::abi::{HasDataLayout, Integer, Primitive, TagEncoding, VariantIdx, Variants}; +use std::borrow::Cow; + +use crate::{ + common::CodegenCx, + debuginfo::{ + metadata::{ + build_field_di_node, build_generic_type_param_di_nodes, type_di_node, + type_map::{self, Stub}, + unknown_file_metadata, UNKNOWN_LINE_NUMBER, + }, + utils::{create_DIArray, get_namespace_for_item, DIB}, + }, + llvm::{ + self, + debuginfo::{DIFlags, DIType}, + }, +}; + +use super::{ + size_and_align_of, + type_map::{DINodeCreationResult, UniqueTypeId}, + SmallVec, +}; + +mod cpp_like; +mod native; + +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) + }; + + let enum_type_and_layout = cx.layout_of(enum_type); + + if wants_c_like_enum_debuginfo(enum_type_and_layout) { + return build_c_style_enum_di_node(cx, enum_adt_def, enum_type_and_layout); + } + + if cpp_like_debuginfo(cx.tcx) { + cpp_like::build_enum_type_di_node(cx, unique_type_id) + } else { + native::build_enum_type_di_node(cx, unique_type_id) + } +} + +pub(super) fn build_generator_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + if cpp_like_debuginfo(cx.tcx) { + cpp_like::build_generator_di_node(cx, unique_type_id) + } else { + native::build_generator_di_node(cx, unique_type_id) + } +} + +/// Build the debuginfo node for a C-style enum, i.e. an enum the variants of which have no fields. +/// +/// The resulting debuginfo will be a DW_TAG_enumeration_type. +fn build_c_style_enum_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_adt_def: AdtDef<'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, +) -> DINodeCreationResult<'ll> { + let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + DINodeCreationResult { + di_node: build_enumeration_type_di_node( + cx, + &compute_debuginfo_type_name(cx.tcx, enum_type_and_layout.ty, false), + tag_base_type(cx, enum_type_and_layout), + &mut enum_adt_def.discriminants(cx.tcx).map(|(variant_index, discr)| { + (discr, Cow::from(enum_adt_def.variant(variant_index).name.as_str())) + }), + containing_scope, + ), + already_stored_in_typemap: false, + } +} + +/// Extract the type with which we want to describe the tag of the given enum or generator. +fn tag_base_type<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, +) -> Ty<'tcx> { + debug_assert!(match enum_type_and_layout.ty.kind() { + ty::Generator(..) => true, + ty::Adt(adt_def, _) => adt_def.is_enum(), + _ => false, + }); + + match enum_type_and_layout.layout.variants() { + // A single-variant enum has no discriminant. + Variants::Single { .. } => { + bug!("tag_base_type() called for enum without tag: {:?}", enum_type_and_layout) + } + + Variants::Multiple { tag_encoding: TagEncoding::Niche { .. }, tag, .. } => { + // Niche tags are always normalized to unsized integers of the correct size. + match tag.primitive() { + Primitive::Int(t, _) => t, + Primitive::F32 => Integer::I32, + Primitive::F64 => Integer::I64, + Primitive::Pointer => { + // If the niche is the NULL value of a reference, then `discr_enum_ty` will be + // a RawPtr. CodeView doesn't know what to do with enums whose base type is a + // pointer so we fix this up to just be `usize`. + // DWARF might be able to deal with this but with an integer type we are on + // the safe side there too. + cx.data_layout().ptr_sized_integer() + } + } + .to_ty(cx.tcx, false) + } + + Variants::Multiple { tag_encoding: TagEncoding::Direct, tag, .. } => { + // Direct tags preserve the sign. + tag.primitive().to_ty(cx.tcx) + } + } +} + +/// Build a DW_TAG_enumeration_type debuginfo node, with the given base type and variants. +/// This is a helper function and does not register anything in the type map by itself. +/// +/// `variants` is an iterator of (discr-value, variant-name). +/// +// NOTE: Handling of discriminant values is somewhat inconsistent. They can appear as u128, +// u64, and i64. Here everything gets mapped to i64 because that's what LLVM's API expects. +fn build_enumeration_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + type_name: &str, + base_type: Ty<'tcx>, + variants: &mut dyn Iterator, Cow<'tcx, str>)>, + containing_scope: &'ll DIType, +) -> &'ll DIType { + let is_unsigned = match base_type.kind() { + ty::Int(_) => false, + ty::Uint(_) => true, + _ => bug!("build_enumeration_type_di_node() called with non-integer tag type."), + }; + + let enumerator_di_nodes: SmallVec> = variants + .map(|(discr, variant_name)| { + unsafe { + Some(llvm::LLVMRustDIBuilderCreateEnumerator( + DIB(cx), + variant_name.as_ptr().cast(), + variant_name.len(), + // FIXME: what if enumeration has i128 discriminant? + discr.val as i64, + is_unsigned, + )) + } + }) + .collect(); + + let (size, align) = cx.size_and_align_of(base_type); + + unsafe { + llvm::LLVMRustDIBuilderCreateEnumerationType( + DIB(cx), + containing_scope, + type_name.as_ptr().cast(), + type_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + create_DIArray(DIB(cx), &enumerator_di_nodes[..]), + type_di_node(cx, base_type), + true, + ) + } +} + +/// Build the debuginfo node for the struct type describing a single variant of an enum. +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// ``` +/// +/// In CPP-like mode, we have the exact same descriptions for each variant too: +/// +/// ```txt +/// DW_TAG_union_type (top-level type for enum) +/// DW_TAG_member (member for variant 1) +/// DW_TAG_member (member for variant 2) +/// DW_TAG_member (member for variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// DW_TAG_enumeration_type (type of tag) +/// ``` +/// +/// The node looks like: +/// +/// ```txt +/// DW_TAG_structure_type +/// DW_AT_name +/// DW_AT_byte_size 0x00000010 +/// DW_AT_alignment 0x00000008 +/// DW_TAG_member +/// DW_AT_name +/// DW_AT_type <0x0000018e> +/// DW_AT_alignment 0x00000004 +/// DW_AT_data_member_location 4 +/// DW_TAG_member +/// DW_AT_name +/// DW_AT_type <0x00000195> +/// DW_AT_alignment 0x00000008 +/// DW_AT_data_member_location 8 +/// ... +/// ``` +/// +/// The type of a variant is always a struct type with the name of the variant +/// and a DW_TAG_member for each field (but not the discriminant). +fn build_enum_variant_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type: Ty<'tcx>, + enum_type_di_node: &'ll DIType, + variant_index: VariantIdx, + variant_def: &VariantDef, + variant_layout: TyAndLayout<'tcx>, +) -> &'ll DIType { + debug_assert_eq!(variant_layout.ty, enum_type); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + UniqueTypeId::for_enum_variant_struct_type(cx.tcx, enum_type, variant_index), + variant_def.name.as_str(), + // NOTE: We use size and align of enum_type, not from variant_layout: + cx.size_and_align_of(enum_type), + Some(enum_type_di_node), + DIFlags::FlagZero, + ), + |cx, struct_type_di_node| { + (0..variant_layout.fields.count()) + .map(|field_index| { + let field_name = if variant_def.ctor_kind != CtorKind::Fn { + // Fields have names + Cow::from(variant_def.fields[field_index].name.as_str()) + } else { + // Tuple-like + super::tuple_field_name(field_index) + }; + + let field_layout = variant_layout.field(cx, field_index); + + build_field_di_node( + cx, + struct_type_di_node, + &field_name, + (field_layout.size, field_layout.align.abi), + variant_layout.fields.offset(field_index), + DIFlags::FlagZero, + type_di_node(cx, field_layout.ty), + ) + }) + .collect() + }, + |cx| build_generic_type_param_di_nodes(cx, enum_type), + ) + .di_node +} + +/// Build the struct type for describing a single generator state. +/// See [build_generator_variant_struct_type_di_node]. +/// +/// ```txt +/// +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// ---> DW_TAG_structure_type (type of variant 1) +/// ---> DW_TAG_structure_type (type of variant 2) +/// ---> DW_TAG_structure_type (type of variant 3) +/// +/// ``` +pub fn build_generator_variant_struct_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + variant_index: VariantIdx, + generator_type_and_layout: TyAndLayout<'tcx>, + generator_type_di_node: &'ll DIType, + generator_layout: &GeneratorLayout<'tcx>, + state_specific_upvar_names: &IndexVec>, + common_upvar_names: &[String], +) -> &'ll DIType { + let variant_name = GeneratorSubsts::variant_name(variant_index); + let unique_type_id = UniqueTypeId::for_enum_variant_struct_type( + cx.tcx, + generator_type_and_layout.ty, + variant_index, + ); + + let variant_layout = generator_type_and_layout.for_variant(cx, variant_index); + + let generator_substs = match generator_type_and_layout.ty.kind() { + ty::Generator(_, substs, _) => substs.as_generator(), + _ => unreachable!(), + }; + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &variant_name, + size_and_align_of(generator_type_and_layout), + Some(generator_type_di_node), + DIFlags::FlagZero, + ), + |cx, variant_struct_type_di_node| { + // Fields that just belong to this variant/state + let state_specific_fields: SmallVec<_> = (0..variant_layout.fields.count()) + .map(|field_index| { + let generator_saved_local = generator_layout.variant_fields[variant_index] + [Field::from_usize(field_index)]; + let field_name_maybe = state_specific_upvar_names[generator_saved_local]; + let field_name = field_name_maybe + .as_ref() + .map(|s| Cow::from(s.as_str())) + .unwrap_or_else(|| super::tuple_field_name(field_index)); + + let field_type = variant_layout.field(cx, field_index).ty; + + build_field_di_node( + cx, + variant_struct_type_di_node, + &field_name, + cx.size_and_align_of(field_type), + variant_layout.fields.offset(field_index), + DIFlags::FlagZero, + type_di_node(cx, field_type), + ) + }) + .collect(); + + // Fields that are common to all states + let common_fields: SmallVec<_> = generator_substs + .prefix_tys() + .enumerate() + .map(|(index, upvar_ty)| { + build_field_di_node( + cx, + variant_struct_type_di_node, + &common_upvar_names[index], + cx.size_and_align_of(upvar_ty), + generator_type_and_layout.fields.offset(index), + DIFlags::FlagZero, + type_di_node(cx, upvar_ty), + ) + }) + .collect(); + + state_specific_fields.into_iter().chain(common_fields.into_iter()).collect() + }, + |cx| build_generic_type_param_di_nodes(cx, generator_type_and_layout.ty), + ) + .di_node +} + +/// Returns the discriminant value corresponding to the variant index. +/// +/// Will return `None` if there is less than two variants (because then the enum won't have) +/// a tag, and if this is the dataful variant of a niche-layout enum (because then there is no +/// single discriminant value). +fn compute_discriminant_value<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + variant_index: VariantIdx, +) -> Option { + match enum_type_and_layout.layout.variants() { + &Variants::Single { .. } => None, + &Variants::Multiple { tag_encoding: TagEncoding::Direct, .. } => Some( + enum_type_and_layout.ty.discriminant_for_variant(cx.tcx, variant_index).unwrap().val + as u64, + ), + &Variants::Multiple { + tag_encoding: TagEncoding::Niche { ref niche_variants, niche_start, dataful_variant }, + tag, + .. + } => { + if variant_index == dataful_variant { + None + } else { + let value = (variant_index.as_u32() as u128) + .wrapping_sub(niche_variants.start().as_u32() as u128) + .wrapping_add(niche_start); + let value = tag.size(cx).truncate(value); + // NOTE(eddyb) do *NOT* remove this assert, until + // we pass the full 128-bit value to LLVM, otherwise + // truncation will be silent and remain undetected. + assert_eq!(value as u64 as u128, value); + Some(value as u64) + } + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs new file mode 100644 index 000000000..f1935e0ec --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/enums/native.rs @@ -0,0 +1,441 @@ +use std::borrow::Cow; + +use crate::{ + common::CodegenCx, + debuginfo::{ + metadata::{ + closure_saved_names_of_captured_variables, + enums::tag_base_type, + file_metadata, generator_layout_and_saved_local_names, size_and_align_of, type_di_node, + type_map::{self, Stub, StubInfo, UniqueTypeId}, + unknown_file_metadata, DINodeCreationResult, SmallVec, NO_GENERICS, + UNKNOWN_LINE_NUMBER, + }, + utils::{create_DIArray, get_namespace_for_item, DIB}, + }, + llvm::{ + self, + debuginfo::{DIFile, DIFlags, DIType}, + }, +}; +use libc::c_uint; +use rustc_codegen_ssa::{ + debuginfo::{type_names::compute_debuginfo_type_name, wants_c_like_enum_debuginfo}, + traits::ConstMethods, +}; +use rustc_middle::{ + bug, + ty::{ + self, + layout::{LayoutOf, TyAndLayout}, + }, +}; +use rustc_target::abi::{Size, TagEncoding, VariantIdx, Variants}; +use smallvec::smallvec; + +/// Build the debuginfo node for an enum type. The listing below shows how such a +/// type looks like at the LLVM IR/DWARF level. It is a `DW_TAG_structure_type` +/// with a single `DW_TAG_variant_part` that in turn contains a `DW_TAG_variant` +/// for each variant of the enum. The variant-part also contains a single member +/// describing the discriminant, and a nested struct type for each of the variants. +/// +/// ```txt +/// ---> DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +pub(super) fn build_enum_type_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let enum_type = unique_type_id.expect_ty(); + let &ty::Adt(enum_adt_def, _) = enum_type.kind() else { + bug!("build_enum_type_di_node() called with non-enum type: `{:?}`", enum_type) + }; + + let containing_scope = get_namespace_for_item(cx, enum_adt_def.did()); + let enum_type_and_layout = cx.layout_of(enum_type); + let enum_type_name = compute_debuginfo_type_name(cx.tcx, enum_type, false); + + debug_assert!(!wants_c_like_enum_debuginfo(enum_type_and_layout)); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &enum_type_name, + size_and_align_of(enum_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + |cx, enum_type_di_node| { + // Build the struct type for each variant. These will be referenced by the + // DW_TAG_variant DIEs inside of the DW_TAG_variant_part DIE. + // We also called the names for the corresponding DW_TAG_variant DIEs here. + let variant_member_infos: SmallVec<_> = enum_adt_def + .variant_range() + .map(|variant_index| VariantMemberInfo { + variant_index, + variant_name: Cow::from(enum_adt_def.variant(variant_index).name.as_str()), + variant_struct_type_di_node: super::build_enum_variant_struct_type_di_node( + cx, + enum_type, + enum_type_di_node, + variant_index, + enum_adt_def.variant(variant_index), + enum_type_and_layout.for_variant(cx, variant_index), + ), + source_info: None, + }) + .collect(); + + smallvec![build_enum_variant_part_di_node( + cx, + enum_type_and_layout, + enum_type_di_node, + &variant_member_infos[..], + )] + }, + // We don't seem to be emitting generic args on the enum type, it seems. Rather + // they get attached to the struct type of each variant. + NO_GENERICS, + ) +} + +/// Build the debuginfo node for a generator environment. It looks the same as the debuginfo for +/// an enum. See [build_enum_type_di_node] for more information. +/// +/// ```txt +/// +/// ---> DW_TAG_structure_type (top-level type for the generator) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// +/// ``` +pub(super) fn build_generator_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, +) -> DINodeCreationResult<'ll> { + let generator_type = unique_type_id.expect_ty(); + let &ty::Generator(generator_def_id, _, _ ) = generator_type.kind() else { + bug!("build_generator_di_node() called with non-generator type: `{:?}`", generator_type) + }; + + let containing_scope = get_namespace_for_item(cx, generator_def_id); + let generator_type_and_layout = cx.layout_of(generator_type); + + debug_assert!(!wants_c_like_enum_debuginfo(generator_type_and_layout)); + + let generator_type_name = compute_debuginfo_type_name(cx.tcx, generator_type, false); + + type_map::build_type_with_children( + cx, + type_map::stub( + cx, + Stub::Struct, + unique_type_id, + &generator_type_name, + size_and_align_of(generator_type_and_layout), + Some(containing_scope), + DIFlags::FlagZero, + ), + |cx, generator_type_di_node| { + let (generator_layout, state_specific_upvar_names) = + generator_layout_and_saved_local_names(cx.tcx, generator_def_id); + + let Variants::Multiple { tag_encoding: TagEncoding::Direct, ref variants, .. } = generator_type_and_layout.variants else { + bug!( + "Encountered generator with non-direct-tag layout: {:?}", + generator_type_and_layout + ) + }; + + let common_upvar_names = + closure_saved_names_of_captured_variables(cx.tcx, generator_def_id); + + // Build variant struct types + let variant_struct_type_di_nodes: SmallVec<_> = variants + .indices() + .map(|variant_index| { + // FIXME: This is problematic because just a number is not a valid identifier. + // GeneratorSubsts::variant_name(variant_index), would be consistent + // with enums? + let variant_name = format!("{}", variant_index.as_usize()).into(); + + let span = generator_layout.variant_source_info[variant_index].span; + let source_info = if !span.is_dummy() { + let loc = cx.lookup_debug_loc(span.lo()); + Some((file_metadata(cx, &loc.file), loc.line)) + } else { + None + }; + + VariantMemberInfo { + variant_index, + variant_name, + variant_struct_type_di_node: + super::build_generator_variant_struct_type_di_node( + cx, + variant_index, + generator_type_and_layout, + generator_type_di_node, + generator_layout, + &state_specific_upvar_names, + &common_upvar_names, + ), + source_info, + } + }) + .collect(); + + smallvec![build_enum_variant_part_di_node( + cx, + generator_type_and_layout, + generator_type_di_node, + &variant_struct_type_di_nodes[..], + )] + }, + // We don't seem to be emitting generic args on the generator type, it seems. Rather + // they get attached to the struct type of each variant. + NO_GENERICS, + ) +} + +/// Builds the DW_TAG_variant_part of an enum or generator debuginfo node: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// ---> DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +fn build_enum_variant_part_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + enum_type_di_node: &'ll DIType, + variant_member_infos: &[VariantMemberInfo<'_, 'll>], +) -> &'ll DIType { + let tag_member_di_node = + build_discr_member_di_node(cx, enum_type_and_layout, enum_type_di_node); + + let variant_part_unique_type_id = + UniqueTypeId::for_enum_variant_part(cx.tcx, enum_type_and_layout.ty); + + let stub = StubInfo::new( + cx, + variant_part_unique_type_id, + |cx, variant_part_unique_type_id_str| unsafe { + let variant_part_name = ""; + llvm::LLVMRustDIBuilderCreateVariantPart( + DIB(cx), + enum_type_di_node, + variant_part_name.as_ptr().cast(), + variant_part_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + enum_type_and_layout.size.bits(), + enum_type_and_layout.align.abi.bits() as u32, + DIFlags::FlagZero, + tag_member_di_node, + create_DIArray(DIB(cx), &[]), + variant_part_unique_type_id_str.as_ptr().cast(), + variant_part_unique_type_id_str.len(), + ) + }, + ); + + type_map::build_type_with_children( + cx, + stub, + |cx, variant_part_di_node| { + variant_member_infos + .iter() + .map(|variant_member_info| { + build_enum_variant_member_di_node( + cx, + enum_type_and_layout, + variant_part_di_node, + variant_member_info, + ) + }) + .collect() + }, + NO_GENERICS, + ) + .di_node +} + +/// Builds the DW_TAG_member describing where we can find the tag of an enum. +/// Returns `None` if the enum does not have a tag. +/// +/// ```txt +/// +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// ---> DW_TAG_member (discriminant member) +/// DW_TAG_variant (variant 1) +/// DW_TAG_variant (variant 2) +/// DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// +/// ``` +fn build_discr_member_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_or_generator_type_and_layout: TyAndLayout<'tcx>, + enum_or_generator_type_di_node: &'ll DIType, +) -> Option<&'ll DIType> { + let tag_name = match enum_or_generator_type_and_layout.ty.kind() { + ty::Generator(..) => "__state", + _ => "", + }; + + // NOTE: This is actually wrong. This will become a member of + // of the DW_TAG_variant_part. But, due to LLVM's API, that + // can only be constructed with this DW_TAG_member already in created. + // In LLVM IR the wrong scope will be listed but when DWARF is + // generated from it, the DW_TAG_member will be a child the + // DW_TAG_variant_part. + let containing_scope = enum_or_generator_type_di_node; + + match enum_or_generator_type_and_layout.layout.variants() { + // A single-variant enum has no discriminant. + &Variants::Single { .. } => None, + + &Variants::Multiple { tag_field, .. } => { + let tag_base_type = tag_base_type(cx, enum_or_generator_type_and_layout); + let (size, align) = cx.size_and_align_of(tag_base_type); + + unsafe { + Some(llvm::LLVMRustDIBuilderCreateMemberType( + DIB(cx), + containing_scope, + tag_name.as_ptr().cast(), + tag_name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + enum_or_generator_type_and_layout.fields.offset(tag_field).bits(), + DIFlags::FlagArtificial, + type_di_node(cx, tag_base_type), + )) + } + } + } +} + +/// Build the debuginfo node for `DW_TAG_variant`: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// ---> DW_TAG_variant (variant 1) +/// ---> DW_TAG_variant (variant 2) +/// ---> DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +/// ``` +/// +/// This node looks like: +/// +/// ```txt +/// DW_TAG_variant +/// DW_AT_discr_value 0 +/// DW_TAG_member +/// DW_AT_name None +/// DW_AT_type <0x000002a1> +/// DW_AT_alignment 0x00000002 +/// DW_AT_data_member_location 0 +/// ``` +/// +/// The DW_AT_discr_value is optional, and is omitted if +/// - This is the only variant of a univariant enum (i.e. their is no discriminant) +/// - This is the "dataful" variant of a niche-layout enum +/// (where only the other variants are identified by a single value) +/// +/// There is only ever a single member, the type of which is a struct that describes the +/// fields of the variant (excluding the discriminant). The name of the member is the name +/// of the variant as given in the source code. The DW_AT_data_member_location is always +/// zero. +/// +/// Note that the LLVM DIBuilder API is a bit unintuitive here. The DW_TAG_variant subtree +/// (including the DW_TAG_member) is built by a single call to +/// `LLVMRustDIBuilderCreateVariantMemberType()`. +fn build_enum_variant_member_di_node<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + enum_type_and_layout: TyAndLayout<'tcx>, + variant_part_di_node: &'ll DIType, + variant_member_info: &VariantMemberInfo<'_, 'll>, +) -> &'ll DIType { + let variant_index = variant_member_info.variant_index; + let discr_value = super::compute_discriminant_value(cx, enum_type_and_layout, variant_index); + + let (file_di_node, line_number) = variant_member_info + .source_info + .unwrap_or_else(|| (unknown_file_metadata(cx), UNKNOWN_LINE_NUMBER)); + + unsafe { + llvm::LLVMRustDIBuilderCreateVariantMemberType( + DIB(cx), + variant_part_di_node, + variant_member_info.variant_name.as_ptr().cast(), + variant_member_info.variant_name.len(), + file_di_node, + line_number, + enum_type_and_layout.size.bits(), + enum_type_and_layout.align.abi.bits() as u32, + Size::ZERO.bits(), + discr_value.map(|v| cx.const_u64(v)), + DIFlags::FlagZero, + variant_member_info.variant_struct_type_di_node, + ) + } +} + +/// Information needed for building a `DW_TAG_variant`: +/// +/// ```txt +/// DW_TAG_structure_type (top-level type for enum) +/// DW_TAG_variant_part (variant part) +/// DW_AT_discr (reference to discriminant DW_TAG_member) +/// DW_TAG_member (discriminant member) +/// ---> DW_TAG_variant (variant 1) +/// ---> DW_TAG_variant (variant 2) +/// ---> DW_TAG_variant (variant 3) +/// DW_TAG_structure_type (type of variant 1) +/// DW_TAG_structure_type (type of variant 2) +/// DW_TAG_structure_type (type of variant 3) +struct VariantMemberInfo<'a, 'll> { + variant_index: VariantIdx, + variant_name: Cow<'a, str>, + variant_struct_type_di_node: &'ll DIType, + source_info: Option<(&'ll DIFile, c_uint)>, +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs new file mode 100644 index 000000000..ce2f419c4 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/metadata/type_map.rs @@ -0,0 +1,267 @@ +use std::cell::RefCell; + +use rustc_data_structures::{ + fingerprint::Fingerprint, + fx::FxHashMap, + stable_hasher::{HashStable, StableHasher}, +}; +use rustc_middle::{ + bug, + ty::{ParamEnv, PolyExistentialTraitRef, Ty, TyCtxt}, +}; +use rustc_target::abi::{Align, Size, VariantIdx}; + +use crate::{ + common::CodegenCx, + debuginfo::utils::{create_DIArray, debug_context, DIB}, + llvm::{ + self, + debuginfo::{DIFlags, DIScope, DIType}, + }, +}; + +use super::{unknown_file_metadata, SmallVec, UNKNOWN_LINE_NUMBER}; + +mod private { + // This type cannot be constructed outside of this module because + // it has a private field. We make use of this in order to prevent + // `UniqueTypeId` from being constructed directly, without asserting + // the preconditions. + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] + pub struct HiddenZst; +} + +/// A unique identifier for anything that we create a debuginfo node for. +/// The types it contains are expected to already be normalized (which +/// is debug_asserted in the constructors). +/// +/// Note that there are some things that only show up in debuginfo, like +/// the separate type descriptions for each enum variant. These get an ID +/// too because they have their own debuginfo node in LLVM IR. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, HashStable)] +pub(super) enum UniqueTypeId<'tcx> { + /// The ID of a regular type as it shows up at the language level. + Ty(Ty<'tcx>, private::HiddenZst), + /// The ID for the single DW_TAG_variant_part nested inside the top-level + /// DW_TAG_structure_type that describes enums and generators. + VariantPart(Ty<'tcx>, private::HiddenZst), + /// The ID for the artificial struct type describing a single enum variant. + VariantStructType(Ty<'tcx>, VariantIdx, private::HiddenZst), + /// The ID of the artificial type we create for VTables. + VTableTy(Ty<'tcx>, Option>, private::HiddenZst), +} + +impl<'tcx> UniqueTypeId<'tcx> { + pub fn for_ty(tcx: TyCtxt<'tcx>, t: Ty<'tcx>) -> Self { + debug_assert_eq!(t, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), t)); + UniqueTypeId::Ty(t, private::HiddenZst) + } + + pub fn for_enum_variant_part(tcx: TyCtxt<'tcx>, enum_ty: Ty<'tcx>) -> Self { + debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + UniqueTypeId::VariantPart(enum_ty, private::HiddenZst) + } + + pub fn for_enum_variant_struct_type( + tcx: TyCtxt<'tcx>, + enum_ty: Ty<'tcx>, + variant_idx: VariantIdx, + ) -> Self { + debug_assert_eq!(enum_ty, tcx.normalize_erasing_regions(ParamEnv::reveal_all(), enum_ty)); + UniqueTypeId::VariantStructType(enum_ty, variant_idx, private::HiddenZst) + } + + pub fn for_vtable_ty( + tcx: TyCtxt<'tcx>, + self_type: Ty<'tcx>, + implemented_trait: Option>, + ) -> Self { + debug_assert_eq!( + self_type, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), self_type) + ); + debug_assert_eq!( + implemented_trait, + tcx.normalize_erasing_regions(ParamEnv::reveal_all(), implemented_trait) + ); + UniqueTypeId::VTableTy(self_type, implemented_trait, private::HiddenZst) + } + + /// Generates a string version of this [UniqueTypeId], which can be used as the `UniqueId` + /// argument of the various `LLVMRustDIBuilderCreate*Type()` methods. + /// + /// Right now this takes the form of a hex-encoded opaque hash value. + pub fn generate_unique_id_string(self, tcx: TyCtxt<'tcx>) -> String { + let mut hasher = StableHasher::new(); + tcx.with_stable_hashing_context(|mut hcx| { + hcx.while_hashing_spans(false, |hcx| self.hash_stable(hcx, &mut hasher)) + }); + hasher.finish::().to_hex() + } + + pub fn expect_ty(self) -> Ty<'tcx> { + match self { + UniqueTypeId::Ty(ty, _) => ty, + _ => bug!("Expected `UniqueTypeId::Ty` but found `{:?}`", self), + } + } +} + +/// The `TypeMap` is where the debug context holds the type metadata nodes +/// created so far. The debuginfo nodes are identified by `UniqueTypeId`. +#[derive(Default)] +pub(crate) struct TypeMap<'ll, 'tcx> { + pub(super) unique_id_to_di_node: RefCell, &'ll DIType>>, +} + +impl<'ll, 'tcx> TypeMap<'ll, 'tcx> { + /// Adds a `UniqueTypeId` to metadata mapping to the `TypeMap`. The method will + /// fail if the mapping already exists. + pub(super) fn insert(&self, unique_type_id: UniqueTypeId<'tcx>, metadata: &'ll DIType) { + if self.unique_id_to_di_node.borrow_mut().insert(unique_type_id, metadata).is_some() { + bug!("type metadata for unique ID '{:?}' is already in the `TypeMap`!", unique_type_id); + } + } + + pub(super) fn di_node_for_unique_id( + &self, + unique_type_id: UniqueTypeId<'tcx>, + ) -> Option<&'ll DIType> { + self.unique_id_to_di_node.borrow().get(&unique_type_id).cloned() + } +} + +pub struct DINodeCreationResult<'ll> { + pub di_node: &'ll DIType, + pub already_stored_in_typemap: bool, +} + +impl<'ll> DINodeCreationResult<'ll> { + pub fn new(di_node: &'ll DIType, already_stored_in_typemap: bool) -> Self { + DINodeCreationResult { di_node, already_stored_in_typemap } + } +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum Stub<'ll> { + Struct, + Union, + VTableTy { vtable_holder: &'ll DIType }, +} + +pub struct StubInfo<'ll, 'tcx> { + metadata: &'ll DIType, + unique_type_id: UniqueTypeId<'tcx>, +} + +impl<'ll, 'tcx> StubInfo<'ll, 'tcx> { + pub(super) fn new( + cx: &CodegenCx<'ll, 'tcx>, + unique_type_id: UniqueTypeId<'tcx>, + build: impl FnOnce(&CodegenCx<'ll, 'tcx>, /* unique_type_id_str: */ &str) -> &'ll DIType, + ) -> StubInfo<'ll, 'tcx> { + let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + let di_node = build(cx, &unique_type_id_str); + StubInfo { metadata: di_node, unique_type_id } + } +} + +/// Create a stub debuginfo node onto which fields and nested types can be attached. +pub(super) fn stub<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + kind: Stub<'ll>, + unique_type_id: UniqueTypeId<'tcx>, + name: &str, + (size, align): (Size, Align), + containing_scope: Option<&'ll DIScope>, + flags: DIFlags, +) -> StubInfo<'ll, 'tcx> { + let empty_array = create_DIArray(DIB(cx), &[]); + let unique_type_id_str = unique_type_id.generate_unique_id_string(cx.tcx); + + let metadata = match kind { + Stub::Struct | Stub::VTableTy { .. } => { + let vtable_holder = match kind { + Stub::VTableTy { vtable_holder } => Some(vtable_holder), + _ => None, + }; + unsafe { + llvm::LLVMRustDIBuilderCreateStructType( + DIB(cx), + containing_scope, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + flags, + None, + empty_array, + 0, + vtable_holder, + unique_type_id_str.as_ptr().cast(), + unique_type_id_str.len(), + ) + } + } + Stub::Union => unsafe { + llvm::LLVMRustDIBuilderCreateUnionType( + DIB(cx), + containing_scope, + name.as_ptr().cast(), + name.len(), + unknown_file_metadata(cx), + UNKNOWN_LINE_NUMBER, + size.bits(), + align.bits() as u32, + flags, + Some(empty_array), + 0, + unique_type_id_str.as_ptr().cast(), + unique_type_id_str.len(), + ) + }, + }; + StubInfo { metadata, unique_type_id } +} + +/// This function enables creating debuginfo nodes that can recursively refer to themselves. +/// It will first insert the given stub into the type map and only then execute the `members` +/// and `generics` closures passed in. These closures have access to the stub so they can +/// directly attach fields to them. If the type of a field transitively refers back +/// to the type currently being built, the stub will already be found in the type map, +/// which effectively breaks the recursion cycle. +pub(super) fn build_type_with_children<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + stub_info: StubInfo<'ll, 'tcx>, + members: impl FnOnce(&CodegenCx<'ll, 'tcx>, &'ll DIType) -> SmallVec<&'ll DIType>, + generics: impl FnOnce(&CodegenCx<'ll, 'tcx>) -> SmallVec<&'ll DIType>, +) -> DINodeCreationResult<'ll> { + debug_assert_eq!( + debug_context(cx).type_map.di_node_for_unique_id(stub_info.unique_type_id), + None + ); + + debug_context(cx).type_map.insert(stub_info.unique_type_id, stub_info.metadata); + + let members: SmallVec<_> = + members(cx, stub_info.metadata).into_iter().map(|node| Some(node)).collect(); + let generics: SmallVec> = + generics(cx).into_iter().map(|node| Some(node)).collect(); + + if !(members.is_empty() && generics.is_empty()) { + unsafe { + let members_array = create_DIArray(DIB(cx), &members[..]); + let generics_array = create_DIArray(DIB(cx), &generics[..]); + llvm::LLVMRustDICompositeTypeReplaceArrays( + DIB(cx), + stub_info.metadata, + Some(members_array), + Some(generics_array), + ); + } + } + + DINodeCreationResult { di_node: stub_info.metadata, already_stored_in_typemap: true } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs new file mode 100644 index 000000000..cf591295b --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/mod.rs @@ -0,0 +1,614 @@ +#![doc = include_str!("doc.md")] + +use rustc_codegen_ssa::mir::debuginfo::VariableKind::*; + +use self::metadata::{file_metadata, type_di_node}; +use self::metadata::{UNKNOWN_COLUMN_NUMBER, UNKNOWN_LINE_NUMBER}; +use self::namespace::mangled_name_of_instance; +use self::utils::{create_DIArray, is_node_local_to_unit, DIB}; + +use crate::abi::FnAbi; +use crate::builder::Builder; +use crate::common::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::{ + DIArray, DIBuilder, DIFile, DIFlags, DILexicalBlock, DILocation, DISPFlags, DIScope, DIType, + DIVariable, +}; +use crate::value::Value; + +use rustc_codegen_ssa::debuginfo::type_names; +use rustc_codegen_ssa::mir::debuginfo::{DebugScope, FunctionDebugContext, VariableKind}; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::fx::FxHashMap; +use rustc_data_structures::sync::Lrc; +use rustc_hir::def_id::{DefId, DefIdMap}; +use rustc_index::vec::IndexVec; +use rustc_middle::mir; +use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::subst::{GenericArgKind, SubstsRef}; +use rustc_middle::ty::{self, Instance, ParamEnv, Ty, TypeVisitable}; +use rustc_session::config::{self, DebugInfo}; +use rustc_session::Session; +use rustc_span::symbol::Symbol; +use rustc_span::{self, BytePos, Pos, SourceFile, SourceFileAndLine, SourceFileHash, Span}; +use rustc_target::abi::Size; + +use libc::c_uint; +use smallvec::SmallVec; +use std::cell::OnceCell; +use std::cell::RefCell; +use std::iter; +use tracing::debug; + +mod create_scope_map; +pub mod gdb; +pub mod metadata; +mod namespace; +mod utils; + +pub use self::create_scope_map::compute_mir_scopes; +pub use self::metadata::build_global_var_di_node; +pub use self::metadata::extend_scope_to_file; + +#[allow(non_upper_case_globals)] +const DW_TAG_auto_variable: c_uint = 0x100; +#[allow(non_upper_case_globals)] +const DW_TAG_arg_variable: c_uint = 0x101; + +/// A context object for maintaining all state needed by the debuginfo module. +pub struct CodegenUnitDebugContext<'ll, 'tcx> { + llcontext: &'ll llvm::Context, + llmod: &'ll llvm::Module, + builder: &'ll mut DIBuilder<'ll>, + created_files: RefCell, &'ll DIFile>>, + + type_map: metadata::TypeMap<'ll, 'tcx>, + namespace_map: RefCell>, + recursion_marker_type: OnceCell<&'ll DIType>, +} + +impl Drop for CodegenUnitDebugContext<'_, '_> { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustDIBuilderDispose(&mut *(self.builder as *mut _)); + } + } +} + +impl<'ll, 'tcx> CodegenUnitDebugContext<'ll, 'tcx> { + pub fn new(llmod: &'ll llvm::Module) -> Self { + debug!("CodegenUnitDebugContext::new"); + let builder = unsafe { llvm::LLVMRustDIBuilderCreate(llmod) }; + // DIBuilder inherits context from the module, so we'd better use the same one + let llcontext = unsafe { llvm::LLVMGetModuleContext(llmod) }; + CodegenUnitDebugContext { + llcontext, + llmod, + builder, + created_files: Default::default(), + type_map: Default::default(), + namespace_map: RefCell::new(Default::default()), + recursion_marker_type: OnceCell::new(), + } + } + + pub fn finalize(&self, sess: &Session) { + unsafe { + llvm::LLVMRustDIBuilderFinalize(self.builder); + + if !sess.target.is_like_msvc { + // Debuginfo generation in LLVM by default uses a higher + // version of dwarf than macOS currently understands. We can + // instruct LLVM to emit an older version of dwarf, however, + // for macOS to understand. For more info see #11352 + // This can be overridden using --llvm-opts -dwarf-version,N. + // Android has the same issue (#22398) + let dwarf_version = sess + .opts + .unstable_opts + .dwarf_version + .unwrap_or(sess.target.default_dwarf_version); + llvm::LLVMRustAddModuleFlag( + self.llmod, + llvm::LLVMModFlagBehavior::Warning, + "Dwarf Version\0".as_ptr().cast(), + dwarf_version, + ); + } else { + // Indicate that we want CodeView debug information on MSVC + llvm::LLVMRustAddModuleFlag( + self.llmod, + llvm::LLVMModFlagBehavior::Warning, + "CodeView\0".as_ptr().cast(), + 1, + ) + } + + // Prevent bitcode readers from deleting the debug info. + let ptr = "Debug Info Version\0".as_ptr(); + llvm::LLVMRustAddModuleFlag( + self.llmod, + llvm::LLVMModFlagBehavior::Warning, + ptr.cast(), + llvm::LLVMRustDebugMetadataVersion(), + ); + } + } +} + +/// Creates any deferred debug metadata nodes +pub fn finalize(cx: &CodegenCx<'_, '_>) { + if let Some(dbg_cx) = &cx.dbg_cx { + debug!("finalize"); + + if gdb::needs_gdb_debug_scripts_section(cx) { + // Add a .debug_gdb_scripts section to this compile-unit. This will + // cause GDB to try and load the gdb_load_rust_pretty_printers.py file, + // which activates the Rust pretty printers for binary this section is + // contained in. + gdb::get_or_insert_gdb_debug_scripts_section_global(cx); + } + + dbg_cx.finalize(cx.sess()); + } +} + +impl<'ll> DebugInfoBuilderMethods for Builder<'_, 'll, '_> { + // FIXME(eddyb) find a common convention for all of the debuginfo-related + // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). + fn dbg_var_addr( + &mut self, + dbg_var: &'ll DIVariable, + dbg_loc: &'ll DILocation, + variable_alloca: Self::Value, + direct_offset: Size, + indirect_offsets: &[Size], + ) { + // Convert the direct and indirect offsets to address ops. + // FIXME(eddyb) use `const`s instead of getting the values via FFI, + // the values should match the ones in the DWARF standard anyway. + let op_deref = || unsafe { llvm::LLVMRustDIBuilderCreateOpDeref() }; + let op_plus_uconst = || unsafe { llvm::LLVMRustDIBuilderCreateOpPlusUconst() }; + let mut addr_ops = SmallVec::<[u64; 8]>::new(); + + if direct_offset.bytes() > 0 { + addr_ops.push(op_plus_uconst()); + addr_ops.push(direct_offset.bytes() as u64); + } + for &offset in indirect_offsets { + addr_ops.push(op_deref()); + if offset.bytes() > 0 { + addr_ops.push(op_plus_uconst()); + addr_ops.push(offset.bytes() as u64); + } + } + + unsafe { + // FIXME(eddyb) replace `llvm.dbg.declare` with `llvm.dbg.addr`. + llvm::LLVMRustDIBuilderInsertDeclareAtEnd( + DIB(self.cx()), + variable_alloca, + dbg_var, + addr_ops.as_ptr(), + addr_ops.len() as c_uint, + dbg_loc, + self.llbb(), + ); + } + } + + fn set_dbg_loc(&mut self, dbg_loc: &'ll DILocation) { + unsafe { + let dbg_loc_as_llval = llvm::LLVMRustMetadataAsValue(self.cx().llcx, dbg_loc); + llvm::LLVMSetCurrentDebugLocation(self.llbuilder, dbg_loc_as_llval); + } + } + + fn insert_reference_to_gdb_debug_scripts_section_global(&mut self) { + gdb::insert_reference_to_gdb_debug_scripts_section_global(self) + } + + fn set_var_name(&mut self, value: &'ll Value, name: &str) { + // Avoid wasting time if LLVM value names aren't even enabled. + if self.sess().fewer_names() { + return; + } + + // Only function parameters and instructions are local to a function, + // don't change the name of anything else (e.g. globals). + let param_or_inst = unsafe { + llvm::LLVMIsAArgument(value).is_some() || llvm::LLVMIsAInstruction(value).is_some() + }; + if !param_or_inst { + return; + } + + // Avoid replacing the name if it already exists. + // While we could combine the names somehow, it'd + // get noisy quick, and the usefulness is dubious. + if llvm::get_value_name(value).is_empty() { + llvm::set_value_name(value, name.as_bytes()); + } + } +} + +/// A source code location used to generate debug information. +// FIXME(eddyb) rename this to better indicate it's a duplicate of +// `rustc_span::Loc` rather than `DILocation`, perhaps by making +// `lookup_char_pos` return the right information instead. +pub struct DebugLoc { + /// Information about the original source file. + pub file: Lrc, + /// The (1-based) line number. + pub line: u32, + /// The (1-based) column number. + pub col: u32, +} + +impl CodegenCx<'_, '_> { + /// Looks up debug source information about a `BytePos`. + // FIXME(eddyb) rename this to better indicate it's a duplicate of + // `lookup_char_pos` rather than `dbg_loc`, perhaps by making + // `lookup_char_pos` return the right information instead. + pub fn lookup_debug_loc(&self, pos: BytePos) -> DebugLoc { + let (file, line, col) = match self.sess().source_map().lookup_line(pos) { + Ok(SourceFileAndLine { sf: file, line }) => { + let line_pos = file.line_begin_pos(pos); + + // Use 1-based indexing. + let line = (line + 1) as u32; + let col = (pos - line_pos).to_u32() + 1; + + (file, line, col) + } + Err(file) => (file, UNKNOWN_LINE_NUMBER, UNKNOWN_COLUMN_NUMBER), + }; + + // For MSVC, omit the column number. + // Otherwise, emit it. This mimics clang behaviour. + // See discussion in https://github.com/rust-lang/rust/issues/42921 + if self.sess().target.is_like_msvc { + DebugLoc { file, line, col: UNKNOWN_COLUMN_NUMBER } + } else { + DebugLoc { file, line, col } + } + } +} + +impl<'ll, 'tcx> DebugInfoMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn create_function_debug_context( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + llfn: &'ll Value, + mir: &mir::Body<'tcx>, + ) -> Option> { + if self.sess().opts.debuginfo == DebugInfo::None { + return None; + } + + // Initialize fn debug context (including scopes). + let empty_scope = DebugScope { + dbg_scope: self.dbg_scope_fn(instance, fn_abi, Some(llfn)), + inlined_at: None, + file_start_pos: BytePos(0), + file_end_pos: BytePos(0), + }; + let mut fn_debug_context = + FunctionDebugContext { scopes: IndexVec::from_elem(empty_scope, &mir.source_scopes) }; + + // Fill in all the scopes, with the information from the MIR body. + compute_mir_scopes(self, instance, mir, &mut fn_debug_context); + + Some(fn_debug_context) + } + + fn dbg_scope_fn( + &self, + instance: Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + maybe_definition_llfn: Option<&'ll Value>, + ) -> &'ll DIScope { + let tcx = self.tcx; + + let def_id = instance.def_id(); + let containing_scope = get_containing_scope(self, instance); + let span = tcx.def_span(def_id); + let loc = self.lookup_debug_loc(span.lo()); + let file_metadata = file_metadata(self, &loc.file); + + let function_type_metadata = unsafe { + let fn_signature = get_function_signature(self, fn_abi); + llvm::LLVMRustDIBuilderCreateSubroutineType(DIB(self), fn_signature) + }; + + let mut name = String::new(); + type_names::push_item_name(tcx, def_id, false, &mut name); + + // Find the enclosing function, in case this is a closure. + let enclosing_fn_def_id = tcx.typeck_root_def_id(def_id); + + // We look up the generics of the enclosing function and truncate the substs + // to their length in order to cut off extra stuff that might be in there for + // closures or generators. + let generics = tcx.generics_of(enclosing_fn_def_id); + let substs = instance.substs.truncate_to(tcx, generics); + + type_names::push_generic_params( + tcx, + tcx.normalize_erasing_regions(ty::ParamEnv::reveal_all(), substs), + &mut name, + ); + + let template_parameters = get_template_parameters(self, generics, substs); + + let linkage_name = &mangled_name_of_instance(self, instance).name; + // Omit the linkage_name if it is the same as subprogram name. + let linkage_name = if &name == linkage_name { "" } else { linkage_name }; + + // FIXME(eddyb) does this need to be separate from `loc.line` for some reason? + let scope_line = loc.line; + + let mut flags = DIFlags::FlagPrototyped; + + if fn_abi.ret.layout.abi.is_uninhabited() { + flags |= DIFlags::FlagNoReturn; + } + + let mut spflags = DISPFlags::SPFlagDefinition; + if is_node_local_to_unit(self, def_id) { + spflags |= DISPFlags::SPFlagLocalToUnit; + } + if self.sess().opts.optimize != config::OptLevel::No { + spflags |= DISPFlags::SPFlagOptimized; + } + if let Some((id, _)) = tcx.entry_fn(()) { + if id == def_id { + spflags |= DISPFlags::SPFlagMainSubprogram; + } + } + + unsafe { + return llvm::LLVMRustDIBuilderCreateFunction( + DIB(self), + containing_scope, + name.as_ptr().cast(), + name.len(), + linkage_name.as_ptr().cast(), + linkage_name.len(), + file_metadata, + loc.line, + function_type_metadata, + scope_line, + flags, + spflags, + maybe_definition_llfn, + template_parameters, + None, + ); + } + + fn get_function_signature<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + ) -> &'ll DIArray { + if cx.sess().opts.debuginfo == DebugInfo::Limited { + return create_DIArray(DIB(cx), &[]); + } + + let mut signature = Vec::with_capacity(fn_abi.args.len() + 1); + + // Return type -- llvm::DIBuilder wants this at index 0 + signature.push(if fn_abi.ret.is_ignore() { + None + } else { + Some(type_di_node(cx, fn_abi.ret.layout.ty)) + }); + + // Arguments types + if cx.sess().target.is_like_msvc { + // FIXME(#42800): + // There is a bug in MSDIA that leads to a crash when it encounters + // a fixed-size array of `u8` or something zero-sized in a + // function-type (see #40477). + // As a workaround, we replace those fixed-size arrays with a + // pointer-type. So a function `fn foo(a: u8, b: [u8; 4])` would + // appear as `fn foo(a: u8, b: *const u8)` in debuginfo, + // and a function `fn bar(x: [(); 7])` as `fn bar(x: *const ())`. + // This transformed type is wrong, but these function types are + // already inaccurate due to ABI adjustments (see #42800). + signature.extend(fn_abi.args.iter().map(|arg| { + let t = arg.layout.ty; + let t = match t.kind() { + ty::Array(ct, _) + if (*ct == cx.tcx.types.u8) || cx.layout_of(*ct).is_zst() => + { + cx.tcx.mk_imm_ptr(*ct) + } + _ => t, + }; + Some(type_di_node(cx, t)) + })); + } else { + signature + .extend(fn_abi.args.iter().map(|arg| Some(type_di_node(cx, arg.layout.ty)))); + } + + create_DIArray(DIB(cx), &signature[..]) + } + + fn get_template_parameters<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + generics: &ty::Generics, + substs: SubstsRef<'tcx>, + ) -> &'ll DIArray { + if substs.types().next().is_none() { + return create_DIArray(DIB(cx), &[]); + } + + // Again, only create type information if full debuginfo is enabled + let template_params: Vec<_> = if cx.sess().opts.debuginfo == DebugInfo::Full { + let names = get_parameter_names(cx, generics); + iter::zip(substs, names) + .filter_map(|(kind, name)| { + if let GenericArgKind::Type(ty) = kind.unpack() { + let actual_type = + cx.tcx.normalize_erasing_regions(ParamEnv::reveal_all(), ty); + let actual_type_metadata = type_di_node(cx, actual_type); + let name = name.as_str(); + Some(unsafe { + Some(llvm::LLVMRustDIBuilderCreateTemplateTypeParameter( + DIB(cx), + None, + name.as_ptr().cast(), + name.len(), + actual_type_metadata, + )) + }) + } else { + None + } + }) + .collect() + } else { + vec![] + }; + + create_DIArray(DIB(cx), &template_params) + } + + fn get_parameter_names(cx: &CodegenCx<'_, '_>, generics: &ty::Generics) -> Vec { + let mut names = generics.parent.map_or_else(Vec::new, |def_id| { + get_parameter_names(cx, cx.tcx.generics_of(def_id)) + }); + names.extend(generics.params.iter().map(|param| param.name)); + names + } + + fn get_containing_scope<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + instance: Instance<'tcx>, + ) -> &'ll DIScope { + // First, let's see if this is a method within an inherent impl. Because + // if yes, we want to make the result subroutine DIE a child of the + // subroutine's self-type. + let self_type = cx.tcx.impl_of_method(instance.def_id()).and_then(|impl_def_id| { + // If the method does *not* belong to a trait, proceed + if cx.tcx.trait_id_of_impl(impl_def_id).is_none() { + let impl_self_ty = cx.tcx.subst_and_normalize_erasing_regions( + instance.substs, + ty::ParamEnv::reveal_all(), + cx.tcx.type_of(impl_def_id), + ); + + // Only "class" methods are generally understood by LLVM, + // so avoid methods on other types (e.g., `<*mut T>::null`). + match impl_self_ty.kind() { + ty::Adt(def, ..) if !def.is_box() => { + // Again, only create type information if full debuginfo is enabled + if cx.sess().opts.debuginfo == DebugInfo::Full + && !impl_self_ty.needs_subst() + { + Some(type_di_node(cx, impl_self_ty)) + } else { + Some(namespace::item_namespace(cx, def.did())) + } + } + _ => None, + } + } else { + // For trait method impls we still use the "parallel namespace" + // strategy + None + } + }); + + self_type.unwrap_or_else(|| { + namespace::item_namespace( + cx, + DefId { + krate: instance.def_id().krate, + index: cx + .tcx + .def_key(instance.def_id()) + .parent + .expect("get_containing_scope: missing parent?"), + }, + ) + }) + } + } + + fn dbg_loc( + &self, + scope: &'ll DIScope, + inlined_at: Option<&'ll DILocation>, + span: Span, + ) -> &'ll DILocation { + let DebugLoc { line, col, .. } = self.lookup_debug_loc(span.lo()); + + unsafe { llvm::LLVMRustDIBuilderCreateDebugLocation(line, col, scope, inlined_at) } + } + + fn create_vtable_debuginfo( + &self, + ty: Ty<'tcx>, + trait_ref: Option>, + vtable: Self::Value, + ) { + metadata::create_vtable_di_node(self, ty, trait_ref, vtable) + } + + fn extend_scope_to_file( + &self, + scope_metadata: &'ll DIScope, + file: &rustc_span::SourceFile, + ) -> &'ll DILexicalBlock { + metadata::extend_scope_to_file(self, scope_metadata, file) + } + + fn debuginfo_finalize(&self) { + finalize(self) + } + + // FIXME(eddyb) find a common convention for all of the debuginfo-related + // names (choose between `dbg`, `debug`, `debuginfo`, `debug_info` etc.). + fn create_dbg_var( + &self, + variable_name: Symbol, + variable_type: Ty<'tcx>, + scope_metadata: &'ll DIScope, + variable_kind: VariableKind, + span: Span, + ) -> &'ll DIVariable { + let loc = self.lookup_debug_loc(span.lo()); + let file_metadata = file_metadata(self, &loc.file); + + let type_metadata = type_di_node(self, variable_type); + + let (argument_index, dwarf_tag) = match variable_kind { + ArgumentVariable(index) => (index as c_uint, DW_TAG_arg_variable), + LocalVariable => (0, DW_TAG_auto_variable), + }; + let align = self.align_of(variable_type); + + let name = variable_name.as_str(); + unsafe { + llvm::LLVMRustDIBuilderCreateVariable( + DIB(self), + dwarf_tag, + scope_metadata, + name.as_ptr().cast(), + name.len(), + file_metadata, + loc.line, + type_metadata, + true, + DIFlags::FlagZero, + argument_index, + align.bytes() as u32, + ) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs new file mode 100644 index 000000000..d5ea48c31 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/namespace.rs @@ -0,0 +1,48 @@ +// Namespace Handling. + +use super::utils::{debug_context, DIB}; +use rustc_codegen_ssa::debuginfo::type_names; +use rustc_middle::ty::{self, Instance}; + +use crate::common::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::DIScope; +use rustc_hir::def_id::DefId; + +pub fn mangled_name_of_instance<'a, 'tcx>( + cx: &CodegenCx<'a, 'tcx>, + instance: Instance<'tcx>, +) -> ty::SymbolName<'tcx> { + let tcx = cx.tcx; + tcx.symbol_name(instance) +} + +pub fn item_namespace<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { + if let Some(&scope) = debug_context(cx).namespace_map.borrow().get(&def_id) { + return scope; + } + + let def_key = cx.tcx.def_key(def_id); + let parent_scope = def_key + .parent + .map(|parent| item_namespace(cx, DefId { krate: def_id.krate, index: parent })); + + let namespace_name_string = { + let mut output = String::new(); + type_names::push_item_name(cx.tcx, def_id, false, &mut output); + output + }; + + let scope = unsafe { + llvm::LLVMRustDIBuilderCreateNameSpace( + DIB(cx), + parent_scope, + namespace_name_string.as_ptr().cast(), + namespace_name_string.len(), + false, // ExportSymbols (only relevant for C++ anonymous namespaces) + ) + }; + + debug_context(cx).namespace_map.borrow_mut().insert(def_id, scope); + scope +} diff --git a/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs new file mode 100644 index 000000000..8f2436739 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/debuginfo/utils.rs @@ -0,0 +1,99 @@ +// Utility Functions. + +use super::namespace::item_namespace; +use super::CodegenUnitDebugContext; + +use rustc_hir::def_id::DefId; +use rustc_middle::ty::layout::{HasParamEnv, LayoutOf}; +use rustc_middle::ty::{self, DefIdTree, Ty}; +use tracing::trace; + +use crate::common::CodegenCx; +use crate::llvm; +use crate::llvm::debuginfo::{DIArray, DIBuilder, DIDescriptor, DIScope}; + +pub fn is_node_local_to_unit(cx: &CodegenCx<'_, '_>, def_id: DefId) -> bool { + // The is_local_to_unit flag indicates whether a function is local to the + // current compilation unit (i.e., if it is *static* in the C-sense). The + // *reachable* set should provide a good approximation of this, as it + // contains everything that might leak out of the current crate (by being + // externally visible or by being inlined into something externally + // visible). It might better to use the `exported_items` set from + // `driver::CrateAnalysis` in the future, but (atm) this set is not + // available in the codegen pass. + !cx.tcx.is_reachable_non_generic(def_id) +} + +#[allow(non_snake_case)] +pub fn create_DIArray<'ll>( + builder: &DIBuilder<'ll>, + arr: &[Option<&'ll DIDescriptor>], +) -> &'ll DIArray { + unsafe { llvm::LLVMRustDIBuilderGetOrCreateArray(builder, arr.as_ptr(), arr.len() as u32) } +} + +#[inline] +pub fn debug_context<'a, 'll, 'tcx>( + cx: &'a CodegenCx<'ll, 'tcx>, +) -> &'a CodegenUnitDebugContext<'ll, 'tcx> { + cx.dbg_cx.as_ref().unwrap() +} + +#[inline] +#[allow(non_snake_case)] +pub fn DIB<'a, 'll>(cx: &'a CodegenCx<'ll, '_>) -> &'a DIBuilder<'ll> { + cx.dbg_cx.as_ref().unwrap().builder +} + +pub fn get_namespace_for_item<'ll>(cx: &CodegenCx<'ll, '_>, def_id: DefId) -> &'ll DIScope { + item_namespace(cx, cx.tcx.parent(def_id)) +} + +#[derive(Debug, PartialEq, Eq)] +pub(crate) enum FatPtrKind { + Slice, + Dyn, +} + +/// Determines if `pointee_ty` is slice-like or trait-object-like, i.e. +/// if the second field of the fat pointer is a length or a vtable-pointer. +/// If `pointee_ty` does not require a fat pointer (because it is Sized) then +/// the function returns `None`. +pub(crate) fn fat_pointer_kind<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + pointee_ty: Ty<'tcx>, +) -> Option { + let pointee_tail_ty = cx.tcx.struct_tail_erasing_lifetimes(pointee_ty, cx.param_env()); + let layout = cx.layout_of(pointee_tail_ty); + trace!( + "fat_pointer_kind: {:?} has layout {:?} (is_unsized? {})", + pointee_tail_ty, + layout, + layout.is_unsized() + ); + + if !layout.is_unsized() { + return None; + } + + match *pointee_tail_ty.kind() { + ty::Str | ty::Slice(_) => Some(FatPtrKind::Slice), + ty::Dynamic(..) => Some(FatPtrKind::Dyn), + ty::Foreign(_) => { + // Assert that pointers to foreign types really are thin: + debug_assert_eq!( + cx.size_of(cx.tcx.mk_imm_ptr(pointee_tail_ty)), + cx.size_of(cx.tcx.mk_imm_ptr(cx.tcx.types.u8)) + ); + None + } + _ => { + // For all other pointee types we should already have returned None + // at the beginning of the function. + panic!( + "fat_pointer_kind() - Encountered unexpected `pointee_tail_ty`: {:?}", + pointee_tail_ty + ) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/declare.rs b/compiler/rustc_codegen_llvm/src/declare.rs new file mode 100644 index 000000000..fa0ecd18f --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/declare.rs @@ -0,0 +1,146 @@ +//! Declare various LLVM values. +//! +//! Prefer using functions and methods from this module rather than calling LLVM +//! functions directly. These functions do some additional work to ensure we do +//! the right thing given the preconceptions of codegen. +//! +//! Some useful guidelines: +//! +//! * Use declare_* family of methods if you are declaring, but are not +//! interested in defining the Value they return. +//! * Use define_* family of methods when you might be defining the Value. +//! * When in doubt, define. + +use crate::abi::{FnAbi, FnAbiLlvmExt}; +use crate::attributes; +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm::AttributePlace::Function; +use crate::type_::Type; +use crate::value::Value; +use rustc_codegen_ssa::traits::TypeMembershipMethods; +use rustc_middle::ty::Ty; +use rustc_symbol_mangling::typeid::typeid_for_fnabi; +use smallvec::SmallVec; +use tracing::debug; + +/// Declare a function. +/// +/// If there’s a value with the same name already declared, the function will +/// update the declaration and return existing Value instead. +fn declare_raw_fn<'ll>( + cx: &CodegenCx<'ll, '_>, + name: &str, + callconv: llvm::CallConv, + unnamed: llvm::UnnamedAddr, + ty: &'ll Type, +) -> &'ll Value { + debug!("declare_raw_fn(name={:?}, ty={:?})", name, ty); + let llfn = unsafe { + llvm::LLVMRustGetOrInsertFunction(cx.llmod, name.as_ptr().cast(), name.len(), ty) + }; + + llvm::SetFunctionCallConv(llfn, callconv); + llvm::SetUnnamedAddress(llfn, unnamed); + + let mut attrs = SmallVec::<[_; 4]>::new(); + + if cx.tcx.sess.opts.cg.no_redzone.unwrap_or(cx.tcx.sess.target.disable_redzone) { + attrs.push(llvm::AttributeKind::NoRedZone.create_attr(cx.llcx)); + } + + attrs.extend(attributes::non_lazy_bind_attr(cx)); + + attributes::apply_to_llfn(llfn, Function, &attrs); + + llfn +} + +impl<'ll, 'tcx> CodegenCx<'ll, 'tcx> { + /// Declare a global value. + /// + /// If there’s a value with the same name already declared, the function will + /// return its Value instead. + pub fn declare_global(&self, name: &str, ty: &'ll Type) -> &'ll Value { + debug!("declare_global(name={:?})", name); + unsafe { llvm::LLVMRustGetOrInsertGlobal(self.llmod, name.as_ptr().cast(), name.len(), ty) } + } + + /// Declare a C ABI function. + /// + /// Only use this for foreign function ABIs and glue. For Rust functions use + /// `declare_fn` instead. + /// + /// If there’s a value with the same name already declared, the function will + /// update the declaration and return existing Value instead. + pub fn declare_cfn( + &self, + name: &str, + unnamed: llvm::UnnamedAddr, + fn_type: &'ll Type, + ) -> &'ll Value { + declare_raw_fn(self, name, llvm::CCallConv, unnamed, fn_type) + } + + /// Declare a Rust function. + /// + /// If there’s a value with the same name already declared, the function will + /// update the declaration and return existing Value instead. + pub fn declare_fn(&self, name: &str, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Value { + debug!("declare_rust_fn(name={:?}, fn_abi={:?})", name, fn_abi); + + // Function addresses in Rust are never significant, allowing functions to + // be merged. + let llfn = declare_raw_fn( + self, + name, + fn_abi.llvm_cconv(), + llvm::UnnamedAddr::Global, + fn_abi.llvm_type(self), + ); + fn_abi.apply_attrs_llfn(self, llfn); + + if self.tcx.sess.is_sanitizer_cfi_enabled() { + let typeid = typeid_for_fnabi(self.tcx, fn_abi); + self.set_type_metadata(llfn, typeid); + } + + llfn + } + + /// Declare a global with an intention to define it. + /// + /// Use this function when you intend to define a global. This function will + /// return `None` if the name already has a definition associated with it. In that + /// case an error should be reported to the user, because it usually happens due + /// to user’s fault (e.g., misuse of `#[no_mangle]` or `#[export_name]` attributes). + pub fn define_global(&self, name: &str, ty: &'ll Type) -> Option<&'ll Value> { + if self.get_defined_value(name).is_some() { + None + } else { + Some(self.declare_global(name, ty)) + } + } + + /// Declare a private global + /// + /// Use this function when you intend to define a global without a name. + pub fn define_private_global(&self, ty: &'ll Type) -> &'ll Value { + unsafe { llvm::LLVMRustInsertPrivateGlobal(self.llmod, ty) } + } + + /// Gets declared value by name. + pub fn get_declared_value(&self, name: &str) -> Option<&'ll Value> { + debug!("get_declared_value(name={:?})", name); + unsafe { llvm::LLVMRustGetNamedValue(self.llmod, name.as_ptr().cast(), name.len()) } + } + + /// Gets defined or externally defined (AvailableExternally linkage) value by + /// name. + pub fn get_defined_value(&self, name: &str) -> Option<&'ll Value> { + self.get_declared_value(name).and_then(|val| { + let declaration = unsafe { llvm::LLVMIsDeclaration(val) != 0 }; + if !declaration { Some(val) } else { None } + }) + } +} diff --git a/compiler/rustc_codegen_llvm/src/intrinsic.rs b/compiler/rustc_codegen_llvm/src/intrinsic.rs new file mode 100644 index 000000000..9f3647492 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/intrinsic.rs @@ -0,0 +1,1924 @@ +use crate::abi::{Abi, FnAbi, FnAbiLlvmExt, LlvmType, PassMode}; +use crate::builder::Builder; +use crate::context::CodegenCx; +use crate::llvm; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::va_arg::emit_va_arg; +use crate::value::Value; + +use rustc_codegen_ssa::base::{compare_simd_types, wants_msvc_seh}; +use rustc_codegen_ssa::common::span_invalid_monomorphization_error; +use rustc_codegen_ssa::common::{IntPredicate, TypeKind}; +use rustc_codegen_ssa::mir::operand::OperandRef; +use rustc_codegen_ssa::mir::place::PlaceRef; +use rustc_codegen_ssa::traits::*; +use rustc_hir as hir; +use rustc_middle::ty::layout::{FnAbiOf, HasTyCtxt, LayoutOf}; +use rustc_middle::ty::{self, Ty}; +use rustc_middle::{bug, span_bug}; +use rustc_span::{sym, symbol::kw, Span, Symbol}; +use rustc_target::abi::{self, Align, HasDataLayout, Primitive}; +use rustc_target::spec::{HasTargetSpec, PanicStrategy}; + +use std::cmp::Ordering; +use std::iter; + +fn get_simple_intrinsic<'ll>( + cx: &CodegenCx<'ll, '_>, + name: Symbol, +) -> Option<(&'ll Type, &'ll Value)> { + let llvm_name = match name { + sym::sqrtf32 => "llvm.sqrt.f32", + sym::sqrtf64 => "llvm.sqrt.f64", + sym::powif32 => "llvm.powi.f32", + sym::powif64 => "llvm.powi.f64", + sym::sinf32 => "llvm.sin.f32", + sym::sinf64 => "llvm.sin.f64", + sym::cosf32 => "llvm.cos.f32", + sym::cosf64 => "llvm.cos.f64", + sym::powf32 => "llvm.pow.f32", + sym::powf64 => "llvm.pow.f64", + sym::expf32 => "llvm.exp.f32", + sym::expf64 => "llvm.exp.f64", + sym::exp2f32 => "llvm.exp2.f32", + sym::exp2f64 => "llvm.exp2.f64", + sym::logf32 => "llvm.log.f32", + sym::logf64 => "llvm.log.f64", + sym::log10f32 => "llvm.log10.f32", + sym::log10f64 => "llvm.log10.f64", + sym::log2f32 => "llvm.log2.f32", + sym::log2f64 => "llvm.log2.f64", + sym::fmaf32 => "llvm.fma.f32", + sym::fmaf64 => "llvm.fma.f64", + sym::fabsf32 => "llvm.fabs.f32", + sym::fabsf64 => "llvm.fabs.f64", + sym::minnumf32 => "llvm.minnum.f32", + sym::minnumf64 => "llvm.minnum.f64", + sym::maxnumf32 => "llvm.maxnum.f32", + sym::maxnumf64 => "llvm.maxnum.f64", + sym::copysignf32 => "llvm.copysign.f32", + sym::copysignf64 => "llvm.copysign.f64", + sym::floorf32 => "llvm.floor.f32", + sym::floorf64 => "llvm.floor.f64", + sym::ceilf32 => "llvm.ceil.f32", + sym::ceilf64 => "llvm.ceil.f64", + sym::truncf32 => "llvm.trunc.f32", + sym::truncf64 => "llvm.trunc.f64", + sym::rintf32 => "llvm.rint.f32", + sym::rintf64 => "llvm.rint.f64", + sym::nearbyintf32 => "llvm.nearbyint.f32", + sym::nearbyintf64 => "llvm.nearbyint.f64", + sym::roundf32 => "llvm.round.f32", + sym::roundf64 => "llvm.round.f64", + _ => return None, + }; + Some(cx.get_intrinsic(llvm_name)) +} + +impl<'ll, 'tcx> IntrinsicCallMethods<'tcx> for Builder<'_, 'll, 'tcx> { + fn codegen_intrinsic_call( + &mut self, + instance: ty::Instance<'tcx>, + fn_abi: &FnAbi<'tcx, Ty<'tcx>>, + args: &[OperandRef<'tcx, &'ll Value>], + llresult: &'ll Value, + span: Span, + ) { + let tcx = self.tcx; + let callee_ty = instance.ty(tcx, ty::ParamEnv::reveal_all()); + + let ty::FnDef(def_id, substs) = *callee_ty.kind() else { + bug!("expected fn item type, found {}", callee_ty); + }; + + let sig = callee_ty.fn_sig(tcx); + let sig = tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig); + let arg_tys = sig.inputs(); + let ret_ty = sig.output(); + let name = tcx.item_name(def_id); + + let llret_ty = self.layout_of(ret_ty).llvm_type(self); + let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); + + let simple = get_simple_intrinsic(self, name); + let llval = match name { + _ if simple.is_some() => { + let (simple_ty, simple_fn) = simple.unwrap(); + self.call( + simple_ty, + simple_fn, + &args.iter().map(|arg| arg.immediate()).collect::>(), + None, + ) + } + sym::likely => { + self.call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(true)]) + } + sym::unlikely => self + .call_intrinsic("llvm.expect.i1", &[args[0].immediate(), self.const_bool(false)]), + kw::Try => { + try_intrinsic( + self, + args[0].immediate(), + args[1].immediate(), + args[2].immediate(), + llresult, + ); + return; + } + sym::breakpoint => self.call_intrinsic("llvm.debugtrap", &[]), + sym::va_copy => { + self.call_intrinsic("llvm.va_copy", &[args[0].immediate(), args[1].immediate()]) + } + sym::va_arg => { + match fn_abi.ret.layout.abi { + abi::Abi::Scalar(scalar) => { + match scalar.primitive() { + Primitive::Int(..) => { + if self.cx().size_of(ret_ty).bytes() < 4 { + // `va_arg` should not be called on an integer type + // less than 4 bytes in length. If it is, promote + // the integer to an `i32` and truncate the result + // back to the smaller type. + let promoted_result = emit_va_arg(self, args[0], tcx.types.i32); + self.trunc(promoted_result, llret_ty) + } else { + emit_va_arg(self, args[0], ret_ty) + } + } + Primitive::F64 | Primitive::Pointer => { + emit_va_arg(self, args[0], ret_ty) + } + // `va_arg` should never be used with the return type f32. + Primitive::F32 => bug!("the va_arg intrinsic does not work with `f32`"), + } + } + _ => bug!("the va_arg intrinsic does not work with non-scalar types"), + } + } + + sym::volatile_load | sym::unaligned_volatile_load => { + let tp_ty = substs.type_at(0); + let ptr = args[0].immediate(); + let load = if let PassMode::Cast(ty) = fn_abi.ret.mode { + let llty = ty.llvm_type(self); + let ptr = self.pointercast(ptr, self.type_ptr_to(llty)); + self.volatile_load(llty, ptr) + } else { + self.volatile_load(self.layout_of(tp_ty).llvm_type(self), ptr) + }; + let align = if name == sym::unaligned_volatile_load { + 1 + } else { + self.align_of(tp_ty).bytes() as u32 + }; + unsafe { + llvm::LLVMSetAlignment(load, align); + } + self.to_immediate(load, self.layout_of(tp_ty)) + } + sym::volatile_store => { + let dst = args[0].deref(self.cx()); + args[1].val.volatile_store(self, dst); + return; + } + sym::unaligned_volatile_store => { + let dst = args[0].deref(self.cx()); + args[1].val.unaligned_volatile_store(self, dst); + return; + } + sym::prefetch_read_data + | sym::prefetch_write_data + | sym::prefetch_read_instruction + | sym::prefetch_write_instruction => { + let (rw, cache_type) = match name { + sym::prefetch_read_data => (0, 1), + sym::prefetch_write_data => (1, 1), + sym::prefetch_read_instruction => (0, 0), + sym::prefetch_write_instruction => (1, 0), + _ => bug!(), + }; + self.call_intrinsic( + "llvm.prefetch", + &[ + args[0].immediate(), + self.const_i32(rw), + args[1].immediate(), + self.const_i32(cache_type), + ], + ) + } + sym::ctlz + | sym::ctlz_nonzero + | sym::cttz + | sym::cttz_nonzero + | sym::ctpop + | sym::bswap + | sym::bitreverse + | sym::rotate_left + | sym::rotate_right + | sym::saturating_add + | sym::saturating_sub => { + let ty = arg_tys[0]; + match int_type_width_signed(ty, self) { + Some((width, signed)) => match name { + sym::ctlz | sym::cttz => { + let y = self.const_bool(false); + self.call_intrinsic( + &format!("llvm.{}.i{}", name, width), + &[args[0].immediate(), y], + ) + } + sym::ctlz_nonzero => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.ctlz.i{}", width); + self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) + } + sym::cttz_nonzero => { + let y = self.const_bool(true); + let llvm_name = &format!("llvm.cttz.i{}", width); + self.call_intrinsic(llvm_name, &[args[0].immediate(), y]) + } + sym::ctpop => self.call_intrinsic( + &format!("llvm.ctpop.i{}", width), + &[args[0].immediate()], + ), + sym::bswap => { + if width == 8 { + args[0].immediate() // byte swap a u8/i8 is just a no-op + } else { + self.call_intrinsic( + &format!("llvm.bswap.i{}", width), + &[args[0].immediate()], + ) + } + } + sym::bitreverse => self.call_intrinsic( + &format!("llvm.bitreverse.i{}", width), + &[args[0].immediate()], + ), + sym::rotate_left | sym::rotate_right => { + let is_left = name == sym::rotate_left; + let val = args[0].immediate(); + let raw_shift = args[1].immediate(); + // rotate = funnel shift with first two args the same + let llvm_name = + &format!("llvm.fsh{}.i{}", if is_left { 'l' } else { 'r' }, width); + self.call_intrinsic(llvm_name, &[val, val, raw_shift]) + } + sym::saturating_add | sym::saturating_sub => { + let is_add = name == sym::saturating_add; + let lhs = args[0].immediate(); + let rhs = args[1].immediate(); + let llvm_name = &format!( + "llvm.{}{}.sat.i{}", + if signed { 's' } else { 'u' }, + if is_add { "add" } else { "sub" }, + width + ); + self.call_intrinsic(llvm_name, &[lhs, rhs]) + } + _ => bug!(), + }, + None => { + span_invalid_monomorphization_error( + tcx.sess, + span, + &format!( + "invalid monomorphization of `{}` intrinsic: \ + expected basic integer type, found `{}`", + name, ty + ), + ); + return; + } + } + } + + sym::raw_eq => { + use abi::Abi::*; + let tp_ty = substs.type_at(0); + let layout = self.layout_of(tp_ty).layout; + let use_integer_compare = match layout.abi() { + Scalar(_) | ScalarPair(_, _) => true, + Uninhabited | Vector { .. } => false, + Aggregate { .. } => { + // For rusty ABIs, small aggregates are actually passed + // as `RegKind::Integer` (see `FnAbi::adjust_for_abi`), + // so we re-use that same threshold here. + layout.size() <= self.data_layout().pointer_size * 2 + } + }; + + let a = args[0].immediate(); + let b = args[1].immediate(); + if layout.size().bytes() == 0 { + self.const_bool(true) + } else if use_integer_compare { + let integer_ty = self.type_ix(layout.size().bits()); + let ptr_ty = self.type_ptr_to(integer_ty); + let a_ptr = self.bitcast(a, ptr_ty); + let a_val = self.load(integer_ty, a_ptr, layout.align().abi); + let b_ptr = self.bitcast(b, ptr_ty); + let b_val = self.load(integer_ty, b_ptr, layout.align().abi); + self.icmp(IntPredicate::IntEQ, a_val, b_val) + } else { + let i8p_ty = self.type_i8p(); + let a_ptr = self.bitcast(a, i8p_ty); + let b_ptr = self.bitcast(b, i8p_ty); + let n = self.const_usize(layout.size().bytes()); + let cmp = self.call_intrinsic("memcmp", &[a_ptr, b_ptr, n]); + match self.cx.sess().target.arch.as_ref() { + "avr" | "msp430" => self.icmp(IntPredicate::IntEQ, cmp, self.const_i16(0)), + _ => self.icmp(IntPredicate::IntEQ, cmp, self.const_i32(0)), + } + } + } + + sym::black_box => { + args[0].val.store(self, result); + + // We need to "use" the argument in some way LLVM can't introspect, and on + // targets that support it we can typically leverage inline assembly to do + // this. LLVM's interpretation of inline assembly is that it's, well, a black + // box. This isn't the greatest implementation since it probably deoptimizes + // more than we want, but it's so far good enough. + crate::asm::inline_asm_call( + self, + "", + "r,~{memory}", + &[result.llval], + self.type_void(), + true, + false, + llvm::AsmDialect::Att, + &[span], + false, + None, + ) + .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`")); + + // We have copied the value to `result` already. + return; + } + + _ if name.as_str().starts_with("simd_") => { + match generic_simd_intrinsic(self, name, callee_ty, args, ret_ty, llret_ty, span) { + Ok(llval) => llval, + Err(()) => return, + } + } + + _ => bug!("unknown intrinsic '{}'", name), + }; + + if !fn_abi.ret.is_ignore() { + if let PassMode::Cast(ty) = fn_abi.ret.mode { + let ptr_llty = self.type_ptr_to(ty.llvm_type(self)); + let ptr = self.pointercast(result.llval, ptr_llty); + self.store(llval, ptr, result.align); + } else { + OperandRef::from_immediate_or_packed_pair(self, llval, result.layout) + .val + .store(self, result); + } + } + } + + fn abort(&mut self) { + self.call_intrinsic("llvm.trap", &[]); + } + + fn assume(&mut self, val: Self::Value) { + self.call_intrinsic("llvm.assume", &[val]); + } + + fn expect(&mut self, cond: Self::Value, expected: bool) -> Self::Value { + self.call_intrinsic("llvm.expect.i1", &[cond, self.const_bool(expected)]) + } + + fn type_test(&mut self, pointer: Self::Value, typeid: Self::Value) -> Self::Value { + // Test the called operand using llvm.type.test intrinsic. The LowerTypeTests link-time + // optimization pass replaces calls to this intrinsic with code to test type membership. + let i8p_ty = self.type_i8p(); + let bitcast = self.bitcast(pointer, i8p_ty); + self.call_intrinsic("llvm.type.test", &[bitcast, typeid]) + } + + fn type_checked_load( + &mut self, + llvtable: &'ll Value, + vtable_byte_offset: u64, + typeid: &'ll Value, + ) -> Self::Value { + let vtable_byte_offset = self.const_i32(vtable_byte_offset as i32); + self.call_intrinsic("llvm.type.checked.load", &[llvtable, vtable_byte_offset, typeid]) + } + + fn va_start(&mut self, va_list: &'ll Value) -> &'ll Value { + self.call_intrinsic("llvm.va_start", &[va_list]) + } + + fn va_end(&mut self, va_list: &'ll Value) -> &'ll Value { + self.call_intrinsic("llvm.va_end", &[va_list]) + } +} + +fn try_intrinsic<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + if bx.sess().panic_strategy() == PanicStrategy::Abort { + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.call(try_func_ty, try_func, &[data], None); + // Return 0 unconditionally from the intrinsic call; + // we can never unwind. + let ret_align = bx.tcx().data_layout.i32_align.abi; + bx.store(bx.const_i32(0), dest, ret_align); + } else if wants_msvc_seh(bx.sess()) { + codegen_msvc_try(bx, try_func, data, catch_func, dest); + } else if bx.sess().target.os == "emscripten" { + codegen_emcc_try(bx, try_func, data, catch_func, dest); + } else { + codegen_gnu_try(bx, try_func, data, catch_func, dest); + } +} + +// MSVC's definition of the `rust_try` function. +// +// This implementation uses the new exception handling instructions in LLVM +// which have support in LLVM for SEH on MSVC targets. Although these +// instructions are meant to work for all targets, as of the time of this +// writing, however, LLVM does not recommend the usage of these new instructions +// as the old ones are still more optimized. +fn codegen_msvc_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { + bx.set_personality_fn(bx.eh_personality()); + + let normal = bx.append_sibling_block("normal"); + let catchswitch = bx.append_sibling_block("catchswitch"); + let catchpad_rust = bx.append_sibling_block("catchpad_rust"); + let catchpad_foreign = bx.append_sibling_block("catchpad_foreign"); + let caught = bx.append_sibling_block("caught"); + + let try_func = llvm::get_param(bx.llfn(), 0); + let data = llvm::get_param(bx.llfn(), 1); + let catch_func = llvm::get_param(bx.llfn(), 2); + + // We're generating an IR snippet that looks like: + // + // declare i32 @rust_try(%try_func, %data, %catch_func) { + // %slot = alloca i8* + // invoke %try_func(%data) to label %normal unwind label %catchswitch + // + // normal: + // ret i32 0 + // + // catchswitch: + // %cs = catchswitch within none [%catchpad_rust, %catchpad_foreign] unwind to caller + // + // catchpad_rust: + // %tok = catchpad within %cs [%type_descriptor, 8, %slot] + // %ptr = load %slot + // call %catch_func(%data, %ptr) + // catchret from %tok to label %caught + // + // catchpad_foreign: + // %tok = catchpad within %cs [null, 64, null] + // call %catch_func(%data, null) + // catchret from %tok to label %caught + // + // caught: + // ret i32 1 + // } + // + // This structure follows the basic usage of throw/try/catch in LLVM. + // For example, compile this C++ snippet to see what LLVM generates: + // + // struct rust_panic { + // rust_panic(const rust_panic&); + // ~rust_panic(); + // + // void* x[2]; + // }; + // + // int __rust_try( + // void (*try_func)(void*), + // void *data, + // void (*catch_func)(void*, void*) noexcept + // ) { + // try { + // try_func(data); + // return 0; + // } catch(rust_panic& a) { + // catch_func(data, &a); + // return 1; + // } catch(...) { + // catch_func(data, NULL); + // return 1; + // } + // } + // + // More information can be found in libstd's seh.rs implementation. + let ptr_align = bx.tcx().data_layout.pointer_align.abi; + let slot = bx.alloca(bx.type_i8p(), ptr_align); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, try_func, &[data], normal, catchswitch, None); + + bx.switch_to_block(normal); + bx.ret(bx.const_i32(0)); + + bx.switch_to_block(catchswitch); + let cs = bx.catch_switch(None, None, &[catchpad_rust, catchpad_foreign]); + + // We can't use the TypeDescriptor defined in libpanic_unwind because it + // might be in another DLL and the SEH encoding only supports specifying + // a TypeDescriptor from the current module. + // + // However this isn't an issue since the MSVC runtime uses string + // comparison on the type name to match TypeDescriptors rather than + // pointer equality. + // + // So instead we generate a new TypeDescriptor in each module that uses + // `try` and let the linker merge duplicate definitions in the same + // module. + // + // When modifying, make sure that the type_name string exactly matches + // the one used in src/libpanic_unwind/seh.rs. + let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_i8p()); + let type_name = bx.const_bytes(b"rust_panic\0"); + let type_info = + bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_i8p()), type_name], false); + let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info)); + unsafe { + llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage); + llvm::SetUniqueComdat(bx.llmod, tydesc); + llvm::LLVMSetInitializer(tydesc, type_info); + } + + // The flag value of 8 indicates that we are catching the exception by + // reference instead of by value. We can't use catch by value because + // that requires copying the exception object, which we don't support + // since our exception object effectively contains a Box. + // + // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang + bx.switch_to_block(catchpad_rust); + let flags = bx.const_i32(8); + let funclet = bx.catch_pad(cs, &[tydesc, flags, slot]); + let ptr = bx.load(bx.type_i8p(), slot, ptr_align); + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + bx.call(catch_ty, catch_func, &[data, ptr], Some(&funclet)); + bx.catch_ret(&funclet, caught); + + // The flag value of 64 indicates a "catch-all". + bx.switch_to_block(catchpad_foreign); + let flags = bx.const_i32(64); + let null = bx.const_null(bx.type_i8p()); + let funclet = bx.catch_pad(cs, &[null, flags, null]); + bx.call(catch_ty, catch_func, &[data, null], Some(&funclet)); + bx.catch_ret(&funclet, caught); + + bx.switch_to_block(caught); + bx.ret(bx.const_i32(1)); + }); + + // Note that no invoke is used here because by definition this function + // can't panic (that's what it's catching). + let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let i32_align = bx.tcx().data_layout.i32_align.abi; + bx.store(ret, dest, i32_align); +} + +// Definition of the standard `try` function for Rust using the GNU-like model +// of exceptions (e.g., the normal semantics of LLVM's `landingpad` and `invoke` +// instructions). +// +// This codegen is a little surprising because we always call a shim +// function instead of inlining the call to `invoke` manually here. This is done +// because in LLVM we're only allowed to have one personality per function +// definition. The call to the `try` intrinsic is being inlined into the +// function calling it, and that function may already have other personality +// functions in play. By calling a shim we're guaranteed that our shim will have +// the right personality function. +fn codegen_gnu_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { + // Codegens the shims described above: + // + // bx: + // invoke %try_func(%data) normal %normal unwind %catch + // + // normal: + // ret 0 + // + // catch: + // (%ptr, _) = landingpad + // call %catch_func(%data, %ptr) + // ret 1 + let then = bx.append_sibling_block("then"); + let catch = bx.append_sibling_block("catch"); + + let try_func = llvm::get_param(bx.llfn(), 0); + let data = llvm::get_param(bx.llfn(), 1); + let catch_func = llvm::get_param(bx.llfn(), 2); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + + bx.switch_to_block(then); + bx.ret(bx.const_i32(0)); + + // Type indicator for the exception being thrown. + // + // The first value in this tuple is a pointer to the exception object + // being thrown. The second value is a "selector" indicating which of + // the landing pad clauses the exception's type had been matched to. + // rust_try ignores the selector. + bx.switch_to_block(catch); + let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); + let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 1); + let tydesc = bx.const_null(bx.type_i8p()); + bx.add_clause(vals, tydesc); + let ptr = bx.extract_value(vals, 0); + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + bx.call(catch_ty, catch_func, &[data, ptr], None); + bx.ret(bx.const_i32(1)); + }); + + // Note that no invoke is used here because by definition this function + // can't panic (that's what it's catching). + let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let i32_align = bx.tcx().data_layout.i32_align.abi; + bx.store(ret, dest, i32_align); +} + +// Variant of codegen_gnu_try used for emscripten where Rust panics are +// implemented using C++ exceptions. Here we use exceptions of a specific type +// (`struct rust_panic`) to represent Rust panics. +fn codegen_emcc_try<'ll>( + bx: &mut Builder<'_, 'll, '_>, + try_func: &'ll Value, + data: &'ll Value, + catch_func: &'ll Value, + dest: &'ll Value, +) { + let (llty, llfn) = get_rust_try_fn(bx, &mut |mut bx| { + // Codegens the shims described above: + // + // bx: + // invoke %try_func(%data) normal %normal unwind %catch + // + // normal: + // ret 0 + // + // catch: + // (%ptr, %selector) = landingpad + // %rust_typeid = @llvm.eh.typeid.for(@_ZTI10rust_panic) + // %is_rust_panic = %selector == %rust_typeid + // %catch_data = alloca { i8*, i8 } + // %catch_data[0] = %ptr + // %catch_data[1] = %is_rust_panic + // call %catch_func(%data, %catch_data) + // ret 1 + let then = bx.append_sibling_block("then"); + let catch = bx.append_sibling_block("catch"); + + let try_func = llvm::get_param(bx.llfn(), 0); + let data = llvm::get_param(bx.llfn(), 1); + let catch_func = llvm::get_param(bx.llfn(), 2); + let try_func_ty = bx.type_func(&[bx.type_i8p()], bx.type_void()); + bx.invoke(try_func_ty, try_func, &[data], then, catch, None); + + bx.switch_to_block(then); + bx.ret(bx.const_i32(0)); + + // Type indicator for the exception being thrown. + // + // The first value in this tuple is a pointer to the exception object + // being thrown. The second value is a "selector" indicating which of + // the landing pad clauses the exception's type had been matched to. + bx.switch_to_block(catch); + let tydesc = bx.eh_catch_typeinfo(); + let lpad_ty = bx.type_struct(&[bx.type_i8p(), bx.type_i32()], false); + let vals = bx.landing_pad(lpad_ty, bx.eh_personality(), 2); + bx.add_clause(vals, tydesc); + bx.add_clause(vals, bx.const_null(bx.type_i8p())); + let ptr = bx.extract_value(vals, 0); + let selector = bx.extract_value(vals, 1); + + // Check if the typeid we got is the one for a Rust panic. + let rust_typeid = bx.call_intrinsic("llvm.eh.typeid.for", &[tydesc]); + let is_rust_panic = bx.icmp(IntPredicate::IntEQ, selector, rust_typeid); + let is_rust_panic = bx.zext(is_rust_panic, bx.type_bool()); + + // We need to pass two values to catch_func (ptr and is_rust_panic), so + // create an alloca and pass a pointer to that. + let ptr_align = bx.tcx().data_layout.pointer_align.abi; + let i8_align = bx.tcx().data_layout.i8_align.abi; + let catch_data_type = bx.type_struct(&[bx.type_i8p(), bx.type_bool()], false); + let catch_data = bx.alloca(catch_data_type, ptr_align); + let catch_data_0 = + bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(0)]); + bx.store(ptr, catch_data_0, ptr_align); + let catch_data_1 = + bx.inbounds_gep(catch_data_type, catch_data, &[bx.const_usize(0), bx.const_usize(1)]); + bx.store(is_rust_panic, catch_data_1, i8_align); + let catch_data = bx.bitcast(catch_data, bx.type_i8p()); + + let catch_ty = bx.type_func(&[bx.type_i8p(), bx.type_i8p()], bx.type_void()); + bx.call(catch_ty, catch_func, &[data, catch_data], None); + bx.ret(bx.const_i32(1)); + }); + + // Note that no invoke is used here because by definition this function + // can't panic (that's what it's catching). + let ret = bx.call(llty, llfn, &[try_func, data, catch_func], None); + let i32_align = bx.tcx().data_layout.i32_align.abi; + bx.store(ret, dest, i32_align); +} + +// Helper function to give a Block to a closure to codegen a shim function. +// This is currently primarily used for the `try` intrinsic functions above. +fn gen_fn<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + name: &str, + rust_fn_sig: ty::PolyFnSig<'tcx>, + codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), +) -> (&'ll Type, &'ll Value) { + let fn_abi = cx.fn_abi_of_fn_ptr(rust_fn_sig, ty::List::empty()); + let llty = fn_abi.llvm_type(cx); + let llfn = cx.declare_fn(name, fn_abi); + cx.set_frame_pointer_type(llfn); + cx.apply_target_cpu_attr(llfn); + // FIXME(eddyb) find a nicer way to do this. + unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) }; + let llbb = Builder::append_block(cx, llfn, "entry-block"); + let bx = Builder::build(cx, llbb); + codegen(bx); + (llty, llfn) +} + +// Helper function used to get a handle to the `__rust_try` function used to +// catch exceptions. +// +// This function is only generated once and is then cached. +fn get_rust_try_fn<'ll, 'tcx>( + cx: &CodegenCx<'ll, 'tcx>, + codegen: &mut dyn FnMut(Builder<'_, 'll, 'tcx>), +) -> (&'ll Type, &'ll Value) { + if let Some(llfn) = cx.rust_try_fn.get() { + return llfn; + } + + // Define the type up front for the signature of the rust_try function. + let tcx = cx.tcx; + let i8p = tcx.mk_mut_ptr(tcx.types.i8); + // `unsafe fn(*mut i8) -> ()` + let try_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( + iter::once(i8p), + tcx.mk_unit(), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + ))); + // `unsafe fn(*mut i8, *mut i8) -> ()` + let catch_fn_ty = tcx.mk_fn_ptr(ty::Binder::dummy(tcx.mk_fn_sig( + [i8p, i8p].iter().cloned(), + tcx.mk_unit(), + false, + hir::Unsafety::Unsafe, + Abi::Rust, + ))); + // `unsafe fn(unsafe fn(*mut i8) -> (), *mut i8, unsafe fn(*mut i8, *mut i8) -> ()) -> i32` + let rust_fn_sig = ty::Binder::dummy(cx.tcx.mk_fn_sig( + [try_fn_ty, i8p, catch_fn_ty].into_iter(), + tcx.types.i32, + false, + hir::Unsafety::Unsafe, + Abi::Rust, + )); + let rust_try = gen_fn(cx, "__rust_try", rust_fn_sig, codegen); + cx.rust_try_fn.set(Some(rust_try)); + rust_try +} + +fn generic_simd_intrinsic<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + name: Symbol, + callee_ty: Ty<'tcx>, + args: &[OperandRef<'tcx, &'ll Value>], + ret_ty: Ty<'tcx>, + llret_ty: &'ll Type, + span: Span, +) -> Result<&'ll Value, ()> { + // macros for error handling: + #[allow(unused_macro_rules)] + macro_rules! emit_error { + ($msg: tt) => { + emit_error!($msg, ) + }; + ($msg: tt, $($fmt: tt)*) => { + span_invalid_monomorphization_error( + bx.sess(), span, + &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), + name, $($fmt)*)); + } + } + + macro_rules! return_error { + ($($fmt: tt)*) => { + { + emit_error!($($fmt)*); + return Err(()); + } + } + } + + macro_rules! require { + ($cond: expr, $($fmt: tt)*) => { + if !$cond { + return_error!($($fmt)*); + } + }; + } + + macro_rules! require_simd { + ($ty: expr, $position: expr) => { + require!($ty.is_simd(), "expected SIMD {} type, found non-SIMD `{}`", $position, $ty) + }; + } + + let tcx = bx.tcx(); + let sig = + tcx.normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), callee_ty.fn_sig(tcx)); + let arg_tys = sig.inputs(); + + if name == sym::simd_select_bitmask { + require_simd!(arg_tys[1], "argument"); + let (len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + + let expected_int_bits = (len.max(8) - 1).next_power_of_two(); + let expected_bytes = len / 8 + ((len % 8 > 0) as u64); + + let mask_ty = arg_tys[0]; + let mask = match mask_ty.kind() { + ty::Int(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => args[0].immediate(), + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + let place = PlaceRef::alloca(bx, args[0].layout); + args[0].val.store(bx, place); + let int_ty = bx.type_ix(expected_bytes * 8); + let ptr = bx.pointercast(place.llval, bx.cx.type_ptr_to(int_ty)); + bx.load(int_ty, ptr, Align::ONE) + } + _ => return_error!( + "invalid bitmask `{}`, expected `u{}` or `[u8; {}]`", + mask_ty, + expected_int_bits, + expected_bytes + ), + }; + + let i1 = bx.type_i1(); + let im = bx.type_ix(len); + let i1xn = bx.type_vector(i1, len); + let m_im = bx.trunc(mask, im); + let m_i1s = bx.bitcast(m_im, i1xn); + return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); + } + + // every intrinsic below takes a SIMD vector as its first argument + require_simd!(arg_tys[0], "input"); + let in_ty = arg_tys[0]; + + let comparison = match name { + sym::simd_eq => Some(hir::BinOpKind::Eq), + sym::simd_ne => Some(hir::BinOpKind::Ne), + sym::simd_lt => Some(hir::BinOpKind::Lt), + sym::simd_le => Some(hir::BinOpKind::Le), + sym::simd_gt => Some(hir::BinOpKind::Gt), + sym::simd_ge => Some(hir::BinOpKind::Ge), + _ => None, + }; + + let (in_len, in_elem) = arg_tys[0].simd_size_and_type(bx.tcx()); + if let Some(cmp_op) = comparison { + require_simd!(ret_ty, "return"); + + let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + "expected return type with length {} (same as input type `{}`), \ + found `{}` with length {}", + in_len, + in_ty, + ret_ty, + out_len + ); + require!( + bx.type_kind(bx.element_type(llret_ty)) == TypeKind::Integer, + "expected return type with integer elements, found `{}` with non-integer `{}`", + ret_ty, + out_ty + ); + + return Ok(compare_simd_types( + bx, + args[0].immediate(), + args[1].immediate(), + in_elem, + llret_ty, + cmp_op, + )); + } + + if let Some(stripped) = name.as_str().strip_prefix("simd_shuffle") { + // If this intrinsic is the older "simd_shuffleN" form, simply parse the integer. + // If there is no suffix, use the index array length. + let n: u64 = if stripped.is_empty() { + // Make sure this is actually an array, since typeck only checks the length-suffixed + // version of this intrinsic. + match args[2].layout.ty.kind() { + ty::Array(ty, len) if matches!(ty.kind(), ty::Uint(ty::UintTy::U32)) => { + len.try_eval_usize(bx.cx.tcx, ty::ParamEnv::reveal_all()).unwrap_or_else(|| { + span_bug!(span, "could not evaluate shuffle index array length") + }) + } + _ => return_error!( + "simd_shuffle index must be an array of `u32`, got `{}`", + args[2].layout.ty + ), + } + } else { + stripped.parse().unwrap_or_else(|_| { + span_bug!(span, "bad `simd_shuffle` instruction only caught in codegen?") + }) + }; + + require_simd!(ret_ty, "return"); + let (out_len, out_ty) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + out_len == n, + "expected return type of length {}, found `{}` with length {}", + n, + ret_ty, + out_len + ); + require!( + in_elem == out_ty, + "expected return element type `{}` (element of input `{}`), \ + found `{}` with element type `{}`", + in_elem, + in_ty, + ret_ty, + out_ty + ); + + let total_len = u128::from(in_len) * 2; + + let vector = args[2].immediate(); + + let indices: Option> = (0..n) + .map(|i| { + let arg_idx = i; + let val = bx.const_get_elt(vector, i as u64); + match bx.const_to_opt_u128(val, true) { + None => { + emit_error!("shuffle index #{} is not a constant", arg_idx); + None + } + Some(idx) if idx >= total_len => { + emit_error!( + "shuffle index #{} is out of bounds (limit {})", + arg_idx, + total_len + ); + None + } + Some(idx) => Some(bx.const_i32(idx as i32)), + } + }) + .collect(); + let Some(indices) = indices else { + return Ok(bx.const_null(llret_ty)); + }; + + return Ok(bx.shuffle_vector( + args[0].immediate(), + args[1].immediate(), + bx.const_vector(&indices), + )); + } + + if name == sym::simd_insert { + require!( + in_elem == arg_tys[2], + "expected inserted type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + arg_tys[2] + ); + return Ok(bx.insert_element( + args[0].immediate(), + args[2].immediate(), + args[1].immediate(), + )); + } + if name == sym::simd_extract { + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); + return Ok(bx.extract_element(args[0].immediate(), args[1].immediate())); + } + + if name == sym::simd_select { + let m_elem_ty = in_elem; + let m_len = in_len; + require_simd!(arg_tys[1], "argument"); + let (v_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + require!( + m_len == v_len, + "mismatched lengths: mask length `{}` != other vector length `{}`", + m_len, + v_len + ); + match m_elem_ty.kind() { + ty::Int(_) => {} + _ => return_error!("mask element type is `{}`, expected `i_`", m_elem_ty), + } + // truncate the mask to a vector of i1s + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, m_len as u64); + let m_i1s = bx.trunc(args[0].immediate(), i1xn); + return Ok(bx.select(m_i1s, args[1].immediate(), args[2].immediate())); + } + + if name == sym::simd_bitmask { + // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a + // vector mask and returns the most significant bit (MSB) of each lane in the form + // of either: + // * an unsigned integer + // * an array of `u8` + // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits. + // + // The bit order of the result depends on the byte endianness, LSB-first for little + // endian and MSB-first for big endian. + let expected_int_bits = in_len.max(8); + let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64); + + // Integer vector : + let (i_xn, in_elem_bitwidth) = match in_elem.kind() { + ty::Int(i) => ( + args[0].immediate(), + i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), + ), + ty::Uint(i) => ( + args[0].immediate(), + i.bit_width().unwrap_or_else(|| bx.data_layout().pointer_size.bits()), + ), + _ => return_error!( + "vector argument `{}`'s element type `{}`, expected integer element type", + in_ty, + in_elem + ), + }; + + // Shift the MSB to the right by "in_elem_bitwidth - 1" into the first bit position. + let shift_indices = + vec![ + bx.cx.const_int(bx.type_ix(in_elem_bitwidth), (in_elem_bitwidth - 1) as _); + in_len as _ + ]; + let i_xn_msb = bx.lshr(i_xn, bx.const_vector(shift_indices.as_slice())); + // Truncate vector to an + let i1xn = bx.trunc(i_xn_msb, bx.type_vector(bx.type_i1(), in_len)); + // Bitcast to iN: + let i_ = bx.bitcast(i1xn, bx.type_ix(in_len)); + + match ret_ty.kind() { + ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => { + // Zero-extend iN to the bitmask type: + return Ok(bx.zext(i_, bx.type_ix(expected_int_bits))); + } + ty::Array(elem, len) + if matches!(elem.kind(), ty::Uint(ty::UintTy::U8)) + && len.try_eval_usize(bx.tcx, ty::ParamEnv::reveal_all()) + == Some(expected_bytes) => + { + // Zero-extend iN to the array length: + let ze = bx.zext(i_, bx.type_ix(expected_bytes * 8)); + + // Convert the integer to a byte array + let ptr = bx.alloca(bx.type_ix(expected_bytes * 8), Align::ONE); + bx.store(ze, ptr, Align::ONE); + let array_ty = bx.type_array(bx.type_i8(), expected_bytes); + let ptr = bx.pointercast(ptr, bx.cx.type_ptr_to(array_ty)); + return Ok(bx.load(array_ty, ptr, Align::ONE)); + } + _ => return_error!( + "cannot return `{}`, expected `u{}` or `[u8; {}]`", + ret_ty, + expected_int_bits, + expected_bytes + ), + } + } + + fn simd_simple_float_intrinsic<'ll, 'tcx>( + name: Symbol, + in_elem: Ty<'_>, + in_ty: Ty<'_>, + in_len: u64, + bx: &mut Builder<'_, 'll, 'tcx>, + span: Span, + args: &[OperandRef<'tcx, &'ll Value>], + ) -> Result<&'ll Value, ()> { + #[allow(unused_macro_rules)] + macro_rules! emit_error { + ($msg: tt) => { + emit_error!($msg, ) + }; + ($msg: tt, $($fmt: tt)*) => { + span_invalid_monomorphization_error( + bx.sess(), span, + &format!(concat!("invalid monomorphization of `{}` intrinsic: ", $msg), + name, $($fmt)*)); + } + } + macro_rules! return_error { + ($($fmt: tt)*) => { + { + emit_error!($($fmt)*); + return Err(()); + } + } + } + + let (elem_ty_str, elem_ty) = if let ty::Float(f) = in_elem.kind() { + let elem_ty = bx.cx.type_float_from_ty(*f); + match f.bit_width() { + 32 => ("f32", elem_ty), + 64 => ("f64", elem_ty), + _ => { + return_error!( + "unsupported element type `{}` of floating-point vector `{}`", + f.name_str(), + in_ty + ); + } + } + } else { + return_error!("`{}` is not a floating-point type", in_ty); + }; + + let vec_ty = bx.type_vector(elem_ty, in_len); + + let (intr_name, fn_ty) = match name { + sym::simd_ceil => ("ceil", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fabs => ("fabs", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fcos => ("cos", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fexp2 => ("exp2", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fexp => ("exp", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_flog10 => ("log10", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_flog2 => ("log2", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_flog => ("log", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_floor => ("floor", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fma => ("fma", bx.type_func(&[vec_ty, vec_ty, vec_ty], vec_ty)), + sym::simd_fpowi => ("powi", bx.type_func(&[vec_ty, bx.type_i32()], vec_ty)), + sym::simd_fpow => ("pow", bx.type_func(&[vec_ty, vec_ty], vec_ty)), + sym::simd_fsin => ("sin", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_fsqrt => ("sqrt", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_round => ("round", bx.type_func(&[vec_ty], vec_ty)), + sym::simd_trunc => ("trunc", bx.type_func(&[vec_ty], vec_ty)), + _ => return_error!("unrecognized intrinsic `{}`", name), + }; + let llvm_name = &format!("llvm.{0}.v{1}{2}", intr_name, in_len, elem_ty_str); + let f = bx.declare_cfn(llvm_name, llvm::UnnamedAddr::No, fn_ty); + let c = + bx.call(fn_ty, f, &args.iter().map(|arg| arg.immediate()).collect::>(), None); + Ok(c) + } + + if std::matches!( + name, + sym::simd_ceil + | sym::simd_fabs + | sym::simd_fcos + | sym::simd_fexp2 + | sym::simd_fexp + | sym::simd_flog10 + | sym::simd_flog2 + | sym::simd_flog + | sym::simd_floor + | sym::simd_fma + | sym::simd_fpow + | sym::simd_fpowi + | sym::simd_fsin + | sym::simd_fsqrt + | sym::simd_round + | sym::simd_trunc + ) { + return simd_simple_float_intrinsic(name, in_elem, in_ty, in_len, bx, span, args); + } + + // FIXME: use: + // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Function.h#L182 + // https://github.com/llvm-mirror/llvm/blob/master/include/llvm/IR/Intrinsics.h#L81 + fn llvm_vector_str( + elem_ty: Ty<'_>, + vec_len: u64, + no_pointers: usize, + bx: &Builder<'_, '_, '_>, + ) -> String { + let p0s: String = "p0".repeat(no_pointers); + match *elem_ty.kind() { + ty::Int(v) => format!( + "v{}{}i{}", + vec_len, + p0s, + // Normalize to prevent crash if v: IntTy::Isize + v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() + ), + ty::Uint(v) => format!( + "v{}{}i{}", + vec_len, + p0s, + // Normalize to prevent crash if v: UIntTy::Usize + v.normalize(bx.target_spec().pointer_width).bit_width().unwrap() + ), + ty::Float(v) => format!("v{}{}f{}", vec_len, p0s, v.bit_width()), + _ => unreachable!(), + } + } + + fn llvm_vector_ty<'ll>( + cx: &CodegenCx<'ll, '_>, + elem_ty: Ty<'_>, + vec_len: u64, + mut no_pointers: usize, + ) -> &'ll Type { + // FIXME: use cx.layout_of(ty).llvm_type() ? + let mut elem_ty = match *elem_ty.kind() { + ty::Int(v) => cx.type_int_from_ty(v), + ty::Uint(v) => cx.type_uint_from_ty(v), + ty::Float(v) => cx.type_float_from_ty(v), + _ => unreachable!(), + }; + while no_pointers > 0 { + elem_ty = cx.type_ptr_to(elem_ty); + no_pointers -= 1; + } + cx.type_vector(elem_ty, vec_len) + } + + if name == sym::simd_gather { + // simd_gather(values: , pointers: , + // mask: ) -> + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + + // All types must be simd vector types + require_simd!(in_ty, "first"); + require_simd!(arg_tys[1], "second"); + require_simd!(arg_tys[2], "third"); + require_simd!(ret_ty, "return"); + + // Of the same length: + let (out_len, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (out_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "second", + in_len, + in_ty, + arg_tys[1], + out_len + ); + require!( + in_len == out_len2, + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "third", + in_len, + in_ty, + arg_tys[2], + out_len2 + ); + + // The return type must match the first argument type + require!(ret_ty == in_ty, "expected return type `{}`, found `{}`", in_ty, ret_ty); + + // This counts how many pointers + fn ptr_count(t: Ty<'_>) -> usize { + match t.kind() { + ty::RawPtr(p) => 1 + ptr_count(p.ty), + _ => 0, + } + } + + // Non-ptr type + fn non_ptr(t: Ty<'_>) -> Ty<'_> { + match t.kind() { + ty::RawPtr(p) => non_ptr(p.ty), + _ => t, + } + } + + // The second argument must be a simd vector with an element type that's a pointer + // to the element type of the first argument + let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); + let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (pointer_count, underlying_ty) = match element_ty1.kind() { + ty::RawPtr(p) if p.ty == in_elem => (ptr_count(element_ty1), non_ptr(element_ty1)), + _ => { + require!( + false, + "expected element type `{}` of second argument `{}` \ + to be a pointer to the element type `{}` of the first \ + argument `{}`, found `{}` != `*_ {}`", + element_ty1, + arg_tys[1], + in_elem, + in_ty, + element_ty1, + in_elem + ); + unreachable!(); + } + }; + assert!(pointer_count > 0); + assert_eq!(pointer_count - 1, ptr_count(element_ty0)); + assert_eq!(underlying_ty, non_ptr(element_ty0)); + + // The element type of the third argument must be a signed integer type of any width: + let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); + match element_ty2.kind() { + ty::Int(_) => (), + _ => { + require!( + false, + "expected element type `{}` of third argument `{}` \ + to be a signed integer type", + element_ty2, + arg_tys[2] + ); + } + } + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, in_len); + (bx.trunc(args[2].immediate(), i1xn), i1xn) + }; + + // Type of the vector of pointers: + let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); + let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); + let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); + + let llvm_intrinsic = + format!("llvm.masked.gather.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); + let fn_ty = bx.type_func( + &[llvm_pointer_vec_ty, alignment_ty, mask_ty, llvm_elem_vec_ty], + llvm_elem_vec_ty, + ); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = + bx.call(fn_ty, f, &[args[1].immediate(), alignment, mask, args[0].immediate()], None); + return Ok(v); + } + + if name == sym::simd_scatter { + // simd_scatter(values: , pointers: , + // mask: ) -> () + // * N: number of elements in the input vectors + // * T: type of the element to load + // * M: any integer width is supported, will be truncated to i1 + + // All types must be simd vector types + require_simd!(in_ty, "first"); + require_simd!(arg_tys[1], "second"); + require_simd!(arg_tys[2], "third"); + + // Of the same length: + let (element_len1, _) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (element_len2, _) = arg_tys[2].simd_size_and_type(bx.tcx()); + require!( + in_len == element_len1, + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "second", + in_len, + in_ty, + arg_tys[1], + element_len1 + ); + require!( + in_len == element_len2, + "expected {} argument with length {} (same as input type `{}`), \ + found `{}` with length {}", + "third", + in_len, + in_ty, + arg_tys[2], + element_len2 + ); + + // This counts how many pointers + fn ptr_count(t: Ty<'_>) -> usize { + match t.kind() { + ty::RawPtr(p) => 1 + ptr_count(p.ty), + _ => 0, + } + } + + // Non-ptr type + fn non_ptr(t: Ty<'_>) -> Ty<'_> { + match t.kind() { + ty::RawPtr(p) => non_ptr(p.ty), + _ => t, + } + } + + // The second argument must be a simd vector with an element type that's a pointer + // to the element type of the first argument + let (_, element_ty0) = arg_tys[0].simd_size_and_type(bx.tcx()); + let (_, element_ty1) = arg_tys[1].simd_size_and_type(bx.tcx()); + let (_, element_ty2) = arg_tys[2].simd_size_and_type(bx.tcx()); + let (pointer_count, underlying_ty) = match element_ty1.kind() { + ty::RawPtr(p) if p.ty == in_elem && p.mutbl == hir::Mutability::Mut => { + (ptr_count(element_ty1), non_ptr(element_ty1)) + } + _ => { + require!( + false, + "expected element type `{}` of second argument `{}` \ + to be a pointer to the element type `{}` of the first \ + argument `{}`, found `{}` != `*mut {}`", + element_ty1, + arg_tys[1], + in_elem, + in_ty, + element_ty1, + in_elem + ); + unreachable!(); + } + }; + assert!(pointer_count > 0); + assert_eq!(pointer_count - 1, ptr_count(element_ty0)); + assert_eq!(underlying_ty, non_ptr(element_ty0)); + + // The element type of the third argument must be a signed integer type of any width: + match element_ty2.kind() { + ty::Int(_) => (), + _ => { + require!( + false, + "expected element type `{}` of third argument `{}` \ + be a signed integer type", + element_ty2, + arg_tys[2] + ); + } + } + + // Alignment of T, must be a constant integer value: + let alignment_ty = bx.type_i32(); + let alignment = bx.const_i32(bx.align_of(in_elem).bytes() as i32); + + // Truncate the mask vector to a vector of i1s: + let (mask, mask_ty) = { + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, in_len); + (bx.trunc(args[2].immediate(), i1xn), i1xn) + }; + + let ret_t = bx.type_void(); + + // Type of the vector of pointers: + let llvm_pointer_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count); + let llvm_pointer_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count, bx); + + // Type of the vector of elements: + let llvm_elem_vec_ty = llvm_vector_ty(bx, underlying_ty, in_len, pointer_count - 1); + let llvm_elem_vec_str = llvm_vector_str(underlying_ty, in_len, pointer_count - 1, bx); + + let llvm_intrinsic = + format!("llvm.masked.scatter.{}.{}", llvm_elem_vec_str, llvm_pointer_vec_str); + let fn_ty = + bx.type_func(&[llvm_elem_vec_ty, llvm_pointer_vec_ty, alignment_ty, mask_ty], ret_t); + let f = bx.declare_cfn(&llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = + bx.call(fn_ty, f, &[args[0].immediate(), args[1].immediate(), alignment, mask], None); + return Ok(v); + } + + macro_rules! arith_red { + ($name:ident : $integer_reduce:ident, $float_reduce:ident, $ordered:expr, $op:ident, + $identity:expr) => { + if name == sym::$name { + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); + return match in_elem.kind() { + ty::Int(_) | ty::Uint(_) => { + let r = bx.$integer_reduce(args[0].immediate()); + if $ordered { + // if overflow occurs, the result is the + // mathematical result modulo 2^n: + Ok(bx.$op(args[1].immediate(), r)) + } else { + Ok(bx.$integer_reduce(args[0].immediate())) + } + } + ty::Float(f) => { + let acc = if $ordered { + // ordered arithmetic reductions take an accumulator + args[1].immediate() + } else { + // unordered arithmetic reductions use the identity accumulator + match f.bit_width() { + 32 => bx.const_real(bx.type_f32(), $identity), + 64 => bx.const_real(bx.type_f64(), $identity), + v => return_error!( + r#" +unsupported {} from `{}` with element `{}` of size `{}` to `{}`"#, + sym::$name, + in_ty, + in_elem, + v, + ret_ty + ), + } + }; + Ok(bx.$float_reduce(acc, args[0].immediate())) + } + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + sym::$name, + in_ty, + in_elem, + ret_ty + ), + }; + } + }; + } + + arith_red!(simd_reduce_add_ordered: vector_reduce_add, vector_reduce_fadd, true, add, 0.0); + arith_red!(simd_reduce_mul_ordered: vector_reduce_mul, vector_reduce_fmul, true, mul, 1.0); + arith_red!( + simd_reduce_add_unordered: vector_reduce_add, + vector_reduce_fadd_fast, + false, + add, + 0.0 + ); + arith_red!( + simd_reduce_mul_unordered: vector_reduce_mul, + vector_reduce_fmul_fast, + false, + mul, + 1.0 + ); + + macro_rules! minmax_red { + ($name:ident: $int_red:ident, $float_red:ident) => { + if name == sym::$name { + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); + return match in_elem.kind() { + ty::Int(_i) => Ok(bx.$int_red(args[0].immediate(), true)), + ty::Uint(_u) => Ok(bx.$int_red(args[0].immediate(), false)), + ty::Float(_f) => Ok(bx.$float_red(args[0].immediate())), + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + sym::$name, + in_ty, + in_elem, + ret_ty + ), + }; + } + }; + } + + minmax_red!(simd_reduce_min: vector_reduce_min, vector_reduce_fmin); + minmax_red!(simd_reduce_max: vector_reduce_max, vector_reduce_fmax); + + minmax_red!(simd_reduce_min_nanless: vector_reduce_min, vector_reduce_fmin_fast); + minmax_red!(simd_reduce_max_nanless: vector_reduce_max, vector_reduce_fmax_fast); + + macro_rules! bitwise_red { + ($name:ident : $red:ident, $boolean:expr) => { + if name == sym::$name { + let input = if !$boolean { + require!( + ret_ty == in_elem, + "expected return type `{}` (element of input `{}`), found `{}`", + in_elem, + in_ty, + ret_ty + ); + args[0].immediate() + } else { + match in_elem.kind() { + ty::Int(_) | ty::Uint(_) => {} + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + sym::$name, + in_ty, + in_elem, + ret_ty + ), + } + + // boolean reductions operate on vectors of i1s: + let i1 = bx.type_i1(); + let i1xn = bx.type_vector(i1, in_len as u64); + bx.trunc(args[0].immediate(), i1xn) + }; + return match in_elem.kind() { + ty::Int(_) | ty::Uint(_) => { + let r = bx.$red(input); + Ok(if !$boolean { r } else { bx.zext(r, bx.type_bool()) }) + } + _ => return_error!( + "unsupported {} from `{}` with element `{}` to `{}`", + sym::$name, + in_ty, + in_elem, + ret_ty + ), + }; + } + }; + } + + bitwise_red!(simd_reduce_and: vector_reduce_and, false); + bitwise_red!(simd_reduce_or: vector_reduce_or, false); + bitwise_red!(simd_reduce_xor: vector_reduce_xor, false); + bitwise_red!(simd_reduce_all: vector_reduce_and, true); + bitwise_red!(simd_reduce_any: vector_reduce_or, true); + + if name == sym::simd_cast || name == sym::simd_as { + require_simd!(ret_ty, "return"); + let (out_len, out_elem) = ret_ty.simd_size_and_type(bx.tcx()); + require!( + in_len == out_len, + "expected return type with length {} (same as input type `{}`), \ + found `{}` with length {}", + in_len, + in_ty, + ret_ty, + out_len + ); + // casting cares about nominal type, not just structural type + if in_elem == out_elem { + return Ok(args[0].immediate()); + } + + enum Style { + Float, + Int(/* is signed? */ bool), + Unsupported, + } + + let (in_style, in_width) = match in_elem.kind() { + // vectors of pointer-sized integers should've been + // disallowed before here, so this unwrap is safe. + ty::Int(i) => ( + Style::Int(true), + i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Uint(u) => ( + Style::Int(false), + u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Float(f) => (Style::Float, f.bit_width()), + _ => (Style::Unsupported, 0), + }; + let (out_style, out_width) = match out_elem.kind() { + ty::Int(i) => ( + Style::Int(true), + i.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Uint(u) => ( + Style::Int(false), + u.normalize(bx.tcx().sess.target.pointer_width).bit_width().unwrap(), + ), + ty::Float(f) => (Style::Float, f.bit_width()), + _ => (Style::Unsupported, 0), + }; + + match (in_style, out_style) { + (Style::Int(in_is_signed), Style::Int(_)) => { + return Ok(match in_width.cmp(&out_width) { + Ordering::Greater => bx.trunc(args[0].immediate(), llret_ty), + Ordering::Equal => args[0].immediate(), + Ordering::Less => { + if in_is_signed { + bx.sext(args[0].immediate(), llret_ty) + } else { + bx.zext(args[0].immediate(), llret_ty) + } + } + }); + } + (Style::Int(in_is_signed), Style::Float) => { + return Ok(if in_is_signed { + bx.sitofp(args[0].immediate(), llret_ty) + } else { + bx.uitofp(args[0].immediate(), llret_ty) + }); + } + (Style::Float, Style::Int(out_is_signed)) => { + return Ok(match (out_is_signed, name == sym::simd_as) { + (false, false) => bx.fptoui(args[0].immediate(), llret_ty), + (true, false) => bx.fptosi(args[0].immediate(), llret_ty), + (_, true) => bx.cast_float_to_int(out_is_signed, args[0].immediate(), llret_ty), + }); + } + (Style::Float, Style::Float) => { + return Ok(match in_width.cmp(&out_width) { + Ordering::Greater => bx.fptrunc(args[0].immediate(), llret_ty), + Ordering::Equal => args[0].immediate(), + Ordering::Less => bx.fpext(args[0].immediate(), llret_ty), + }); + } + _ => { /* Unsupported. Fallthrough. */ } + } + require!( + false, + "unsupported cast from `{}` with element `{}` to `{}` with element `{}`", + in_ty, + in_elem, + ret_ty, + out_elem + ); + } + macro_rules! arith_binary { + ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { + $(if name == sym::$name { + match in_elem.kind() { + $($(ty::$p(_))|* => { + return Ok(bx.$call(args[0].immediate(), args[1].immediate())) + })* + _ => {}, + } + require!(false, + "unsupported operation on `{}` with element `{}`", + in_ty, + in_elem) + })* + } + } + arith_binary! { + simd_add: Uint, Int => add, Float => fadd; + simd_sub: Uint, Int => sub, Float => fsub; + simd_mul: Uint, Int => mul, Float => fmul; + simd_div: Uint => udiv, Int => sdiv, Float => fdiv; + simd_rem: Uint => urem, Int => srem, Float => frem; + simd_shl: Uint, Int => shl; + simd_shr: Uint => lshr, Int => ashr; + simd_and: Uint, Int => and; + simd_or: Uint, Int => or; + simd_xor: Uint, Int => xor; + simd_fmax: Float => maxnum; + simd_fmin: Float => minnum; + + } + macro_rules! arith_unary { + ($($name: ident: $($($p: ident),* => $call: ident),*;)*) => { + $(if name == sym::$name { + match in_elem.kind() { + $($(ty::$p(_))|* => { + return Ok(bx.$call(args[0].immediate())) + })* + _ => {}, + } + require!(false, + "unsupported operation on `{}` with element `{}`", + in_ty, + in_elem) + })* + } + } + arith_unary! { + simd_neg: Int => neg, Float => fneg; + } + + if name == sym::simd_arith_offset { + // This also checks that the first operand is a ptr type. + let pointee = in_elem.builtin_deref(true).unwrap_or_else(|| { + span_bug!(span, "must be called with a vector of pointer types as first argument") + }); + let layout = bx.layout_of(pointee.ty); + let ptrs = args[0].immediate(); + // The second argument must be a ptr-sized integer. + // (We don't care about the signedness, this is wrapping anyway.) + let (_offsets_len, offsets_elem) = arg_tys[1].simd_size_and_type(bx.tcx()); + if !matches!(offsets_elem.kind(), ty::Int(ty::IntTy::Isize) | ty::Uint(ty::UintTy::Usize)) { + span_bug!( + span, + "must be called with a vector of pointer-sized integers as second argument" + ); + } + let offsets = args[1].immediate(); + + return Ok(bx.gep(bx.backend_type(layout), ptrs, &[offsets])); + } + + if name == sym::simd_saturating_add || name == sym::simd_saturating_sub { + let lhs = args[0].immediate(); + let rhs = args[1].immediate(); + let is_add = name == sym::simd_saturating_add; + let ptr_bits = bx.tcx().data_layout.pointer_size.bits() as _; + let (signed, elem_width, elem_ty) = match *in_elem.kind() { + ty::Int(i) => (true, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_int_from_ty(i)), + ty::Uint(i) => (false, i.bit_width().unwrap_or(ptr_bits), bx.cx.type_uint_from_ty(i)), + _ => { + return_error!( + "expected element type `{}` of vector type `{}` \ + to be a signed or unsigned integer type", + arg_tys[0].simd_size_and_type(bx.tcx()).1, + arg_tys[0] + ); + } + }; + let llvm_intrinsic = &format!( + "llvm.{}{}.sat.v{}i{}", + if signed { 's' } else { 'u' }, + if is_add { "add" } else { "sub" }, + in_len, + elem_width + ); + let vec_ty = bx.cx.type_vector(elem_ty, in_len as u64); + + let fn_ty = bx.type_func(&[vec_ty, vec_ty], vec_ty); + let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty); + let v = bx.call(fn_ty, f, &[lhs, rhs], None); + return Ok(v); + } + + span_bug!(span, "unknown SIMD intrinsic"); +} + +// Returns the width of an int Ty, and if it's signed or not +// Returns None if the type is not an integer +// FIXME: there’s multiple of this functions, investigate using some of the already existing +// stuffs. +fn int_type_width_signed(ty: Ty<'_>, cx: &CodegenCx<'_, '_>) -> Option<(u64, bool)> { + match ty.kind() { + ty::Int(t) => { + Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), true)) + } + ty::Uint(t) => { + Some((t.bit_width().unwrap_or(u64::from(cx.tcx.sess.target.pointer_width)), false)) + } + _ => None, + } +} diff --git a/compiler/rustc_codegen_llvm/src/lib.rs b/compiler/rustc_codegen_llvm/src/lib.rs new file mode 100644 index 000000000..636d689a3 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/lib.rs @@ -0,0 +1,442 @@ +//! The Rust compiler. +//! +//! # Note +//! +//! This API is completely unstable and subject to change. + +#![doc(html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/")] +#![feature(hash_raw_entry)] +#![feature(let_chains)] +#![feature(let_else)] +#![feature(extern_types)] +#![feature(once_cell)] +#![feature(iter_intersperse)] +#![recursion_limit = "256"] +#![allow(rustc::potential_query_instability)] + +#[macro_use] +extern crate rustc_macros; + +use back::write::{create_informational_target_machine, create_target_machine}; + +pub use llvm_util::target_features; +use rustc_ast::expand::allocator::AllocatorKind; +use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule}; +use rustc_codegen_ssa::back::write::{ + CodegenContext, FatLTOInput, ModuleConfig, TargetMachineFactoryConfig, TargetMachineFactoryFn, +}; +use rustc_codegen_ssa::traits::*; +use rustc_codegen_ssa::ModuleCodegen; +use rustc_codegen_ssa::{CodegenResults, CompiledModule}; +use rustc_data_structures::fx::FxHashMap; +use rustc_errors::{ErrorGuaranteed, FatalError, Handler}; +use rustc_metadata::EncodedMetadata; +use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; +use rustc_middle::ty::query::Providers; +use rustc_middle::ty::TyCtxt; +use rustc_session::config::{OptLevel, OutputFilenames, PrintRequest}; +use rustc_session::Session; +use rustc_span::symbol::Symbol; + +use std::any::Any; +use std::ffi::CStr; + +mod back { + pub mod archive; + pub mod lto; + mod profiling; + pub mod write; +} + +mod abi; +mod allocator; +mod asm; +mod attributes; +mod base; +mod builder; +mod callee; +mod common; +mod consts; +mod context; +mod coverageinfo; +mod debuginfo; +mod declare; +mod intrinsic; + +// The following is a work around that replaces `pub mod llvm;` and that fixes issue 53912. +#[path = "llvm/mod.rs"] +mod llvm_; +pub mod llvm { + pub use super::llvm_::*; +} + +mod llvm_util; +mod mono_item; +mod type_; +mod type_of; +mod va_arg; +mod value; + +#[derive(Clone)] +pub struct LlvmCodegenBackend(()); + +struct TimeTraceProfiler { + enabled: bool, +} + +impl TimeTraceProfiler { + fn new(enabled: bool) -> Self { + if enabled { + unsafe { llvm::LLVMTimeTraceProfilerInitialize() } + } + TimeTraceProfiler { enabled } + } +} + +impl Drop for TimeTraceProfiler { + fn drop(&mut self) { + if self.enabled { + unsafe { llvm::LLVMTimeTraceProfilerFinishThread() } + } + } +} + +impl ExtraBackendMethods for LlvmCodegenBackend { + fn codegen_allocator<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + module_name: &str, + kind: AllocatorKind, + has_alloc_error_handler: bool, + ) -> ModuleLlvm { + let mut module_llvm = ModuleLlvm::new_metadata(tcx, module_name); + unsafe { + allocator::codegen(tcx, &mut module_llvm, module_name, kind, has_alloc_error_handler); + } + module_llvm + } + fn compile_codegen_unit( + &self, + tcx: TyCtxt<'_>, + cgu_name: Symbol, + ) -> (ModuleCodegen, u64) { + base::compile_codegen_unit(tcx, cgu_name) + } + fn target_machine_factory( + &self, + sess: &Session, + optlvl: OptLevel, + target_features: &[String], + ) -> TargetMachineFactoryFn { + back::write::target_machine_factory(sess, optlvl, target_features) + } + fn target_cpu<'b>(&self, sess: &'b Session) -> &'b str { + llvm_util::target_cpu(sess) + } + fn tune_cpu<'b>(&self, sess: &'b Session) -> Option<&'b str> { + llvm_util::tune_cpu(sess) + } + + fn spawn_thread(time_trace: bool, f: F) -> std::thread::JoinHandle + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::spawn(move || { + let _profiler = TimeTraceProfiler::new(time_trace); + f() + }) + } + + fn spawn_named_thread( + time_trace: bool, + name: String, + f: F, + ) -> std::io::Result> + where + F: FnOnce() -> T, + F: Send + 'static, + T: Send + 'static, + { + std::thread::Builder::new().name(name).spawn(move || { + let _profiler = TimeTraceProfiler::new(time_trace); + f() + }) + } +} + +impl WriteBackendMethods for LlvmCodegenBackend { + type Module = ModuleLlvm; + type ModuleBuffer = back::lto::ModuleBuffer; + type Context = llvm::Context; + type TargetMachine = &'static mut llvm::TargetMachine; + type ThinData = back::lto::ThinData; + type ThinBuffer = back::lto::ThinBuffer; + fn print_pass_timings(&self) { + unsafe { + llvm::LLVMRustPrintPassTimings(); + } + } + fn run_link( + cgcx: &CodegenContext, + diag_handler: &Handler, + modules: Vec>, + ) -> Result, FatalError> { + back::write::link(cgcx, diag_handler, modules) + } + fn run_fat_lto( + cgcx: &CodegenContext, + modules: Vec>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + ) -> Result, FatalError> { + back::lto::run_fat(cgcx, modules, cached_modules) + } + fn run_thin_lto( + cgcx: &CodegenContext, + modules: Vec<(String, Self::ThinBuffer)>, + cached_modules: Vec<(SerializedModule, WorkProduct)>, + ) -> Result<(Vec>, Vec), FatalError> { + back::lto::run_thin(cgcx, modules, cached_modules) + } + unsafe fn optimize( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: &ModuleCodegen, + config: &ModuleConfig, + ) -> Result<(), FatalError> { + back::write::optimize(cgcx, diag_handler, module, config) + } + fn optimize_fat( + cgcx: &CodegenContext, + module: &mut ModuleCodegen, + ) -> Result<(), FatalError> { + let diag_handler = cgcx.create_diag_handler(); + back::lto::run_pass_manager(cgcx, &diag_handler, module, false) + } + unsafe fn optimize_thin( + cgcx: &CodegenContext, + thin: ThinModule, + ) -> Result, FatalError> { + back::lto::optimize_thin_module(thin, cgcx) + } + unsafe fn codegen( + cgcx: &CodegenContext, + diag_handler: &Handler, + module: ModuleCodegen, + config: &ModuleConfig, + ) -> Result { + back::write::codegen(cgcx, diag_handler, module, config) + } + fn prepare_thin(module: ModuleCodegen) -> (String, Self::ThinBuffer) { + back::lto::prepare_thin(module) + } + fn serialize_module(module: ModuleCodegen) -> (String, Self::ModuleBuffer) { + (module.name, back::lto::ModuleBuffer::new(module.module_llvm.llmod())) + } +} + +unsafe impl Send for LlvmCodegenBackend {} // Llvm is on a per-thread basis +unsafe impl Sync for LlvmCodegenBackend {} + +impl LlvmCodegenBackend { + pub fn new() -> Box { + Box::new(LlvmCodegenBackend(())) + } +} + +impl CodegenBackend for LlvmCodegenBackend { + fn init(&self, sess: &Session) { + llvm_util::init(sess); // Make sure llvm is inited + } + + fn provide(&self, providers: &mut Providers) { + providers.global_backend_features = + |tcx, ()| llvm_util::global_llvm_features(tcx.sess, true) + } + + fn print(&self, req: PrintRequest, sess: &Session) { + match req { + PrintRequest::RelocationModels => { + println!("Available relocation models:"); + for name in &[ + "static", + "pic", + "pie", + "dynamic-no-pic", + "ropi", + "rwpi", + "ropi-rwpi", + "default", + ] { + println!(" {}", name); + } + println!(); + } + PrintRequest::CodeModels => { + println!("Available code models:"); + for name in &["tiny", "small", "kernel", "medium", "large"] { + println!(" {}", name); + } + println!(); + } + PrintRequest::TlsModels => { + println!("Available TLS models:"); + for name in &["global-dynamic", "local-dynamic", "initial-exec", "local-exec"] { + println!(" {}", name); + } + println!(); + } + PrintRequest::StackProtectorStrategies => { + println!( + r#"Available stack protector strategies: + all + Generate stack canaries in all functions. + + strong + Generate stack canaries in a function if it either: + - has a local variable of `[T; N]` type, regardless of `T` and `N` + - takes the address of a local variable. + + (Note that a local variable being borrowed is not equivalent to its + address being taken: e.g. some borrows may be removed by optimization, + while by-value argument passing may be implemented with reference to a + local stack variable in the ABI.) + + basic + Generate stack canaries in functions with local variables of `[T; N]` + type, where `T` is byte-sized and `N` >= 8. + + none + Do not generate stack canaries. +"# + ); + } + req => llvm_util::print(req, sess), + } + } + + fn print_passes(&self) { + llvm_util::print_passes(); + } + + fn print_version(&self) { + llvm_util::print_version(); + } + + fn target_features(&self, sess: &Session, allow_unstable: bool) -> Vec { + target_features(sess, allow_unstable) + } + + fn codegen_crate<'tcx>( + &self, + tcx: TyCtxt<'tcx>, + metadata: EncodedMetadata, + need_metadata_module: bool, + ) -> Box { + Box::new(rustc_codegen_ssa::base::codegen_crate( + LlvmCodegenBackend(()), + tcx, + crate::llvm_util::target_cpu(tcx.sess).to_string(), + metadata, + need_metadata_module, + )) + } + + fn join_codegen( + &self, + ongoing_codegen: Box, + sess: &Session, + outputs: &OutputFilenames, + ) -> Result<(CodegenResults, FxHashMap), ErrorGuaranteed> { + let (codegen_results, work_products) = ongoing_codegen + .downcast::>() + .expect("Expected LlvmCodegenBackend's OngoingCodegen, found Box") + .join(sess); + + sess.time("llvm_dump_timing_file", || { + if sess.opts.unstable_opts.llvm_time_trace { + let file_name = outputs.with_extension("llvm_timings.json"); + llvm_util::time_trace_profiler_finish(&file_name); + } + }); + + Ok((codegen_results, work_products)) + } + + fn link( + &self, + sess: &Session, + codegen_results: CodegenResults, + outputs: &OutputFilenames, + ) -> Result<(), ErrorGuaranteed> { + use crate::back::archive::LlvmArchiveBuilderBuilder; + use rustc_codegen_ssa::back::link::link_binary; + + // Run the linker on any artifacts that resulted from the LLVM run. + // This should produce either a finished executable or library. + link_binary(sess, &LlvmArchiveBuilderBuilder, &codegen_results, outputs) + } +} + +pub struct ModuleLlvm { + llcx: &'static mut llvm::Context, + llmod_raw: *const llvm::Module, + tm: &'static mut llvm::TargetMachine, +} + +unsafe impl Send for ModuleLlvm {} +unsafe impl Sync for ModuleLlvm {} + +impl ModuleLlvm { + fn new(tcx: TyCtxt<'_>, mod_name: &str) -> Self { + unsafe { + let llcx = llvm::LLVMRustContextCreate(tcx.sess.fewer_names()); + let llmod_raw = context::create_module(tcx, llcx, mod_name) as *const _; + ModuleLlvm { llmod_raw, llcx, tm: create_target_machine(tcx, mod_name) } + } + } + + fn new_metadata(tcx: TyCtxt<'_>, mod_name: &str) -> Self { + unsafe { + let llcx = llvm::LLVMRustContextCreate(tcx.sess.fewer_names()); + let llmod_raw = context::create_module(tcx, llcx, mod_name) as *const _; + ModuleLlvm { llmod_raw, llcx, tm: create_informational_target_machine(tcx.sess) } + } + } + + fn parse( + cgcx: &CodegenContext, + name: &CStr, + buffer: &[u8], + handler: &Handler, + ) -> Result { + unsafe { + let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names); + let llmod_raw = back::lto::parse_module(llcx, name, buffer, handler)?; + let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, name.to_str().unwrap()); + let tm = match (cgcx.tm_factory)(tm_factory_config) { + Ok(m) => m, + Err(e) => { + handler.struct_err(&e).emit(); + return Err(FatalError); + } + }; + + Ok(ModuleLlvm { llmod_raw, llcx, tm }) + } + } + + fn llmod(&self) -> &llvm::Module { + unsafe { &*self.llmod_raw } + } +} + +impl Drop for ModuleLlvm { + fn drop(&mut self) { + unsafe { + llvm::LLVMRustDisposeTargetMachine(&mut *(self.tm as *mut _)); + llvm::LLVMContextDispose(&mut *(self.llcx as *mut _)); + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs new file mode 100644 index 000000000..64db4f746 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/archive_ro.rs @@ -0,0 +1,105 @@ +//! A wrapper around LLVM's archive (.a) code + +use rustc_fs_util::path_to_c_string; +use std::path::Path; +use std::slice; +use std::str; + +pub struct ArchiveRO { + pub raw: &'static mut super::Archive, +} + +unsafe impl Send for ArchiveRO {} + +pub struct Iter<'a> { + raw: &'a mut super::ArchiveIterator<'a>, +} + +pub struct Child<'a> { + pub raw: &'a mut super::ArchiveChild<'a>, +} + +impl ArchiveRO { + /// Opens a static archive for read-only purposes. This is more optimized + /// than the `open` method because it uses LLVM's internal `Archive` class + /// rather than shelling out to `ar` for everything. + /// + /// If this archive is used with a mutable method, then an error will be + /// raised. + pub fn open(dst: &Path) -> Result { + unsafe { + let s = path_to_c_string(dst); + let ar = super::LLVMRustOpenArchive(s.as_ptr()).ok_or_else(|| { + super::last_error().unwrap_or_else(|| "failed to open archive".to_owned()) + })?; + Ok(ArchiveRO { raw: ar }) + } + } + + pub fn iter(&self) -> Iter<'_> { + unsafe { Iter { raw: super::LLVMRustArchiveIteratorNew(self.raw) } } + } +} + +impl Drop for ArchiveRO { + fn drop(&mut self) { + unsafe { + super::LLVMRustDestroyArchive(&mut *(self.raw as *mut _)); + } + } +} + +impl<'a> Iterator for Iter<'a> { + type Item = Result, String>; + + fn next(&mut self) -> Option, String>> { + unsafe { + match super::LLVMRustArchiveIteratorNext(self.raw) { + Some(raw) => Some(Ok(Child { raw })), + None => super::last_error().map(Err), + } + } + } +} + +impl<'a> Drop for Iter<'a> { + fn drop(&mut self) { + unsafe { + super::LLVMRustArchiveIteratorFree(&mut *(self.raw as *mut _)); + } + } +} + +impl<'a> Child<'a> { + pub fn name(&self) -> Option<&'a str> { + unsafe { + let mut name_len = 0; + let name_ptr = super::LLVMRustArchiveChildName(self.raw, &mut name_len); + if name_ptr.is_null() { + None + } else { + let name = slice::from_raw_parts(name_ptr as *const u8, name_len as usize); + str::from_utf8(name).ok().map(|s| s.trim()) + } + } + } + + pub fn data(&self) -> &'a [u8] { + unsafe { + let mut data_len = 0; + let data_ptr = super::LLVMRustArchiveChildData(self.raw, &mut data_len); + if data_ptr.is_null() { + panic!("failed to read data from archive child"); + } + slice::from_raw_parts(data_ptr as *const u8, data_len as usize) + } + } +} + +impl<'a> Drop for Child<'a> { + fn drop(&mut self) { + unsafe { + super::LLVMRustArchiveChildFree(&mut *(self.raw as *mut _)); + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs new file mode 100644 index 000000000..45de284d2 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/diagnostic.rs @@ -0,0 +1,213 @@ +//! LLVM diagnostic reports. + +pub use self::Diagnostic::*; +pub use self::OptimizationDiagnosticKind::*; + +use crate::value::Value; +use libc::c_uint; + +use super::{DiagnosticInfo, SMDiagnostic}; +use rustc_span::InnerSpan; + +#[derive(Copy, Clone)] +pub enum OptimizationDiagnosticKind { + OptimizationRemark, + OptimizationMissed, + OptimizationAnalysis, + OptimizationAnalysisFPCommute, + OptimizationAnalysisAliasing, + OptimizationFailure, + OptimizationRemarkOther, +} + +pub struct OptimizationDiagnostic<'ll> { + pub kind: OptimizationDiagnosticKind, + pub pass_name: String, + pub function: &'ll Value, + pub line: c_uint, + pub column: c_uint, + pub filename: String, + pub message: String, +} + +impl<'ll> OptimizationDiagnostic<'ll> { + unsafe fn unpack(kind: OptimizationDiagnosticKind, di: &'ll DiagnosticInfo) -> Self { + let mut function = None; + let mut line = 0; + let mut column = 0; + + let mut message = None; + let mut filename = None; + let pass_name = super::build_string(|pass_name| { + message = super::build_string(|message| { + filename = super::build_string(|filename| { + super::LLVMRustUnpackOptimizationDiagnostic( + di, + pass_name, + &mut function, + &mut line, + &mut column, + filename, + message, + ) + }) + .ok() + }) + .ok() + }) + .ok(); + + let mut filename = filename.unwrap_or_default(); + if filename.is_empty() { + filename.push_str(""); + } + + OptimizationDiagnostic { + kind, + pass_name: pass_name.expect("got a non-UTF8 pass name from LLVM"), + function: function.unwrap(), + line, + column, + filename, + message: message.expect("got a non-UTF8 OptimizationDiagnostic message from LLVM"), + } + } +} + +pub struct SrcMgrDiagnostic { + pub level: super::DiagnosticLevel, + pub message: String, + pub source: Option<(String, Vec)>, +} + +impl SrcMgrDiagnostic { + pub unsafe fn unpack(diag: &SMDiagnostic) -> SrcMgrDiagnostic { + // Recover the post-substitution assembly code from LLVM for better + // diagnostics. + let mut have_source = false; + let mut buffer = String::new(); + let mut level = super::DiagnosticLevel::Error; + let mut loc = 0; + let mut ranges = [0; 8]; + let mut num_ranges = ranges.len() / 2; + let message = super::build_string(|message| { + buffer = super::build_string(|buffer| { + have_source = super::LLVMRustUnpackSMDiagnostic( + diag, + message, + buffer, + &mut level, + &mut loc, + ranges.as_mut_ptr(), + &mut num_ranges, + ); + }) + .expect("non-UTF8 inline asm"); + }) + .expect("non-UTF8 SMDiagnostic"); + + SrcMgrDiagnostic { + message, + level, + source: have_source.then(|| { + let mut spans = vec![InnerSpan::new(loc as usize, loc as usize)]; + for i in 0..num_ranges { + spans.push(InnerSpan::new(ranges[i * 2] as usize, ranges[i * 2 + 1] as usize)); + } + (buffer, spans) + }), + } + } +} + +#[derive(Clone)] +pub struct InlineAsmDiagnostic { + pub level: super::DiagnosticLevel, + pub cookie: c_uint, + pub message: String, + pub source: Option<(String, Vec)>, +} + +impl InlineAsmDiagnostic { + unsafe fn unpackInlineAsm(di: &DiagnosticInfo) -> Self { + let mut cookie = 0; + let mut message = None; + let mut level = super::DiagnosticLevel::Error; + + super::LLVMRustUnpackInlineAsmDiagnostic(di, &mut level, &mut cookie, &mut message); + + InlineAsmDiagnostic { + level, + cookie, + message: super::twine_to_string(message.unwrap()), + source: None, + } + } + + unsafe fn unpackSrcMgr(di: &DiagnosticInfo) -> Self { + let mut cookie = 0; + let smdiag = SrcMgrDiagnostic::unpack(super::LLVMRustGetSMDiagnostic(di, &mut cookie)); + InlineAsmDiagnostic { + level: smdiag.level, + cookie, + message: smdiag.message, + source: smdiag.source, + } + } +} + +pub enum Diagnostic<'ll> { + Optimization(OptimizationDiagnostic<'ll>), + InlineAsm(InlineAsmDiagnostic), + PGO(&'ll DiagnosticInfo), + Linker(&'ll DiagnosticInfo), + Unsupported(&'ll DiagnosticInfo), + + /// LLVM has other types that we do not wrap here. + UnknownDiagnostic(&'ll DiagnosticInfo), +} + +impl<'ll> Diagnostic<'ll> { + pub unsafe fn unpack(di: &'ll DiagnosticInfo) -> Self { + use super::DiagnosticKind as Dk; + let kind = super::LLVMRustGetDiagInfoKind(di); + + match kind { + Dk::InlineAsm => InlineAsm(InlineAsmDiagnostic::unpackInlineAsm(di)), + + Dk::OptimizationRemark => { + Optimization(OptimizationDiagnostic::unpack(OptimizationRemark, di)) + } + Dk::OptimizationRemarkOther => { + Optimization(OptimizationDiagnostic::unpack(OptimizationRemarkOther, di)) + } + Dk::OptimizationRemarkMissed => { + Optimization(OptimizationDiagnostic::unpack(OptimizationMissed, di)) + } + + Dk::OptimizationRemarkAnalysis => { + Optimization(OptimizationDiagnostic::unpack(OptimizationAnalysis, di)) + } + + Dk::OptimizationRemarkAnalysisFPCommute => { + Optimization(OptimizationDiagnostic::unpack(OptimizationAnalysisFPCommute, di)) + } + + Dk::OptimizationRemarkAnalysisAliasing => { + Optimization(OptimizationDiagnostic::unpack(OptimizationAnalysisAliasing, di)) + } + + Dk::OptimizationFailure => { + Optimization(OptimizationDiagnostic::unpack(OptimizationFailure, di)) + } + + Dk::PGOProfile => PGO(di), + Dk::Linker => Linker(di), + Dk::Unsupported => Unsupported(di), + + Dk::SrcMgr => InlineAsm(InlineAsmDiagnostic::unpackSrcMgr(di)), + + _ => UnknownDiagnostic(di), + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/ffi.rs b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs new file mode 100644 index 000000000..3139f93bf --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/ffi.rs @@ -0,0 +1,2547 @@ +#![allow(non_camel_case_types)] +#![allow(non_upper_case_globals)] + +use rustc_codegen_ssa::coverageinfo::map as coverage_map; + +use super::debuginfo::{ + DIArray, DIBasicType, DIBuilder, DICompositeType, DIDerivedType, DIDescriptor, DIEnumerator, + DIFile, DIFlags, DIGlobalVariableExpression, DILexicalBlock, DILocation, DINameSpace, + DISPFlags, DIScope, DISubprogram, DISubrange, DITemplateTypeParameter, DIType, DIVariable, + DebugEmissionKind, +}; + +use libc::{c_char, c_int, c_uint, size_t}; +use libc::{c_ulonglong, c_void}; + +use std::marker::PhantomData; + +use super::RustString; + +pub type Bool = c_uint; + +pub const True: Bool = 1 as Bool; +pub const False: Bool = 0 as Bool; + +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub enum LLVMRustResult { + Success, + Failure, +} + +// Rust version of the C struct with the same name in rustc_llvm/llvm-wrapper/RustWrapper.cpp. +#[repr(C)] +pub struct LLVMRustCOFFShortExport { + pub name: *const c_char, + pub ordinal_present: bool, + // value of `ordinal` only important when `ordinal_present` is true + pub ordinal: u16, +} + +impl LLVMRustCOFFShortExport { + pub fn new(name: *const c_char, ordinal: Option) -> LLVMRustCOFFShortExport { + LLVMRustCOFFShortExport { + name, + ordinal_present: ordinal.is_some(), + ordinal: ordinal.unwrap_or(0), + } + } +} + +/// Translation of LLVM's MachineTypes enum, defined in llvm\include\llvm\BinaryFormat\COFF.h. +/// +/// We include only architectures supported on Windows. +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum LLVMMachineType { + AMD64 = 0x8664, + I386 = 0x14c, + ARM64 = 0xaa64, + ARM = 0x01c0, +} + +/// LLVM's Module::ModFlagBehavior, defined in llvm/include/llvm/IR/Module.h. +/// +/// When merging modules (e.g. during LTO), their metadata flags are combined. Conflicts are +/// resolved according to the merge behaviors specified here. Flags differing only in merge +/// behavior are still considered to be in conflict. +/// +/// In order for Rust-C LTO to work, we must specify behaviors compatible with Clang. Notably, +/// 'Error' and 'Warning' cannot be mixed for a given flag. +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum LLVMModFlagBehavior { + Error = 1, + Warning = 2, + Require = 3, + Override = 4, + Append = 5, + AppendUnique = 6, + Max = 7, +} + +// Consts for the LLVM CallConv type, pre-cast to usize. + +/// LLVM CallingConv::ID. Should we wrap this? +#[derive(Copy, Clone, PartialEq, Debug)] +#[repr(C)] +pub enum CallConv { + CCallConv = 0, + FastCallConv = 8, + ColdCallConv = 9, + X86StdcallCallConv = 64, + X86FastcallCallConv = 65, + ArmAapcsCallConv = 67, + Msp430Intr = 69, + X86_ThisCall = 70, + PtxKernel = 71, + X86_64_SysV = 78, + X86_64_Win64 = 79, + X86_VectorCall = 80, + X86_Intr = 83, + AvrNonBlockingInterrupt = 84, + AvrInterrupt = 85, + AmdGpuKernel = 91, +} + +/// LLVMRustLinkage +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum Linkage { + ExternalLinkage = 0, + AvailableExternallyLinkage = 1, + LinkOnceAnyLinkage = 2, + LinkOnceODRLinkage = 3, + WeakAnyLinkage = 4, + WeakODRLinkage = 5, + AppendingLinkage = 6, + InternalLinkage = 7, + PrivateLinkage = 8, + ExternalWeakLinkage = 9, + CommonLinkage = 10, +} + +// LLVMRustVisibility +#[repr(C)] +#[derive(Copy, Clone, PartialEq)] +pub enum Visibility { + Default = 0, + Hidden = 1, + Protected = 2, +} + +/// LLVMUnnamedAddr +#[repr(C)] +pub enum UnnamedAddr { + No, + Local, + Global, +} + +/// LLVMDLLStorageClass +#[derive(Copy, Clone)] +#[repr(C)] +pub enum DLLStorageClass { + #[allow(dead_code)] + Default = 0, + DllImport = 1, // Function to be imported from DLL. + #[allow(dead_code)] + DllExport = 2, // Function to be accessible from DLL. +} + +/// Matches LLVMRustAttribute in LLVMWrapper.h +/// Semantically a subset of the C++ enum llvm::Attribute::AttrKind, +/// though it is not ABI compatible (since it's a C++ enum) +#[repr(C)] +#[derive(Copy, Clone, Debug)] +pub enum AttributeKind { + AlwaysInline = 0, + ByVal = 1, + Cold = 2, + InlineHint = 3, + MinSize = 4, + Naked = 5, + NoAlias = 6, + NoCapture = 7, + NoInline = 8, + NonNull = 9, + NoRedZone = 10, + NoReturn = 11, + NoUnwind = 12, + OptimizeForSize = 13, + ReadOnly = 14, + SExt = 15, + StructRet = 16, + UWTable = 17, + ZExt = 18, + InReg = 19, + SanitizeThread = 20, + SanitizeAddress = 21, + SanitizeMemory = 22, + NonLazyBind = 23, + OptimizeNone = 24, + ReturnsTwice = 25, + ReadNone = 26, + InaccessibleMemOnly = 27, + SanitizeHWAddress = 28, + WillReturn = 29, + StackProtectReq = 30, + StackProtectStrong = 31, + StackProtect = 32, + NoUndef = 33, + SanitizeMemTag = 34, + NoCfCheck = 35, + ShadowCallStack = 36, + AllocSize = 37, + AllocatedPointer = 38, + AllocAlign = 39, +} + +/// LLVMIntPredicate +#[derive(Copy, Clone)] +#[repr(C)] +pub enum IntPredicate { + IntEQ = 32, + IntNE = 33, + IntUGT = 34, + IntUGE = 35, + IntULT = 36, + IntULE = 37, + IntSGT = 38, + IntSGE = 39, + IntSLT = 40, + IntSLE = 41, +} + +impl IntPredicate { + pub fn from_generic(intpre: rustc_codegen_ssa::common::IntPredicate) -> Self { + match intpre { + rustc_codegen_ssa::common::IntPredicate::IntEQ => IntPredicate::IntEQ, + rustc_codegen_ssa::common::IntPredicate::IntNE => IntPredicate::IntNE, + rustc_codegen_ssa::common::IntPredicate::IntUGT => IntPredicate::IntUGT, + rustc_codegen_ssa::common::IntPredicate::IntUGE => IntPredicate::IntUGE, + rustc_codegen_ssa::common::IntPredicate::IntULT => IntPredicate::IntULT, + rustc_codegen_ssa::common::IntPredicate::IntULE => IntPredicate::IntULE, + rustc_codegen_ssa::common::IntPredicate::IntSGT => IntPredicate::IntSGT, + rustc_codegen_ssa::common::IntPredicate::IntSGE => IntPredicate::IntSGE, + rustc_codegen_ssa::common::IntPredicate::IntSLT => IntPredicate::IntSLT, + rustc_codegen_ssa::common::IntPredicate::IntSLE => IntPredicate::IntSLE, + } + } +} + +/// LLVMRealPredicate +#[derive(Copy, Clone)] +#[repr(C)] +pub enum RealPredicate { + RealPredicateFalse = 0, + RealOEQ = 1, + RealOGT = 2, + RealOGE = 3, + RealOLT = 4, + RealOLE = 5, + RealONE = 6, + RealORD = 7, + RealUNO = 8, + RealUEQ = 9, + RealUGT = 10, + RealUGE = 11, + RealULT = 12, + RealULE = 13, + RealUNE = 14, + RealPredicateTrue = 15, +} + +impl RealPredicate { + pub fn from_generic(realp: rustc_codegen_ssa::common::RealPredicate) -> Self { + match realp { + rustc_codegen_ssa::common::RealPredicate::RealPredicateFalse => { + RealPredicate::RealPredicateFalse + } + rustc_codegen_ssa::common::RealPredicate::RealOEQ => RealPredicate::RealOEQ, + rustc_codegen_ssa::common::RealPredicate::RealOGT => RealPredicate::RealOGT, + rustc_codegen_ssa::common::RealPredicate::RealOGE => RealPredicate::RealOGE, + rustc_codegen_ssa::common::RealPredicate::RealOLT => RealPredicate::RealOLT, + rustc_codegen_ssa::common::RealPredicate::RealOLE => RealPredicate::RealOLE, + rustc_codegen_ssa::common::RealPredicate::RealONE => RealPredicate::RealONE, + rustc_codegen_ssa::common::RealPredicate::RealORD => RealPredicate::RealORD, + rustc_codegen_ssa::common::RealPredicate::RealUNO => RealPredicate::RealUNO, + rustc_codegen_ssa::common::RealPredicate::RealUEQ => RealPredicate::RealUEQ, + rustc_codegen_ssa::common::RealPredicate::RealUGT => RealPredicate::RealUGT, + rustc_codegen_ssa::common::RealPredicate::RealUGE => RealPredicate::RealUGE, + rustc_codegen_ssa::common::RealPredicate::RealULT => RealPredicate::RealULT, + rustc_codegen_ssa::common::RealPredicate::RealULE => RealPredicate::RealULE, + rustc_codegen_ssa::common::RealPredicate::RealUNE => RealPredicate::RealUNE, + rustc_codegen_ssa::common::RealPredicate::RealPredicateTrue => { + RealPredicate::RealPredicateTrue + } + } + } +} + +/// LLVMTypeKind +#[derive(Copy, Clone, PartialEq, Debug)] +#[repr(C)] +pub enum TypeKind { + Void = 0, + Half = 1, + Float = 2, + Double = 3, + X86_FP80 = 4, + FP128 = 5, + PPC_FP128 = 6, + Label = 7, + Integer = 8, + Function = 9, + Struct = 10, + Array = 11, + Pointer = 12, + Vector = 13, + Metadata = 14, + X86_MMX = 15, + Token = 16, + ScalableVector = 17, + BFloat = 18, + X86_AMX = 19, +} + +impl TypeKind { + pub fn to_generic(self) -> rustc_codegen_ssa::common::TypeKind { + match self { + TypeKind::Void => rustc_codegen_ssa::common::TypeKind::Void, + TypeKind::Half => rustc_codegen_ssa::common::TypeKind::Half, + TypeKind::Float => rustc_codegen_ssa::common::TypeKind::Float, + TypeKind::Double => rustc_codegen_ssa::common::TypeKind::Double, + TypeKind::X86_FP80 => rustc_codegen_ssa::common::TypeKind::X86_FP80, + TypeKind::FP128 => rustc_codegen_ssa::common::TypeKind::FP128, + TypeKind::PPC_FP128 => rustc_codegen_ssa::common::TypeKind::PPC_FP128, + TypeKind::Label => rustc_codegen_ssa::common::TypeKind::Label, + TypeKind::Integer => rustc_codegen_ssa::common::TypeKind::Integer, + TypeKind::Function => rustc_codegen_ssa::common::TypeKind::Function, + TypeKind::Struct => rustc_codegen_ssa::common::TypeKind::Struct, + TypeKind::Array => rustc_codegen_ssa::common::TypeKind::Array, + TypeKind::Pointer => rustc_codegen_ssa::common::TypeKind::Pointer, + TypeKind::Vector => rustc_codegen_ssa::common::TypeKind::Vector, + TypeKind::Metadata => rustc_codegen_ssa::common::TypeKind::Metadata, + TypeKind::X86_MMX => rustc_codegen_ssa::common::TypeKind::X86_MMX, + TypeKind::Token => rustc_codegen_ssa::common::TypeKind::Token, + TypeKind::ScalableVector => rustc_codegen_ssa::common::TypeKind::ScalableVector, + TypeKind::BFloat => rustc_codegen_ssa::common::TypeKind::BFloat, + TypeKind::X86_AMX => rustc_codegen_ssa::common::TypeKind::X86_AMX, + } + } +} + +/// LLVMAtomicRmwBinOp +#[derive(Copy, Clone)] +#[repr(C)] +pub enum AtomicRmwBinOp { + AtomicXchg = 0, + AtomicAdd = 1, + AtomicSub = 2, + AtomicAnd = 3, + AtomicNand = 4, + AtomicOr = 5, + AtomicXor = 6, + AtomicMax = 7, + AtomicMin = 8, + AtomicUMax = 9, + AtomicUMin = 10, +} + +impl AtomicRmwBinOp { + pub fn from_generic(op: rustc_codegen_ssa::common::AtomicRmwBinOp) -> Self { + match op { + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXchg => AtomicRmwBinOp::AtomicXchg, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicAdd => AtomicRmwBinOp::AtomicAdd, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicSub => AtomicRmwBinOp::AtomicSub, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicAnd => AtomicRmwBinOp::AtomicAnd, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicNand => AtomicRmwBinOp::AtomicNand, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicOr => AtomicRmwBinOp::AtomicOr, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicXor => AtomicRmwBinOp::AtomicXor, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicMax => AtomicRmwBinOp::AtomicMax, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicMin => AtomicRmwBinOp::AtomicMin, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicUMax => AtomicRmwBinOp::AtomicUMax, + rustc_codegen_ssa::common::AtomicRmwBinOp::AtomicUMin => AtomicRmwBinOp::AtomicUMin, + } + } +} + +/// LLVMAtomicOrdering +#[derive(Copy, Clone)] +#[repr(C)] +pub enum AtomicOrdering { + #[allow(dead_code)] + NotAtomic = 0, + Unordered = 1, + Monotonic = 2, + // Consume = 3, // Not specified yet. + Acquire = 4, + Release = 5, + AcquireRelease = 6, + SequentiallyConsistent = 7, +} + +impl AtomicOrdering { + pub fn from_generic(ao: rustc_codegen_ssa::common::AtomicOrdering) -> Self { + match ao { + rustc_codegen_ssa::common::AtomicOrdering::Unordered => AtomicOrdering::Unordered, + rustc_codegen_ssa::common::AtomicOrdering::Relaxed => AtomicOrdering::Monotonic, + rustc_codegen_ssa::common::AtomicOrdering::Acquire => AtomicOrdering::Acquire, + rustc_codegen_ssa::common::AtomicOrdering::Release => AtomicOrdering::Release, + rustc_codegen_ssa::common::AtomicOrdering::AcquireRelease => { + AtomicOrdering::AcquireRelease + } + rustc_codegen_ssa::common::AtomicOrdering::SequentiallyConsistent => { + AtomicOrdering::SequentiallyConsistent + } + } + } +} + +/// LLVMRustSynchronizationScope +#[derive(Copy, Clone)] +#[repr(C)] +pub enum SynchronizationScope { + SingleThread, + CrossThread, +} + +impl SynchronizationScope { + pub fn from_generic(sc: rustc_codegen_ssa::common::SynchronizationScope) -> Self { + match sc { + rustc_codegen_ssa::common::SynchronizationScope::SingleThread => { + SynchronizationScope::SingleThread + } + rustc_codegen_ssa::common::SynchronizationScope::CrossThread => { + SynchronizationScope::CrossThread + } + } + } +} + +/// LLVMRustFileType +#[derive(Copy, Clone)] +#[repr(C)] +pub enum FileType { + AssemblyFile, + ObjectFile, +} + +/// LLVMMetadataType +#[derive(Copy, Clone)] +#[repr(C)] +pub enum MetadataType { + MD_dbg = 0, + MD_tbaa = 1, + MD_prof = 2, + MD_fpmath = 3, + MD_range = 4, + MD_tbaa_struct = 5, + MD_invariant_load = 6, + MD_alias_scope = 7, + MD_noalias = 8, + MD_nontemporal = 9, + MD_mem_parallel_loop_access = 10, + MD_nonnull = 11, + MD_align = 17, + MD_type = 19, + MD_vcall_visibility = 28, + MD_noundef = 29, +} + +/// LLVMRustAsmDialect +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum AsmDialect { + Att, + Intel, +} + +/// LLVMRustCodeGenOptLevel +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum CodeGenOptLevel { + None, + Less, + Default, + Aggressive, +} + +/// LLVMRustPassBuilderOptLevel +#[repr(C)] +pub enum PassBuilderOptLevel { + O0, + O1, + O2, + O3, + Os, + Oz, +} + +/// LLVMRustOptStage +#[derive(PartialEq)] +#[repr(C)] +pub enum OptStage { + PreLinkNoLTO, + PreLinkThinLTO, + PreLinkFatLTO, + ThinLTO, + FatLTO, +} + +/// LLVMRustSanitizerOptions +#[repr(C)] +pub struct SanitizerOptions { + pub sanitize_address: bool, + pub sanitize_address_recover: bool, + pub sanitize_memory: bool, + pub sanitize_memory_recover: bool, + pub sanitize_memory_track_origins: c_int, + pub sanitize_thread: bool, + pub sanitize_hwaddress: bool, + pub sanitize_hwaddress_recover: bool, +} + +/// LLVMRelocMode +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum RelocModel { + Static, + PIC, + DynamicNoPic, + ROPI, + RWPI, + ROPI_RWPI, +} + +/// LLVMRustCodeModel +#[derive(Copy, Clone)] +#[repr(C)] +pub enum CodeModel { + Tiny, + Small, + Kernel, + Medium, + Large, + None, +} + +/// LLVMRustDiagnosticKind +#[derive(Copy, Clone)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub enum DiagnosticKind { + Other, + InlineAsm, + StackSize, + DebugMetadataVersion, + SampleProfile, + OptimizationRemark, + OptimizationRemarkMissed, + OptimizationRemarkAnalysis, + OptimizationRemarkAnalysisFPCommute, + OptimizationRemarkAnalysisAliasing, + OptimizationRemarkOther, + OptimizationFailure, + PGOProfile, + Linker, + Unsupported, + SrcMgr, +} + +/// LLVMRustDiagnosticLevel +#[derive(Copy, Clone)] +#[repr(C)] +#[allow(dead_code)] // Variants constructed by C++. +pub enum DiagnosticLevel { + Error, + Warning, + Note, + Remark, +} + +/// LLVMRustArchiveKind +#[derive(Copy, Clone)] +#[repr(C)] +pub enum ArchiveKind { + K_GNU, + K_BSD, + K_DARWIN, + K_COFF, +} + +// LLVMRustThinLTOData +extern "C" { + pub type ThinLTOData; +} + +// LLVMRustThinLTOBuffer +extern "C" { + pub type ThinLTOBuffer; +} + +/// LLVMRustThinLTOModule +#[repr(C)] +pub struct ThinLTOModule { + pub identifier: *const c_char, + pub data: *const u8, + pub len: usize, +} + +/// LLVMThreadLocalMode +#[derive(Copy, Clone)] +#[repr(C)] +pub enum ThreadLocalMode { + NotThreadLocal, + GeneralDynamic, + LocalDynamic, + InitialExec, + LocalExec, +} + +/// LLVMRustChecksumKind +#[derive(Copy, Clone)] +#[repr(C)] +pub enum ChecksumKind { + None, + MD5, + SHA1, + SHA256, +} + +extern "C" { + type Opaque; +} +#[repr(C)] +struct InvariantOpaque<'a> { + _marker: PhantomData<&'a mut &'a ()>, + _opaque: Opaque, +} + +// Opaque pointer types +extern "C" { + pub type Module; +} +extern "C" { + pub type Context; +} +extern "C" { + pub type Type; +} +extern "C" { + pub type Value; +} +extern "C" { + pub type ConstantInt; +} +extern "C" { + pub type Attribute; +} +extern "C" { + pub type Metadata; +} +extern "C" { + pub type BasicBlock; +} +#[repr(C)] +pub struct Builder<'a>(InvariantOpaque<'a>); +#[repr(C)] +pub struct PassManager<'a>(InvariantOpaque<'a>); +extern "C" { + pub type PassManagerBuilder; +} +extern "C" { + pub type Pass; +} +extern "C" { + pub type TargetMachine; +} +extern "C" { + pub type Archive; +} +#[repr(C)] +pub struct ArchiveIterator<'a>(InvariantOpaque<'a>); +#[repr(C)] +pub struct ArchiveChild<'a>(InvariantOpaque<'a>); +extern "C" { + pub type Twine; +} +extern "C" { + pub type DiagnosticInfo; +} +extern "C" { + pub type SMDiagnostic; +} +#[repr(C)] +pub struct RustArchiveMember<'a>(InvariantOpaque<'a>); +#[repr(C)] +pub struct OperandBundleDef<'a>(InvariantOpaque<'a>); +#[repr(C)] +pub struct Linker<'a>(InvariantOpaque<'a>); + +extern "C" { + pub type DiagnosticHandler; +} + +pub type DiagnosticHandlerTy = unsafe extern "C" fn(&DiagnosticInfo, *mut c_void); +pub type InlineAsmDiagHandlerTy = unsafe extern "C" fn(&SMDiagnostic, *const c_void, c_uint); + +pub mod coverageinfo { + use super::coverage_map; + + /// Aligns with [llvm::coverage::CounterMappingRegion::RegionKind](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/include/llvm/ProfileData/Coverage/CoverageMapping.h#L209-L230) + #[derive(Copy, Clone, Debug)] + #[repr(C)] + pub enum RegionKind { + /// A CodeRegion associates some code with a counter + CodeRegion = 0, + + /// An ExpansionRegion represents a file expansion region that associates + /// a source range with the expansion of a virtual source file, such as + /// for a macro instantiation or #include file. + ExpansionRegion = 1, + + /// A SkippedRegion represents a source range with code that was skipped + /// by a preprocessor or similar means. + SkippedRegion = 2, + + /// A GapRegion is like a CodeRegion, but its count is only set as the + /// line execution count when its the only region in the line. + GapRegion = 3, + + /// A BranchRegion represents leaf-level boolean expressions and is + /// associated with two counters, each representing the number of times the + /// expression evaluates to true or false. + BranchRegion = 4, + } + + /// This struct provides LLVM's representation of a "CoverageMappingRegion", encoded into the + /// coverage map, in accordance with the + /// [LLVM Code Coverage Mapping Format](https://github.com/rust-lang/llvm-project/blob/rustc/13.0-2021-09-30/llvm/docs/CoverageMappingFormat.rst#llvm-code-coverage-mapping-format). + /// The struct composes fields representing the `Counter` type and value(s) (injected counter + /// ID, or expression type and operands), the source file (an indirect index into a "filenames + /// array", encoded separately), and source location (start and end positions of the represented + /// code region). + /// + /// Matches LLVMRustCounterMappingRegion. + #[derive(Copy, Clone, Debug)] + #[repr(C)] + pub struct CounterMappingRegion { + /// The counter type and type-dependent counter data, if any. + counter: coverage_map::Counter, + + /// If the `RegionKind` is a `BranchRegion`, this represents the counter + /// for the false branch of the region. + false_counter: coverage_map::Counter, + + /// An indirect reference to the source filename. In the LLVM Coverage Mapping Format, the + /// file_id is an index into a function-specific `virtual_file_mapping` array of indexes + /// that, in turn, are used to look up the filename for this region. + file_id: u32, + + /// If the `RegionKind` is an `ExpansionRegion`, the `expanded_file_id` can be used to find + /// the mapping regions created as a result of macro expansion, by checking if their file id + /// matches the expanded file id. + expanded_file_id: u32, + + /// 1-based starting line of the mapping region. + start_line: u32, + + /// 1-based starting column of the mapping region. + start_col: u32, + + /// 1-based ending line of the mapping region. + end_line: u32, + + /// 1-based ending column of the mapping region. If the high bit is set, the current + /// mapping region is a gap area. + end_col: u32, + + kind: RegionKind, + } + + impl CounterMappingRegion { + pub(crate) fn code_region( + counter: coverage_map::Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter: coverage_map::Counter::zero(), + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::CodeRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn branch_region( + counter: coverage_map::Counter, + false_counter: coverage_map::Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter, + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::BranchRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn expansion_region( + file_id: u32, + expanded_file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter: coverage_map::Counter::zero(), + false_counter: coverage_map::Counter::zero(), + file_id, + expanded_file_id, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::ExpansionRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn skipped_region( + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter: coverage_map::Counter::zero(), + false_counter: coverage_map::Counter::zero(), + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col, + kind: RegionKind::SkippedRegion, + } + } + + // This function might be used in the future; the LLVM API is still evolving, as is coverage + // support. + #[allow(dead_code)] + pub(crate) fn gap_region( + counter: coverage_map::Counter, + file_id: u32, + start_line: u32, + start_col: u32, + end_line: u32, + end_col: u32, + ) -> Self { + Self { + counter, + false_counter: coverage_map::Counter::zero(), + file_id, + expanded_file_id: 0, + start_line, + start_col, + end_line, + end_col: (1_u32 << 31) | end_col, + kind: RegionKind::GapRegion, + } + } + } +} + +pub mod debuginfo { + use super::{InvariantOpaque, Metadata}; + use bitflags::bitflags; + + #[repr(C)] + pub struct DIBuilder<'a>(InvariantOpaque<'a>); + + pub type DIDescriptor = Metadata; + pub type DILocation = Metadata; + pub type DIScope = DIDescriptor; + pub type DIFile = DIScope; + pub type DILexicalBlock = DIScope; + pub type DISubprogram = DIScope; + pub type DINameSpace = DIScope; + pub type DIType = DIDescriptor; + pub type DIBasicType = DIType; + pub type DIDerivedType = DIType; + pub type DICompositeType = DIDerivedType; + pub type DIVariable = DIDescriptor; + pub type DIGlobalVariableExpression = DIDescriptor; + pub type DIArray = DIDescriptor; + pub type DISubrange = DIDescriptor; + pub type DIEnumerator = DIDescriptor; + pub type DITemplateTypeParameter = DIDescriptor; + + // These values **must** match with LLVMRustDIFlags!! + bitflags! { + #[repr(transparent)] + #[derive(Default)] + pub struct DIFlags: u32 { + const FlagZero = 0; + const FlagPrivate = 1; + const FlagProtected = 2; + const FlagPublic = 3; + const FlagFwdDecl = (1 << 2); + const FlagAppleBlock = (1 << 3); + const FlagBlockByrefStruct = (1 << 4); + const FlagVirtual = (1 << 5); + const FlagArtificial = (1 << 6); + const FlagExplicit = (1 << 7); + const FlagPrototyped = (1 << 8); + const FlagObjcClassComplete = (1 << 9); + const FlagObjectPointer = (1 << 10); + const FlagVector = (1 << 11); + const FlagStaticMember = (1 << 12); + const FlagLValueReference = (1 << 13); + const FlagRValueReference = (1 << 14); + const FlagExternalTypeRef = (1 << 15); + const FlagIntroducedVirtual = (1 << 18); + const FlagBitField = (1 << 19); + const FlagNoReturn = (1 << 20); + } + } + + // These values **must** match with LLVMRustDISPFlags!! + bitflags! { + #[repr(transparent)] + #[derive(Default)] + pub struct DISPFlags: u32 { + const SPFlagZero = 0; + const SPFlagVirtual = 1; + const SPFlagPureVirtual = 2; + const SPFlagLocalToUnit = (1 << 2); + const SPFlagDefinition = (1 << 3); + const SPFlagOptimized = (1 << 4); + const SPFlagMainSubprogram = (1 << 5); + } + } + + /// LLVMRustDebugEmissionKind + #[derive(Copy, Clone)] + #[repr(C)] + pub enum DebugEmissionKind { + NoDebug, + FullDebug, + LineTablesOnly, + } + + impl DebugEmissionKind { + pub fn from_generic(kind: rustc_session::config::DebugInfo) -> Self { + use rustc_session::config::DebugInfo; + match kind { + DebugInfo::None => DebugEmissionKind::NoDebug, + DebugInfo::Limited => DebugEmissionKind::LineTablesOnly, + DebugInfo::Full => DebugEmissionKind::FullDebug, + } + } + } +} + +use bitflags::bitflags; +// These values **must** match with LLVMRustAllocKindFlags +bitflags! { + #[repr(transparent)] + #[derive(Default)] + pub struct AllocKindFlags : u64 { + const Unknown = 0; + const Alloc = 1; + const Realloc = 1 << 1; + const Free = 1 << 2; + const Uninitialized = 1 << 3; + const Zeroed = 1 << 4; + const Aligned = 1 << 5; + } +} + +extern "C" { + pub type ModuleBuffer; +} + +pub type SelfProfileBeforePassCallback = + unsafe extern "C" fn(*mut c_void, *const c_char, *const c_char); +pub type SelfProfileAfterPassCallback = unsafe extern "C" fn(*mut c_void); + +extern "C" { + pub fn LLVMRustInstallFatalErrorHandler(); + pub fn LLVMRustDisableSystemDialogsOnCrash(); + + // Create and destroy contexts. + pub fn LLVMRustContextCreate(shouldDiscardNames: bool) -> &'static mut Context; + pub fn LLVMContextDispose(C: &'static mut Context); + pub fn LLVMGetMDKindIDInContext(C: &Context, Name: *const c_char, SLen: c_uint) -> c_uint; + + // Create modules. + pub fn LLVMModuleCreateWithNameInContext(ModuleID: *const c_char, C: &Context) -> &Module; + pub fn LLVMGetModuleContext(M: &Module) -> &Context; + pub fn LLVMCloneModule(M: &Module) -> &Module; + + /// Data layout. See Module::getDataLayout. + pub fn LLVMGetDataLayoutStr(M: &Module) -> *const c_char; + pub fn LLVMSetDataLayout(M: &Module, Triple: *const c_char); + + /// See Module::setModuleInlineAsm. + pub fn LLVMRustAppendModuleInlineAsm(M: &Module, Asm: *const c_char, AsmLen: size_t); + + /// See llvm::LLVMTypeKind::getTypeID. + pub fn LLVMRustGetTypeKind(Ty: &Type) -> TypeKind; + + // Operations on integer types + pub fn LLVMInt1TypeInContext(C: &Context) -> &Type; + pub fn LLVMInt8TypeInContext(C: &Context) -> &Type; + pub fn LLVMInt16TypeInContext(C: &Context) -> &Type; + pub fn LLVMInt32TypeInContext(C: &Context) -> &Type; + pub fn LLVMInt64TypeInContext(C: &Context) -> &Type; + pub fn LLVMIntTypeInContext(C: &Context, NumBits: c_uint) -> &Type; + + pub fn LLVMGetIntTypeWidth(IntegerTy: &Type) -> c_uint; + + // Operations on real types + pub fn LLVMFloatTypeInContext(C: &Context) -> &Type; + pub fn LLVMDoubleTypeInContext(C: &Context) -> &Type; + + // Operations on function types + pub fn LLVMFunctionType<'a>( + ReturnType: &'a Type, + ParamTypes: *const &'a Type, + ParamCount: c_uint, + IsVarArg: Bool, + ) -> &'a Type; + pub fn LLVMCountParamTypes(FunctionTy: &Type) -> c_uint; + pub fn LLVMGetParamTypes<'a>(FunctionTy: &'a Type, Dest: *mut &'a Type); + + // Operations on struct types + pub fn LLVMStructTypeInContext<'a>( + C: &'a Context, + ElementTypes: *const &'a Type, + ElementCount: c_uint, + Packed: Bool, + ) -> &'a Type; + + // Operations on array, pointer, and vector types (sequence types) + pub fn LLVMRustArrayType(ElementType: &Type, ElementCount: u64) -> &Type; + pub fn LLVMPointerType(ElementType: &Type, AddressSpace: c_uint) -> &Type; + pub fn LLVMVectorType(ElementType: &Type, ElementCount: c_uint) -> &Type; + + pub fn LLVMGetElementType(Ty: &Type) -> &Type; + pub fn LLVMGetVectorSize(VectorTy: &Type) -> c_uint; + + // Operations on other types + pub fn LLVMVoidTypeInContext(C: &Context) -> &Type; + pub fn LLVMRustMetadataTypeInContext(C: &Context) -> &Type; + + // Operations on all values + pub fn LLVMTypeOf(Val: &Value) -> &Type; + pub fn LLVMGetValueName2(Val: &Value, Length: *mut size_t) -> *const c_char; + pub fn LLVMSetValueName2(Val: &Value, Name: *const c_char, NameLen: size_t); + pub fn LLVMReplaceAllUsesWith<'a>(OldVal: &'a Value, NewVal: &'a Value); + pub fn LLVMSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Node: &'a Value); + pub fn LLVMGlobalSetMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMRustGlobalAddMetadata<'a>(Val: &'a Value, KindID: c_uint, Metadata: &'a Metadata); + pub fn LLVMValueAsMetadata(Node: &Value) -> &Metadata; + + // Operations on constants of any type + pub fn LLVMConstNull(Ty: &Type) -> &Value; + pub fn LLVMGetUndef(Ty: &Type) -> &Value; + + // Operations on metadata + pub fn LLVMMDStringInContext(C: &Context, Str: *const c_char, SLen: c_uint) -> &Value; + pub fn LLVMMDNodeInContext<'a>( + C: &'a Context, + Vals: *const &'a Value, + Count: c_uint, + ) -> &'a Value; + pub fn LLVMMDNodeInContext2<'a>( + C: &'a Context, + Vals: *const &'a Metadata, + Count: size_t, + ) -> &'a Metadata; + pub fn LLVMAddNamedMetadataOperand<'a>(M: &'a Module, Name: *const c_char, Val: &'a Value); + + // Operations on scalar constants + pub fn LLVMConstInt(IntTy: &Type, N: c_ulonglong, SignExtend: Bool) -> &Value; + pub fn LLVMConstIntOfArbitraryPrecision(IntTy: &Type, Wn: c_uint, Ws: *const u64) -> &Value; + pub fn LLVMConstReal(RealTy: &Type, N: f64) -> &Value; + pub fn LLVMConstIntGetZExtValue(ConstantVal: &ConstantInt) -> c_ulonglong; + pub fn LLVMRustConstInt128Get( + ConstantVal: &ConstantInt, + SExt: bool, + high: &mut u64, + low: &mut u64, + ) -> bool; + + // Operations on composite constants + pub fn LLVMConstStringInContext( + C: &Context, + Str: *const c_char, + Length: c_uint, + DontNullTerminate: Bool, + ) -> &Value; + pub fn LLVMConstStructInContext<'a>( + C: &'a Context, + ConstantVals: *const &'a Value, + Count: c_uint, + Packed: Bool, + ) -> &'a Value; + + pub fn LLVMConstArray<'a>( + ElementTy: &'a Type, + ConstantVals: *const &'a Value, + Length: c_uint, + ) -> &'a Value; + pub fn LLVMConstVector(ScalarConstantVals: *const &Value, Size: c_uint) -> &Value; + + // Constant expressions + pub fn LLVMRustConstInBoundsGEP2<'a>( + ty: &'a Type, + ConstantVal: &'a Value, + ConstantIndices: *const &'a Value, + NumIndices: c_uint, + ) -> &'a Value; + pub fn LLVMConstZExt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMConstPtrToInt<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMConstIntToPtr<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMConstBitCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMConstPointerCast<'a>(ConstantVal: &'a Value, ToType: &'a Type) -> &'a Value; + pub fn LLVMGetAggregateElement(ConstantVal: &Value, Idx: c_uint) -> Option<&Value>; + + // Operations on global variables, functions, and aliases (globals) + pub fn LLVMIsDeclaration(Global: &Value) -> Bool; + pub fn LLVMRustGetLinkage(Global: &Value) -> Linkage; + pub fn LLVMRustSetLinkage(Global: &Value, RustLinkage: Linkage); + pub fn LLVMSetSection(Global: &Value, Section: *const c_char); + pub fn LLVMRustGetVisibility(Global: &Value) -> Visibility; + pub fn LLVMRustSetVisibility(Global: &Value, Viz: Visibility); + pub fn LLVMRustSetDSOLocal(Global: &Value, is_dso_local: bool); + pub fn LLVMGetAlignment(Global: &Value) -> c_uint; + pub fn LLVMSetAlignment(Global: &Value, Bytes: c_uint); + pub fn LLVMSetDLLStorageClass(V: &Value, C: DLLStorageClass); + + // Operations on global variables + pub fn LLVMIsAGlobalVariable(GlobalVar: &Value) -> Option<&Value>; + pub fn LLVMAddGlobal<'a>(M: &'a Module, Ty: &'a Type, Name: *const c_char) -> &'a Value; + pub fn LLVMGetNamedGlobal(M: &Module, Name: *const c_char) -> Option<&Value>; + pub fn LLVMRustGetOrInsertGlobal<'a>( + M: &'a Module, + Name: *const c_char, + NameLen: size_t, + T: &'a Type, + ) -> &'a Value; + pub fn LLVMRustInsertPrivateGlobal<'a>(M: &'a Module, T: &'a Type) -> &'a Value; + pub fn LLVMGetFirstGlobal(M: &Module) -> Option<&Value>; + pub fn LLVMGetNextGlobal(GlobalVar: &Value) -> Option<&Value>; + pub fn LLVMDeleteGlobal(GlobalVar: &Value); + pub fn LLVMGetInitializer(GlobalVar: &Value) -> Option<&Value>; + pub fn LLVMSetInitializer<'a>(GlobalVar: &'a Value, ConstantVal: &'a Value); + pub fn LLVMIsThreadLocal(GlobalVar: &Value) -> Bool; + pub fn LLVMSetThreadLocalMode(GlobalVar: &Value, Mode: ThreadLocalMode); + pub fn LLVMIsGlobalConstant(GlobalVar: &Value) -> Bool; + pub fn LLVMSetGlobalConstant(GlobalVar: &Value, IsConstant: Bool); + pub fn LLVMRustGetNamedValue( + M: &Module, + Name: *const c_char, + NameLen: size_t, + ) -> Option<&Value>; + pub fn LLVMSetTailCall(CallInst: &Value, IsTailCall: Bool); + + // Operations on attributes + pub fn LLVMRustCreateAttrNoValue(C: &Context, attr: AttributeKind) -> &Attribute; + pub fn LLVMCreateStringAttribute( + C: &Context, + Name: *const c_char, + NameLen: c_uint, + Value: *const c_char, + ValueLen: c_uint, + ) -> &Attribute; + pub fn LLVMRustCreateAlignmentAttr(C: &Context, bytes: u64) -> &Attribute; + pub fn LLVMRustCreateDereferenceableAttr(C: &Context, bytes: u64) -> &Attribute; + pub fn LLVMRustCreateDereferenceableOrNullAttr(C: &Context, bytes: u64) -> &Attribute; + pub fn LLVMRustCreateByValAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateStructRetAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateElementTypeAttr<'a>(C: &'a Context, ty: &'a Type) -> &'a Attribute; + pub fn LLVMRustCreateUWTableAttr(C: &Context, async_: bool) -> &Attribute; + pub fn LLVMRustCreateAllocSizeAttr(C: &Context, size_arg: u32) -> &Attribute; + pub fn LLVMRustCreateAllocKindAttr(C: &Context, size_arg: u64) -> &Attribute; + + // Operations on functions + pub fn LLVMRustGetOrInsertFunction<'a>( + M: &'a Module, + Name: *const c_char, + NameLen: size_t, + FunctionTy: &'a Type, + ) -> &'a Value; + pub fn LLVMSetFunctionCallConv(Fn: &Value, CC: c_uint); + pub fn LLVMRustAddFunctionAttributes<'a>( + Fn: &'a Value, + index: c_uint, + Attrs: *const &'a Attribute, + AttrsLen: size_t, + ); + + // Operations on parameters + pub fn LLVMIsAArgument(Val: &Value) -> Option<&Value>; + pub fn LLVMCountParams(Fn: &Value) -> c_uint; + pub fn LLVMGetParam(Fn: &Value, Index: c_uint) -> &Value; + + // Operations on basic blocks + pub fn LLVMGetBasicBlockParent(BB: &BasicBlock) -> &Value; + pub fn LLVMAppendBasicBlockInContext<'a>( + C: &'a Context, + Fn: &'a Value, + Name: *const c_char, + ) -> &'a BasicBlock; + + // Operations on instructions + pub fn LLVMIsAInstruction(Val: &Value) -> Option<&Value>; + pub fn LLVMGetFirstBasicBlock(Fn: &Value) -> &BasicBlock; + + // Operations on call sites + pub fn LLVMSetInstructionCallConv(Instr: &Value, CC: c_uint); + pub fn LLVMRustAddCallSiteAttributes<'a>( + Instr: &'a Value, + index: c_uint, + Attrs: *const &'a Attribute, + AttrsLen: size_t, + ); + + // Operations on load/store instructions (only) + pub fn LLVMSetVolatile(MemoryAccessInst: &Value, volatile: Bool); + + // Operations on phi nodes + pub fn LLVMAddIncoming<'a>( + PhiNode: &'a Value, + IncomingValues: *const &'a Value, + IncomingBlocks: *const &'a BasicBlock, + Count: c_uint, + ); + + // Instruction builders + pub fn LLVMCreateBuilderInContext(C: &Context) -> &mut Builder<'_>; + pub fn LLVMPositionBuilderAtEnd<'a>(Builder: &Builder<'a>, Block: &'a BasicBlock); + pub fn LLVMGetInsertBlock<'a>(Builder: &Builder<'a>) -> &'a BasicBlock; + pub fn LLVMDisposeBuilder<'a>(Builder: &'a mut Builder<'a>); + + // Metadata + pub fn LLVMSetCurrentDebugLocation<'a>(Builder: &Builder<'a>, L: &'a Value); + + // Terminators + pub fn LLVMBuildRetVoid<'a>(B: &Builder<'a>) -> &'a Value; + pub fn LLVMBuildRet<'a>(B: &Builder<'a>, V: &'a Value) -> &'a Value; + pub fn LLVMBuildBr<'a>(B: &Builder<'a>, Dest: &'a BasicBlock) -> &'a Value; + pub fn LLVMBuildCondBr<'a>( + B: &Builder<'a>, + If: &'a Value, + Then: &'a BasicBlock, + Else: &'a BasicBlock, + ) -> &'a Value; + pub fn LLVMBuildSwitch<'a>( + B: &Builder<'a>, + V: &'a Value, + Else: &'a BasicBlock, + NumCases: c_uint, + ) -> &'a Value; + pub fn LLVMRustBuildInvoke<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Then: &'a BasicBlock, + Catch: &'a BasicBlock, + Bundle: Option<&OperandBundleDef<'a>>, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildLandingPad<'a>( + B: &Builder<'a>, + Ty: &'a Type, + PersFn: Option<&'a Value>, + NumClauses: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildResume<'a>(B: &Builder<'a>, Exn: &'a Value) -> &'a Value; + pub fn LLVMBuildUnreachable<'a>(B: &Builder<'a>) -> &'a Value; + + pub fn LLVMRustBuildCleanupPad<'a>( + B: &Builder<'a>, + ParentPad: Option<&'a Value>, + ArgCnt: c_uint, + Args: *const &'a Value, + Name: *const c_char, + ) -> Option<&'a Value>; + pub fn LLVMRustBuildCleanupRet<'a>( + B: &Builder<'a>, + CleanupPad: &'a Value, + UnwindBB: Option<&'a BasicBlock>, + ) -> Option<&'a Value>; + pub fn LLVMRustBuildCatchPad<'a>( + B: &Builder<'a>, + ParentPad: &'a Value, + ArgCnt: c_uint, + Args: *const &'a Value, + Name: *const c_char, + ) -> Option<&'a Value>; + pub fn LLVMRustBuildCatchRet<'a>( + B: &Builder<'a>, + Pad: &'a Value, + BB: &'a BasicBlock, + ) -> Option<&'a Value>; + pub fn LLVMRustBuildCatchSwitch<'a>( + Builder: &Builder<'a>, + ParentPad: Option<&'a Value>, + BB: Option<&'a BasicBlock>, + NumHandlers: c_uint, + Name: *const c_char, + ) -> Option<&'a Value>; + pub fn LLVMRustAddHandler<'a>(CatchSwitch: &'a Value, Handler: &'a BasicBlock); + pub fn LLVMSetPersonalityFn<'a>(Func: &'a Value, Pers: &'a Value); + + // Add a case to the switch instruction + pub fn LLVMAddCase<'a>(Switch: &'a Value, OnVal: &'a Value, Dest: &'a BasicBlock); + + // Add a clause to the landing pad instruction + pub fn LLVMAddClause<'a>(LandingPad: &'a Value, ClauseVal: &'a Value); + + // Set the cleanup on a landing pad instruction + pub fn LLVMSetCleanup(LandingPad: &Value, Val: Bool); + + // Arithmetic + pub fn LLVMBuildAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildUDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildExactUDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildExactSDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFDiv<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildURem<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSRem<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFRem<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildShl<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildLShr<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildAShr<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNSWAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNUWAdd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNSWSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNUWSub<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNSWMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNUWMul<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildAnd<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildOr<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildXor<'a>( + B: &Builder<'a>, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; + pub fn LLVMBuildFNeg<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; + pub fn LLVMBuildNot<'a>(B: &Builder<'a>, V: &'a Value, Name: *const c_char) -> &'a Value; + pub fn LLVMRustSetFastMath(Instr: &Value); + + // Memory + pub fn LLVMBuildAlloca<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; + pub fn LLVMBuildArrayAlloca<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Val: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildLoad2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + PointerVal: &'a Value, + Name: *const c_char, + ) -> &'a Value; + + pub fn LLVMBuildStore<'a>(B: &Builder<'a>, Val: &'a Value, Ptr: &'a Value) -> &'a Value; + + pub fn LLVMBuildGEP2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Pointer: &'a Value, + Indices: *const &'a Value, + NumIndices: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildInBoundsGEP2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Pointer: &'a Value, + Indices: *const &'a Value, + NumIndices: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildStructGEP2<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Pointer: &'a Value, + Idx: c_uint, + Name: *const c_char, + ) -> &'a Value; + + // Casts + pub fn LLVMBuildTrunc<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildZExt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSExt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFPToUI<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFPToSI<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildUIToFP<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildSIToFP<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFPTrunc<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFPExt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildPtrToInt<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildIntToPtr<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildBitCast<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildPointerCast<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMRustBuildIntCast<'a>( + B: &Builder<'a>, + Val: &'a Value, + DestTy: &'a Type, + IsSigned: bool, + ) -> &'a Value; + + // Comparisons + pub fn LLVMBuildICmp<'a>( + B: &Builder<'a>, + Op: c_uint, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildFCmp<'a>( + B: &Builder<'a>, + Op: c_uint, + LHS: &'a Value, + RHS: &'a Value, + Name: *const c_char, + ) -> &'a Value; + + // Miscellaneous instructions + pub fn LLVMBuildPhi<'a>(B: &Builder<'a>, Ty: &'a Type, Name: *const c_char) -> &'a Value; + pub fn LLVMRustGetInstrProfIncrementIntrinsic(M: &Module) -> &Value; + pub fn LLVMRustBuildCall<'a>( + B: &Builder<'a>, + Ty: &'a Type, + Fn: &'a Value, + Args: *const &'a Value, + NumArgs: c_uint, + Bundle: Option<&OperandBundleDef<'a>>, + ) -> &'a Value; + pub fn LLVMRustBuildMemCpy<'a>( + B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + IsVolatile: bool, + ) -> &'a Value; + pub fn LLVMRustBuildMemMove<'a>( + B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Src: &'a Value, + SrcAlign: c_uint, + Size: &'a Value, + IsVolatile: bool, + ) -> &'a Value; + pub fn LLVMRustBuildMemSet<'a>( + B: &Builder<'a>, + Dst: &'a Value, + DstAlign: c_uint, + Val: &'a Value, + Size: &'a Value, + IsVolatile: bool, + ) -> &'a Value; + pub fn LLVMBuildSelect<'a>( + B: &Builder<'a>, + If: &'a Value, + Then: &'a Value, + Else: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildVAArg<'a>( + B: &Builder<'a>, + list: &'a Value, + Ty: &'a Type, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildExtractElement<'a>( + B: &Builder<'a>, + VecVal: &'a Value, + Index: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildInsertElement<'a>( + B: &Builder<'a>, + VecVal: &'a Value, + EltVal: &'a Value, + Index: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildShuffleVector<'a>( + B: &Builder<'a>, + V1: &'a Value, + V2: &'a Value, + Mask: &'a Value, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildExtractValue<'a>( + B: &Builder<'a>, + AggVal: &'a Value, + Index: c_uint, + Name: *const c_char, + ) -> &'a Value; + pub fn LLVMBuildInsertValue<'a>( + B: &Builder<'a>, + AggVal: &'a Value, + EltVal: &'a Value, + Index: c_uint, + Name: *const c_char, + ) -> &'a Value; + + pub fn LLVMRustBuildVectorReduceFAdd<'a>( + B: &Builder<'a>, + Acc: &'a Value, + Src: &'a Value, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceFMul<'a>( + B: &Builder<'a>, + Acc: &'a Value, + Src: &'a Value, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceAdd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceMul<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceAnd<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceOr<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceXor<'a>(B: &Builder<'a>, Src: &'a Value) -> &'a Value; + pub fn LLVMRustBuildVectorReduceMin<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsSigned: bool, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceMax<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsSigned: bool, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceFMin<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsNaN: bool, + ) -> &'a Value; + pub fn LLVMRustBuildVectorReduceFMax<'a>( + B: &Builder<'a>, + Src: &'a Value, + IsNaN: bool, + ) -> &'a Value; + + pub fn LLVMRustBuildMinNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; + pub fn LLVMRustBuildMaxNum<'a>(B: &Builder<'a>, LHS: &'a Value, LHS: &'a Value) -> &'a Value; + + // Atomic Operations + pub fn LLVMRustBuildAtomicLoad<'a>( + B: &Builder<'a>, + ElementType: &'a Type, + PointerVal: &'a Value, + Name: *const c_char, + Order: AtomicOrdering, + ) -> &'a Value; + + pub fn LLVMRustBuildAtomicStore<'a>( + B: &Builder<'a>, + Val: &'a Value, + Ptr: &'a Value, + Order: AtomicOrdering, + ) -> &'a Value; + + pub fn LLVMRustBuildAtomicCmpXchg<'a>( + B: &Builder<'a>, + LHS: &'a Value, + CMP: &'a Value, + RHS: &'a Value, + Order: AtomicOrdering, + FailureOrder: AtomicOrdering, + Weak: Bool, + ) -> &'a Value; + + pub fn LLVMBuildAtomicRMW<'a>( + B: &Builder<'a>, + Op: AtomicRmwBinOp, + LHS: &'a Value, + RHS: &'a Value, + Order: AtomicOrdering, + SingleThreaded: Bool, + ) -> &'a Value; + + pub fn LLVMRustBuildAtomicFence( + B: &Builder<'_>, + Order: AtomicOrdering, + Scope: SynchronizationScope, + ); + + /// Writes a module to the specified path. Returns 0 on success. + pub fn LLVMWriteBitcodeToFile(M: &Module, Path: *const c_char) -> c_int; + + /// Creates a pass manager. + pub fn LLVMCreatePassManager<'a>() -> &'a mut PassManager<'a>; + + /// Creates a function-by-function pass manager + pub fn LLVMCreateFunctionPassManagerForModule(M: &Module) -> &mut PassManager<'_>; + + /// Disposes a pass manager. + pub fn LLVMDisposePassManager<'a>(PM: &'a mut PassManager<'a>); + + /// Runs a pass manager on a module. + pub fn LLVMRunPassManager<'a>(PM: &PassManager<'a>, M: &'a Module) -> Bool; + + pub fn LLVMInitializePasses(); + + pub fn LLVMTimeTraceProfilerInitialize(); + + pub fn LLVMTimeTraceProfilerFinishThread(); + + pub fn LLVMTimeTraceProfilerFinish(FileName: *const c_char); + + pub fn LLVMAddAnalysisPasses<'a>(T: &'a TargetMachine, PM: &PassManager<'a>); + + pub fn LLVMRustPassManagerBuilderCreate() -> &'static mut PassManagerBuilder; + pub fn LLVMRustPassManagerBuilderDispose(PMB: &'static mut PassManagerBuilder); + pub fn LLVMRustPassManagerBuilderUseInlinerWithThreshold( + PMB: &PassManagerBuilder, + threshold: c_uint, + ); + pub fn LLVMRustPassManagerBuilderPopulateModulePassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + ); + + pub fn LLVMRustPassManagerBuilderPopulateFunctionPassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + ); + pub fn LLVMRustPassManagerBuilderPopulateLTOPassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + Internalize: Bool, + RunInliner: Bool, + ); + pub fn LLVMRustPassManagerBuilderPopulateThinLTOPassManager( + PMB: &PassManagerBuilder, + PM: &PassManager<'_>, + ); + + pub fn LLVMGetHostCPUFeatures() -> *mut c_char; + + pub fn LLVMDisposeMessage(message: *mut c_char); + + pub fn LLVMIsMultithreaded() -> Bool; + + /// Returns a string describing the last error caused by an LLVMRust* call. + pub fn LLVMRustGetLastError() -> *const c_char; + + /// Print the pass timings since static dtors aren't picking them up. + pub fn LLVMRustPrintPassTimings(); + + pub fn LLVMStructCreateNamed(C: &Context, Name: *const c_char) -> &Type; + + pub fn LLVMStructSetBody<'a>( + StructTy: &'a Type, + ElementTypes: *const &'a Type, + ElementCount: c_uint, + Packed: Bool, + ); + + /// Prepares inline assembly. + pub fn LLVMRustInlineAsm( + Ty: &Type, + AsmString: *const c_char, + AsmStringLen: size_t, + Constraints: *const c_char, + ConstraintsLen: size_t, + SideEffects: Bool, + AlignStack: Bool, + Dialect: AsmDialect, + CanThrow: Bool, + ) -> &Value; + pub fn LLVMRustInlineAsmVerify( + Ty: &Type, + Constraints: *const c_char, + ConstraintsLen: size_t, + ) -> bool; + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteFilenamesSectionToBuffer( + Filenames: *const *const c_char, + FilenamesLen: size_t, + BufferOut: &RustString, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMappingToBuffer( + VirtualFileMappingIDs: *const c_uint, + NumVirtualFileMappingIDs: c_uint, + Expressions: *const coverage_map::CounterExpression, + NumExpressions: c_uint, + MappingRegions: *const coverageinfo::CounterMappingRegion, + NumMappingRegions: c_uint, + BufferOut: &RustString, + ); + + pub fn LLVMRustCoverageCreatePGOFuncNameVar(F: &Value, FuncName: *const c_char) -> &Value; + pub fn LLVMRustCoverageHashCString(StrVal: *const c_char) -> u64; + pub fn LLVMRustCoverageHashByteArray(Bytes: *const c_char, NumBytes: size_t) -> u64; + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMapSectionNameToString(M: &Module, Str: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteFuncSectionNameToString(M: &Module, Str: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustCoverageWriteMappingVarNameToString(Str: &RustString); + + pub fn LLVMRustCoverageMappingVersion() -> u32; + pub fn LLVMRustDebugMetadataVersion() -> u32; + pub fn LLVMRustVersionMajor() -> u32; + pub fn LLVMRustVersionMinor() -> u32; + pub fn LLVMRustVersionPatch() -> u32; + + /// Add LLVM module flags. + /// + /// In order for Rust-C LTO to work, module flags must be compatible with Clang. What + /// "compatible" means depends on the merge behaviors involved. + pub fn LLVMRustAddModuleFlag( + M: &Module, + merge_behavior: LLVMModFlagBehavior, + name: *const c_char, + value: u32, + ); + pub fn LLVMRustHasModuleFlag(M: &Module, name: *const c_char, len: size_t) -> bool; + + pub fn LLVMRustMetadataAsValue<'a>(C: &'a Context, MD: &'a Metadata) -> &'a Value; + + pub fn LLVMRustDIBuilderCreate(M: &Module) -> &mut DIBuilder<'_>; + + pub fn LLVMRustDIBuilderDispose<'a>(Builder: &'a mut DIBuilder<'a>); + + pub fn LLVMRustDIBuilderFinalize(Builder: &DIBuilder<'_>); + + pub fn LLVMRustDIBuilderCreateCompileUnit<'a>( + Builder: &DIBuilder<'a>, + Lang: c_uint, + File: &'a DIFile, + Producer: *const c_char, + ProducerLen: size_t, + isOptimized: bool, + Flags: *const c_char, + RuntimeVer: c_uint, + SplitName: *const c_char, + SplitNameLen: size_t, + kind: DebugEmissionKind, + DWOId: u64, + SplitDebugInlining: bool, + ) -> &'a DIDescriptor; + + pub fn LLVMRustDIBuilderCreateFile<'a>( + Builder: &DIBuilder<'a>, + Filename: *const c_char, + FilenameLen: size_t, + Directory: *const c_char, + DirectoryLen: size_t, + CSKind: ChecksumKind, + Checksum: *const c_char, + ChecksumLen: size_t, + ) -> &'a DIFile; + + pub fn LLVMRustDIBuilderCreateSubroutineType<'a>( + Builder: &DIBuilder<'a>, + ParameterTypes: &'a DIArray, + ) -> &'a DICompositeType; + + pub fn LLVMRustDIBuilderCreateFunction<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + ScopeLine: c_uint, + Flags: DIFlags, + SPFlags: DISPFlags, + MaybeFn: Option<&'a Value>, + TParam: &'a DIArray, + Decl: Option<&'a DIDescriptor>, + ) -> &'a DISubprogram; + + pub fn LLVMRustDIBuilderCreateBasicType<'a>( + Builder: &DIBuilder<'a>, + Name: *const c_char, + NameLen: size_t, + SizeInBits: u64, + Encoding: c_uint, + ) -> &'a DIBasicType; + + pub fn LLVMRustDIBuilderCreateTypedef<'a>( + Builder: &DIBuilder<'a>, + Type: &'a DIBasicType, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Scope: Option<&'a DIScope>, + ) -> &'a DIDerivedType; + + pub fn LLVMRustDIBuilderCreatePointerType<'a>( + Builder: &DIBuilder<'a>, + PointeeTy: &'a DIType, + SizeInBits: u64, + AlignInBits: u32, + AddressSpace: c_uint, + Name: *const c_char, + NameLen: size_t, + ) -> &'a DIDerivedType; + + pub fn LLVMRustDIBuilderCreateStructType<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIDescriptor>, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + DerivedFrom: Option<&'a DIType>, + Elements: &'a DIArray, + RunTimeLang: c_uint, + VTableHolder: Option<&'a DIType>, + UniqueId: *const c_char, + UniqueIdLen: size_t, + ) -> &'a DICompositeType; + + pub fn LLVMRustDIBuilderCreateMemberType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Flags: DIFlags, + Ty: &'a DIType, + ) -> &'a DIDerivedType; + + pub fn LLVMRustDIBuilderCreateVariantMemberType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + OffsetInBits: u64, + Discriminant: Option<&'a Value>, + Flags: DIFlags, + Ty: &'a DIType, + ) -> &'a DIType; + + pub fn LLVMRustDIBuilderCreateLexicalBlock<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + File: &'a DIFile, + Line: c_uint, + Col: c_uint, + ) -> &'a DILexicalBlock; + + pub fn LLVMRustDIBuilderCreateLexicalBlockFile<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + File: &'a DIFile, + ) -> &'a DILexicalBlock; + + pub fn LLVMRustDIBuilderCreateStaticVariable<'a>( + Builder: &DIBuilder<'a>, + Context: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + LinkageName: *const c_char, + LinkageNameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + isLocalToUnit: bool, + Val: &'a Value, + Decl: Option<&'a DIDescriptor>, + AlignInBits: u32, + ) -> &'a DIGlobalVariableExpression; + + pub fn LLVMRustDIBuilderCreateVariable<'a>( + Builder: &DIBuilder<'a>, + Tag: c_uint, + Scope: &'a DIDescriptor, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + Ty: &'a DIType, + AlwaysPreserve: bool, + Flags: DIFlags, + ArgNo: c_uint, + AlignInBits: u32, + ) -> &'a DIVariable; + + pub fn LLVMRustDIBuilderCreateArrayType<'a>( + Builder: &DIBuilder<'a>, + Size: u64, + AlignInBits: u32, + Ty: &'a DIType, + Subscripts: &'a DIArray, + ) -> &'a DIType; + + pub fn LLVMRustDIBuilderGetOrCreateSubrange<'a>( + Builder: &DIBuilder<'a>, + Lo: i64, + Count: i64, + ) -> &'a DISubrange; + + pub fn LLVMRustDIBuilderGetOrCreateArray<'a>( + Builder: &DIBuilder<'a>, + Ptr: *const Option<&'a DIDescriptor>, + Count: c_uint, + ) -> &'a DIArray; + + pub fn LLVMRustDIBuilderInsertDeclareAtEnd<'a>( + Builder: &DIBuilder<'a>, + Val: &'a Value, + VarInfo: &'a DIVariable, + AddrOps: *const u64, + AddrOpsCount: c_uint, + DL: &'a DILocation, + InsertAtEnd: &'a BasicBlock, + ) -> &'a Value; + + pub fn LLVMRustDIBuilderCreateEnumerator<'a>( + Builder: &DIBuilder<'a>, + Name: *const c_char, + NameLen: size_t, + Value: i64, + IsUnsigned: bool, + ) -> &'a DIEnumerator; + + pub fn LLVMRustDIBuilderCreateEnumerationType<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Elements: &'a DIArray, + ClassType: &'a DIType, + IsScoped: bool, + ) -> &'a DIType; + + pub fn LLVMRustDIBuilderCreateUnionType<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNumber: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + Elements: Option<&'a DIArray>, + RunTimeLang: c_uint, + UniqueId: *const c_char, + UniqueIdLen: size_t, + ) -> &'a DIType; + + pub fn LLVMRustDIBuilderCreateVariantPart<'a>( + Builder: &DIBuilder<'a>, + Scope: &'a DIScope, + Name: *const c_char, + NameLen: size_t, + File: &'a DIFile, + LineNo: c_uint, + SizeInBits: u64, + AlignInBits: u32, + Flags: DIFlags, + Discriminator: Option<&'a DIDerivedType>, + Elements: &'a DIArray, + UniqueId: *const c_char, + UniqueIdLen: size_t, + ) -> &'a DIDerivedType; + + pub fn LLVMSetUnnamedAddress(Global: &Value, UnnamedAddr: UnnamedAddr); + + pub fn LLVMRustDIBuilderCreateTemplateTypeParameter<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + Ty: &'a DIType, + ) -> &'a DITemplateTypeParameter; + + pub fn LLVMRustDIBuilderCreateNameSpace<'a>( + Builder: &DIBuilder<'a>, + Scope: Option<&'a DIScope>, + Name: *const c_char, + NameLen: size_t, + ExportSymbols: bool, + ) -> &'a DINameSpace; + + pub fn LLVMRustDICompositeTypeReplaceArrays<'a>( + Builder: &DIBuilder<'a>, + CompositeType: &'a DIType, + Elements: Option<&'a DIArray>, + Params: Option<&'a DIArray>, + ); + + pub fn LLVMRustDIBuilderCreateDebugLocation<'a>( + Line: c_uint, + Column: c_uint, + Scope: &'a DIScope, + InlinedAt: Option<&'a DILocation>, + ) -> &'a DILocation; + pub fn LLVMRustDIBuilderCreateOpDeref() -> u64; + pub fn LLVMRustDIBuilderCreateOpPlusUconst() -> u64; + + #[allow(improper_ctypes)] + pub fn LLVMRustWriteTypeToString(Type: &Type, s: &RustString); + #[allow(improper_ctypes)] + pub fn LLVMRustWriteValueToString(value_ref: &Value, s: &RustString); + + pub fn LLVMIsAConstantInt(value_ref: &Value) -> Option<&ConstantInt>; + + pub fn LLVMRustFindAndCreatePass(Pass: *const c_char) -> Option<&'static mut Pass>; + pub fn LLVMRustCreateAddressSanitizerFunctionPass(Recover: bool) -> &'static mut Pass; + pub fn LLVMRustCreateModuleAddressSanitizerPass(Recover: bool) -> &'static mut Pass; + pub fn LLVMRustCreateMemorySanitizerPass( + TrackOrigins: c_int, + Recover: bool, + ) -> &'static mut Pass; + pub fn LLVMRustCreateThreadSanitizerPass() -> &'static mut Pass; + pub fn LLVMRustCreateHWAddressSanitizerPass(Recover: bool) -> &'static mut Pass; + pub fn LLVMRustAddPass(PM: &PassManager<'_>, Pass: &'static mut Pass); + pub fn LLVMRustAddLastExtensionPasses( + PMB: &PassManagerBuilder, + Passes: *const &'static mut Pass, + NumPasses: size_t, + ); + + pub fn LLVMRustHasFeature(T: &TargetMachine, s: *const c_char) -> bool; + + pub fn LLVMRustPrintTargetCPUs(T: &TargetMachine); + pub fn LLVMRustGetTargetFeaturesCount(T: &TargetMachine) -> size_t; + pub fn LLVMRustGetTargetFeature( + T: &TargetMachine, + Index: size_t, + Feature: &mut *const c_char, + Desc: &mut *const c_char, + ); + + pub fn LLVMRustGetHostCPUName(len: *mut usize) -> *const c_char; + pub fn LLVMRustCreateTargetMachine( + Triple: *const c_char, + CPU: *const c_char, + Features: *const c_char, + Abi: *const c_char, + Model: CodeModel, + Reloc: RelocModel, + Level: CodeGenOptLevel, + UseSoftFP: bool, + FunctionSections: bool, + DataSections: bool, + UniqueSectionNames: bool, + TrapUnreachable: bool, + Singlethread: bool, + AsmComments: bool, + EmitStackSizeSection: bool, + RelaxELFRelocations: bool, + UseInitArray: bool, + SplitDwarfFile: *const c_char, + ) -> Option<&'static mut TargetMachine>; + pub fn LLVMRustDisposeTargetMachine(T: &'static mut TargetMachine); + pub fn LLVMRustAddBuilderLibraryInfo<'a>( + PMB: &'a PassManagerBuilder, + M: &'a Module, + DisableSimplifyLibCalls: bool, + ); + pub fn LLVMRustConfigurePassManagerBuilder( + PMB: &PassManagerBuilder, + OptLevel: CodeGenOptLevel, + MergeFunctions: bool, + SLPVectorize: bool, + LoopVectorize: bool, + PrepareForThinLTO: bool, + PGOGenPath: *const c_char, + PGOUsePath: *const c_char, + PGOSampleUsePath: *const c_char, + SizeLevel: c_int, + ); + pub fn LLVMRustAddLibraryInfo<'a>( + PM: &PassManager<'a>, + M: &'a Module, + DisableSimplifyLibCalls: bool, + ); + pub fn LLVMRustRunFunctionPassManager<'a>(PM: &PassManager<'a>, M: &'a Module); + pub fn LLVMRustWriteOutputFile<'a>( + T: &'a TargetMachine, + PM: &PassManager<'a>, + M: &'a Module, + Output: *const c_char, + DwoOutput: *const c_char, + FileType: FileType, + ) -> LLVMRustResult; + pub fn LLVMRustOptimizeWithNewPassManager<'a>( + M: &'a Module, + TM: &'a TargetMachine, + OptLevel: PassBuilderOptLevel, + OptStage: OptStage, + NoPrepopulatePasses: bool, + VerifyIR: bool, + UseThinLTOBuffers: bool, + MergeFunctions: bool, + UnrollLoops: bool, + SLPVectorize: bool, + LoopVectorize: bool, + DisableSimplifyLibCalls: bool, + EmitLifetimeMarkers: bool, + SanitizerOptions: Option<&SanitizerOptions>, + PGOGenPath: *const c_char, + PGOUsePath: *const c_char, + InstrumentCoverage: bool, + InstrumentGCOV: bool, + PGOSampleUsePath: *const c_char, + DebugInfoForProfiling: bool, + llvm_selfprofiler: *mut c_void, + begin_callback: SelfProfileBeforePassCallback, + end_callback: SelfProfileAfterPassCallback, + ExtraPasses: *const c_char, + ExtraPassesLen: size_t, + LLVMPlugins: *const c_char, + LLVMPluginsLen: size_t, + ) -> LLVMRustResult; + pub fn LLVMRustPrintModule( + M: &Module, + Output: *const c_char, + Demangle: extern "C" fn(*const c_char, size_t, *mut c_char, size_t) -> size_t, + ) -> LLVMRustResult; + pub fn LLVMRustSetLLVMOptions(Argc: c_int, Argv: *const *const c_char); + pub fn LLVMRustPrintPasses(); + pub fn LLVMRustSetNormalizedTarget(M: &Module, triple: *const c_char); + pub fn LLVMRustAddAlwaysInlinePass(P: &PassManagerBuilder, AddLifetimes: bool); + pub fn LLVMRustRunRestrictionPass(M: &Module, syms: *const *const c_char, len: size_t); + + pub fn LLVMRustOpenArchive(path: *const c_char) -> Option<&'static mut Archive>; + pub fn LLVMRustArchiveIteratorNew(AR: &Archive) -> &mut ArchiveIterator<'_>; + pub fn LLVMRustArchiveIteratorNext<'a>( + AIR: &ArchiveIterator<'a>, + ) -> Option<&'a mut ArchiveChild<'a>>; + pub fn LLVMRustArchiveChildName(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; + pub fn LLVMRustArchiveChildData(ACR: &ArchiveChild<'_>, size: &mut size_t) -> *const c_char; + pub fn LLVMRustArchiveChildFree<'a>(ACR: &'a mut ArchiveChild<'a>); + pub fn LLVMRustArchiveIteratorFree<'a>(AIR: &'a mut ArchiveIterator<'a>); + pub fn LLVMRustDestroyArchive(AR: &'static mut Archive); + + #[allow(improper_ctypes)] + pub fn LLVMRustWriteTwineToString(T: &Twine, s: &RustString); + + #[allow(improper_ctypes)] + pub fn LLVMRustUnpackOptimizationDiagnostic<'a>( + DI: &'a DiagnosticInfo, + pass_name_out: &RustString, + function_out: &mut Option<&'a Value>, + loc_line_out: &mut c_uint, + loc_column_out: &mut c_uint, + loc_filename_out: &RustString, + message_out: &RustString, + ); + + pub fn LLVMRustUnpackInlineAsmDiagnostic<'a>( + DI: &'a DiagnosticInfo, + level_out: &mut DiagnosticLevel, + cookie_out: &mut c_uint, + message_out: &mut Option<&'a Twine>, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustWriteDiagnosticInfoToString(DI: &DiagnosticInfo, s: &RustString); + pub fn LLVMRustGetDiagInfoKind(DI: &DiagnosticInfo) -> DiagnosticKind; + + pub fn LLVMRustGetSMDiagnostic<'a>( + DI: &'a DiagnosticInfo, + cookie_out: &mut c_uint, + ) -> &'a SMDiagnostic; + + pub fn LLVMRustSetInlineAsmDiagnosticHandler( + C: &Context, + H: InlineAsmDiagHandlerTy, + CX: *mut c_void, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustUnpackSMDiagnostic( + d: &SMDiagnostic, + message_out: &RustString, + buffer_out: &RustString, + level_out: &mut DiagnosticLevel, + loc_out: &mut c_uint, + ranges_out: *mut c_uint, + num_ranges: &mut usize, + ) -> bool; + + pub fn LLVMRustWriteArchive( + Dst: *const c_char, + NumMembers: size_t, + Members: *const &RustArchiveMember<'_>, + WriteSymbtab: bool, + Kind: ArchiveKind, + ) -> LLVMRustResult; + pub fn LLVMRustArchiveMemberNew<'a>( + Filename: *const c_char, + Name: *const c_char, + Child: Option<&ArchiveChild<'a>>, + ) -> &'a mut RustArchiveMember<'a>; + pub fn LLVMRustArchiveMemberFree<'a>(Member: &'a mut RustArchiveMember<'a>); + + pub fn LLVMRustWriteImportLibrary( + ImportName: *const c_char, + Path: *const c_char, + Exports: *const LLVMRustCOFFShortExport, + NumExports: usize, + Machine: u16, + MinGW: bool, + ) -> LLVMRustResult; + + pub fn LLVMRustSetDataLayoutFromTargetMachine<'a>(M: &'a Module, TM: &'a TargetMachine); + + pub fn LLVMRustBuildOperandBundleDef<'a>( + Name: *const c_char, + Inputs: *const &'a Value, + NumInputs: c_uint, + ) -> &'a mut OperandBundleDef<'a>; + pub fn LLVMRustFreeOperandBundleDef<'a>(Bundle: &'a mut OperandBundleDef<'a>); + + pub fn LLVMRustPositionBuilderAtStart<'a>(B: &Builder<'a>, BB: &'a BasicBlock); + + pub fn LLVMRustSetComdat<'a>(M: &'a Module, V: &'a Value, Name: *const c_char, NameLen: size_t); + pub fn LLVMRustSetModulePICLevel(M: &Module); + pub fn LLVMRustSetModulePIELevel(M: &Module); + pub fn LLVMRustSetModuleCodeModel(M: &Module, Model: CodeModel); + pub fn LLVMRustModuleBufferCreate(M: &Module) -> &'static mut ModuleBuffer; + pub fn LLVMRustModuleBufferPtr(p: &ModuleBuffer) -> *const u8; + pub fn LLVMRustModuleBufferLen(p: &ModuleBuffer) -> usize; + pub fn LLVMRustModuleBufferFree(p: &'static mut ModuleBuffer); + pub fn LLVMRustModuleCost(M: &Module) -> u64; + + pub fn LLVMRustThinLTOBufferCreate(M: &Module, is_thin: bool) -> &'static mut ThinLTOBuffer; + pub fn LLVMRustThinLTOBufferFree(M: &'static mut ThinLTOBuffer); + pub fn LLVMRustThinLTOBufferPtr(M: &ThinLTOBuffer) -> *const c_char; + pub fn LLVMRustThinLTOBufferLen(M: &ThinLTOBuffer) -> size_t; + pub fn LLVMRustCreateThinLTOData( + Modules: *const ThinLTOModule, + NumModules: c_uint, + PreservedSymbols: *const *const c_char, + PreservedSymbolsLen: c_uint, + ) -> Option<&'static mut ThinLTOData>; + pub fn LLVMRustPrepareThinLTORename( + Data: &ThinLTOData, + Module: &Module, + Target: &TargetMachine, + ) -> bool; + pub fn LLVMRustPrepareThinLTOResolveWeak(Data: &ThinLTOData, Module: &Module) -> bool; + pub fn LLVMRustPrepareThinLTOInternalize(Data: &ThinLTOData, Module: &Module) -> bool; + pub fn LLVMRustPrepareThinLTOImport( + Data: &ThinLTOData, + Module: &Module, + Target: &TargetMachine, + ) -> bool; + pub fn LLVMRustFreeThinLTOData(Data: &'static mut ThinLTOData); + pub fn LLVMRustParseBitcodeForLTO( + Context: &Context, + Data: *const u8, + len: usize, + Identifier: *const c_char, + ) -> Option<&Module>; + pub fn LLVMRustGetBitcodeSliceFromObjectData( + Data: *const u8, + len: usize, + out_len: &mut usize, + ) -> *const u8; + pub fn LLVMRustThinLTOGetDICompileUnit( + M: &Module, + CU1: &mut *mut c_void, + CU2: &mut *mut c_void, + ); + pub fn LLVMRustThinLTOPatchDICompileUnit(M: &Module, CU: *mut c_void); + + pub fn LLVMRustLinkerNew(M: &Module) -> &mut Linker<'_>; + pub fn LLVMRustLinkerAdd( + linker: &Linker<'_>, + bytecode: *const c_char, + bytecode_len: usize, + ) -> bool; + pub fn LLVMRustLinkerFree<'a>(linker: &'a mut Linker<'a>); + #[allow(improper_ctypes)] + pub fn LLVMRustComputeLTOCacheKey( + key_out: &RustString, + mod_id: *const c_char, + data: &ThinLTOData, + ); + + pub fn LLVMRustContextGetDiagnosticHandler(Context: &Context) -> Option<&DiagnosticHandler>; + pub fn LLVMRustContextSetDiagnosticHandler( + context: &Context, + diagnostic_handler: Option<&DiagnosticHandler>, + ); + pub fn LLVMRustContextConfigureDiagnosticHandler( + context: &Context, + diagnostic_handler_callback: DiagnosticHandlerTy, + diagnostic_handler_context: *mut c_void, + remark_all_passes: bool, + remark_passes: *const *const c_char, + remark_passes_len: usize, + ); + + #[allow(improper_ctypes)] + pub fn LLVMRustGetMangledName(V: &Value, out: &RustString); + + pub fn LLVMRustGetElementTypeArgIndex(CallSite: &Value) -> i32; +} diff --git a/compiler/rustc_codegen_llvm/src/llvm/mod.rs b/compiler/rustc_codegen_llvm/src/llvm/mod.rs new file mode 100644 index 000000000..6602a4ab8 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm/mod.rs @@ -0,0 +1,318 @@ +#![allow(non_snake_case)] + +pub use self::AtomicRmwBinOp::*; +pub use self::CallConv::*; +pub use self::CodeGenOptSize::*; +pub use self::IntPredicate::*; +pub use self::Linkage::*; +pub use self::MetadataType::*; +pub use self::RealPredicate::*; + +use libc::c_uint; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_llvm::RustString; +use std::cell::RefCell; +use std::ffi::{CStr, CString}; +use std::str::FromStr; +use std::string::FromUtf8Error; + +pub mod archive_ro; +pub mod diagnostic; +mod ffi; + +pub use self::ffi::*; + +impl LLVMRustResult { + pub fn into_result(self) -> Result<(), ()> { + match self { + LLVMRustResult::Success => Ok(()), + LLVMRustResult::Failure => Err(()), + } + } +} + +pub fn AddFunctionAttributes<'ll>(llfn: &'ll Value, idx: AttributePlace, attrs: &[&'ll Attribute]) { + unsafe { + LLVMRustAddFunctionAttributes(llfn, idx.as_uint(), attrs.as_ptr(), attrs.len()); + } +} + +pub fn AddCallSiteAttributes<'ll>( + callsite: &'ll Value, + idx: AttributePlace, + attrs: &[&'ll Attribute], +) { + unsafe { + LLVMRustAddCallSiteAttributes(callsite, idx.as_uint(), attrs.as_ptr(), attrs.len()); + } +} + +pub fn CreateAttrStringValue<'ll>(llcx: &'ll Context, attr: &str, value: &str) -> &'ll Attribute { + unsafe { + LLVMCreateStringAttribute( + llcx, + attr.as_ptr().cast(), + attr.len().try_into().unwrap(), + value.as_ptr().cast(), + value.len().try_into().unwrap(), + ) + } +} + +pub fn CreateAttrString<'ll>(llcx: &'ll Context, attr: &str) -> &'ll Attribute { + unsafe { + LLVMCreateStringAttribute( + llcx, + attr.as_ptr().cast(), + attr.len().try_into().unwrap(), + std::ptr::null(), + 0, + ) + } +} + +pub fn CreateAlignmentAttr(llcx: &Context, bytes: u64) -> &Attribute { + unsafe { LLVMRustCreateAlignmentAttr(llcx, bytes) } +} + +pub fn CreateDereferenceableAttr(llcx: &Context, bytes: u64) -> &Attribute { + unsafe { LLVMRustCreateDereferenceableAttr(llcx, bytes) } +} + +pub fn CreateDereferenceableOrNullAttr(llcx: &Context, bytes: u64) -> &Attribute { + unsafe { LLVMRustCreateDereferenceableOrNullAttr(llcx, bytes) } +} + +pub fn CreateByValAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { + unsafe { LLVMRustCreateByValAttr(llcx, ty) } +} + +pub fn CreateStructRetAttr<'ll>(llcx: &'ll Context, ty: &'ll Type) -> &'ll Attribute { + unsafe { LLVMRustCreateStructRetAttr(llcx, ty) } +} + +pub fn CreateUWTableAttr(llcx: &Context, async_: bool) -> &Attribute { + unsafe { LLVMRustCreateUWTableAttr(llcx, async_) } +} + +pub fn CreateAllocSizeAttr(llcx: &Context, size_arg: u32) -> &Attribute { + unsafe { LLVMRustCreateAllocSizeAttr(llcx, size_arg) } +} + +pub fn CreateAllocKindAttr(llcx: &Context, kind_arg: AllocKindFlags) -> &Attribute { + unsafe { LLVMRustCreateAllocKindAttr(llcx, kind_arg.bits()) } +} + +#[derive(Copy, Clone)] +pub enum AttributePlace { + ReturnValue, + Argument(u32), + Function, +} + +impl AttributePlace { + pub fn as_uint(self) -> c_uint { + match self { + AttributePlace::ReturnValue => 0, + AttributePlace::Argument(i) => 1 + i, + AttributePlace::Function => !0, + } + } +} + +#[derive(Copy, Clone, PartialEq)] +#[repr(C)] +pub enum CodeGenOptSize { + CodeGenOptSizeNone = 0, + CodeGenOptSizeDefault = 1, + CodeGenOptSizeAggressive = 2, +} + +impl FromStr for ArchiveKind { + type Err = (); + + fn from_str(s: &str) -> Result { + match s { + "gnu" => Ok(ArchiveKind::K_GNU), + "bsd" => Ok(ArchiveKind::K_BSD), + "darwin" => Ok(ArchiveKind::K_DARWIN), + "coff" => Ok(ArchiveKind::K_COFF), + _ => Err(()), + } + } +} + +pub fn SetInstructionCallConv(instr: &Value, cc: CallConv) { + unsafe { + LLVMSetInstructionCallConv(instr, cc as c_uint); + } +} +pub fn SetFunctionCallConv(fn_: &Value, cc: CallConv) { + unsafe { + LLVMSetFunctionCallConv(fn_, cc as c_uint); + } +} + +// Externally visible symbols that might appear in multiple codegen units need to appear in +// their own comdat section so that the duplicates can be discarded at link time. This can for +// example happen for generics when using multiple codegen units. This function simply uses the +// value's name as the comdat value to make sure that it is in a 1-to-1 relationship to the +// function. +// For more details on COMDAT sections see e.g., https://www.airs.com/blog/archives/52 +pub fn SetUniqueComdat(llmod: &Module, val: &Value) { + unsafe { + let name = get_value_name(val); + LLVMRustSetComdat(llmod, val, name.as_ptr().cast(), name.len()); + } +} + +pub fn SetUnnamedAddress(global: &Value, unnamed: UnnamedAddr) { + unsafe { + LLVMSetUnnamedAddress(global, unnamed); + } +} + +pub fn set_thread_local_mode(global: &Value, mode: ThreadLocalMode) { + unsafe { + LLVMSetThreadLocalMode(global, mode); + } +} + +impl AttributeKind { + /// Create an LLVM Attribute with no associated value. + pub fn create_attr(self, llcx: &Context) -> &Attribute { + unsafe { LLVMRustCreateAttrNoValue(llcx, self) } + } +} + +pub fn set_section(llglobal: &Value, section_name: &str) { + let section_name_cstr = CString::new(section_name).expect("unexpected CString error"); + unsafe { + LLVMSetSection(llglobal, section_name_cstr.as_ptr()); + } +} + +pub fn add_global<'a>(llmod: &'a Module, ty: &'a Type, name: &str) -> &'a Value { + let name_cstr = CString::new(name).expect("unexpected CString error"); + unsafe { LLVMAddGlobal(llmod, ty, name_cstr.as_ptr()) } +} + +pub fn set_initializer(llglobal: &Value, constant_val: &Value) { + unsafe { + LLVMSetInitializer(llglobal, constant_val); + } +} + +pub fn set_global_constant(llglobal: &Value, is_constant: bool) { + unsafe { + LLVMSetGlobalConstant(llglobal, if is_constant { ffi::True } else { ffi::False }); + } +} + +pub fn set_linkage(llglobal: &Value, linkage: Linkage) { + unsafe { + LLVMRustSetLinkage(llglobal, linkage); + } +} + +pub fn set_visibility(llglobal: &Value, visibility: Visibility) { + unsafe { + LLVMRustSetVisibility(llglobal, visibility); + } +} + +pub fn set_alignment(llglobal: &Value, bytes: usize) { + unsafe { + ffi::LLVMSetAlignment(llglobal, bytes as c_uint); + } +} + +pub fn set_comdat(llmod: &Module, llglobal: &Value, name: &str) { + unsafe { + LLVMRustSetComdat(llmod, llglobal, name.as_ptr().cast(), name.len()); + } +} + +/// Safe wrapper around `LLVMGetParam`, because segfaults are no fun. +pub fn get_param(llfn: &Value, index: c_uint) -> &Value { + unsafe { + assert!( + index < LLVMCountParams(llfn), + "out of bounds argument access: {} out of {} arguments", + index, + LLVMCountParams(llfn) + ); + LLVMGetParam(llfn, index) + } +} + +/// Safe wrapper for `LLVMGetValueName2` into a byte slice +pub fn get_value_name(value: &Value) -> &[u8] { + unsafe { + let mut len = 0; + let data = LLVMGetValueName2(value, &mut len); + std::slice::from_raw_parts(data.cast(), len) + } +} + +/// Safe wrapper for `LLVMSetValueName2` from a byte slice +pub fn set_value_name(value: &Value, name: &[u8]) { + unsafe { + let data = name.as_ptr().cast(); + LLVMSetValueName2(value, data, name.len()); + } +} + +pub fn build_string(f: impl FnOnce(&RustString)) -> Result { + let sr = RustString { bytes: RefCell::new(Vec::new()) }; + f(&sr); + String::from_utf8(sr.bytes.into_inner()) +} + +pub fn build_byte_buffer(f: impl FnOnce(&RustString)) -> Vec { + let sr = RustString { bytes: RefCell::new(Vec::new()) }; + f(&sr); + sr.bytes.into_inner() +} + +pub fn twine_to_string(tr: &Twine) -> String { + unsafe { + build_string(|s| LLVMRustWriteTwineToString(tr, s)).expect("got a non-UTF8 Twine from LLVM") + } +} + +pub fn last_error() -> Option { + unsafe { + let cstr = LLVMRustGetLastError(); + if cstr.is_null() { + None + } else { + let err = CStr::from_ptr(cstr).to_bytes(); + let err = String::from_utf8_lossy(err).to_string(); + libc::free(cstr as *mut _); + Some(err) + } + } +} + +pub struct OperandBundleDef<'a> { + pub raw: &'a mut ffi::OperandBundleDef<'a>, +} + +impl<'a> OperandBundleDef<'a> { + pub fn new(name: &str, vals: &[&'a Value]) -> Self { + let name = SmallCStr::new(name); + let def = unsafe { + LLVMRustBuildOperandBundleDef(name.as_ptr(), vals.as_ptr(), vals.len() as c_uint) + }; + OperandBundleDef { raw: def } + } +} + +impl Drop for OperandBundleDef<'_> { + fn drop(&mut self) { + unsafe { + LLVMRustFreeOperandBundleDef(&mut *(self.raw as *mut _)); + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/llvm_util.rs b/compiler/rustc_codegen_llvm/src/llvm_util.rs new file mode 100644 index 000000000..a0a640473 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/llvm_util.rs @@ -0,0 +1,562 @@ +use crate::back::write::create_informational_target_machine; +use crate::{llvm, llvm_util}; +use libc::c_int; +use libloading::Library; +use rustc_codegen_ssa::target_features::{ + supported_target_features, tied_target_features, RUSTC_SPECIFIC_FEATURES, +}; +use rustc_data_structures::fx::{FxHashMap, FxHashSet}; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_fs_util::path_to_c_string; +use rustc_middle::bug; +use rustc_session::config::PrintRequest; +use rustc_session::Session; +use rustc_span::symbol::Symbol; +use rustc_target::spec::{MergeFunctions, PanicStrategy}; +use smallvec::{smallvec, SmallVec}; +use std::ffi::{CStr, CString}; +use tracing::debug; + +use std::mem; +use std::path::Path; +use std::ptr; +use std::slice; +use std::str; +use std::sync::Once; + +static INIT: Once = Once::new(); + +pub(crate) fn init(sess: &Session) { + unsafe { + // Before we touch LLVM, make sure that multithreading is enabled. + if llvm::LLVMIsMultithreaded() != 1 { + bug!("LLVM compiled without support for threads"); + } + INIT.call_once(|| { + configure_llvm(sess); + }); + } +} + +fn require_inited() { + if !INIT.is_completed() { + bug!("LLVM is not initialized"); + } +} + +unsafe fn configure_llvm(sess: &Session) { + let n_args = sess.opts.cg.llvm_args.len() + sess.target.llvm_args.len(); + let mut llvm_c_strs = Vec::with_capacity(n_args + 1); + let mut llvm_args = Vec::with_capacity(n_args + 1); + + llvm::LLVMRustInstallFatalErrorHandler(); + // On Windows, an LLVM assertion will open an Abort/Retry/Ignore dialog + // box for the purpose of launching a debugger. However, on CI this will + // cause it to hang until it times out, which can take several hours. + if std::env::var_os("CI").is_some() { + llvm::LLVMRustDisableSystemDialogsOnCrash(); + } + + fn llvm_arg_to_arg_name(full_arg: &str) -> &str { + full_arg.trim().split(|c: char| c == '=' || c.is_whitespace()).next().unwrap_or("") + } + + let cg_opts = sess.opts.cg.llvm_args.iter().map(AsRef::as_ref); + let tg_opts = sess.target.llvm_args.iter().map(AsRef::as_ref); + let sess_args = cg_opts.chain(tg_opts); + + let user_specified_args: FxHashSet<_> = + sess_args.clone().map(|s| llvm_arg_to_arg_name(s)).filter(|s| !s.is_empty()).collect(); + + { + // This adds the given argument to LLVM. Unless `force` is true + // user specified arguments are *not* overridden. + let mut add = |arg: &str, force: bool| { + if force || !user_specified_args.contains(llvm_arg_to_arg_name(arg)) { + let s = CString::new(arg).unwrap(); + llvm_args.push(s.as_ptr()); + llvm_c_strs.push(s); + } + }; + // Set the llvm "program name" to make usage and invalid argument messages more clear. + add("rustc -Cllvm-args=\"...\" with", true); + if sess.time_llvm_passes() { + add("-time-passes", false); + } + if sess.print_llvm_passes() { + add("-debug-pass=Structure", false); + } + if sess.target.generate_arange_section + && !sess.opts.unstable_opts.no_generate_arange_section + { + add("-generate-arange-section", false); + } + + // Disable the machine outliner by default in LLVM versions 11 and LLVM + // version 12, where it leads to miscompilation. + // + // Ref: + // - https://github.com/rust-lang/rust/issues/85351 + // - https://reviews.llvm.org/D103167 + if llvm_util::get_version() < (13, 0, 0) { + add("-enable-machine-outliner=never", false); + } + + match sess.opts.unstable_opts.merge_functions.unwrap_or(sess.target.merge_functions) { + MergeFunctions::Disabled | MergeFunctions::Trampolines => {} + MergeFunctions::Aliases => { + add("-mergefunc-use-aliases", false); + } + } + + if sess.target.os == "emscripten" && sess.panic_strategy() == PanicStrategy::Unwind { + add("-enable-emscripten-cxx-exceptions", false); + } + + // HACK(eddyb) LLVM inserts `llvm.assume` calls to preserve align attributes + // during inlining. Unfortunately these may block other optimizations. + add("-preserve-alignment-assumptions-during-inlining=false", false); + + // Use non-zero `import-instr-limit` multiplier for cold callsites. + add("-import-cold-multiplier=0.1", false); + + for arg in sess_args { + add(&(*arg), true); + } + } + + if sess.opts.unstable_opts.llvm_time_trace { + llvm::LLVMTimeTraceProfilerInitialize(); + } + + llvm::LLVMInitializePasses(); + + // Use the legacy plugin registration if we don't use the new pass manager + if !should_use_new_llvm_pass_manager( + &sess.opts.unstable_opts.new_llvm_pass_manager, + &sess.target.arch, + ) { + // Register LLVM plugins by loading them into the compiler process. + for plugin in &sess.opts.unstable_opts.llvm_plugins { + let lib = Library::new(plugin).unwrap_or_else(|e| bug!("couldn't load plugin: {}", e)); + debug!("LLVM plugin loaded successfully {:?} ({})", lib, plugin); + + // Intentionally leak the dynamic library. We can't ever unload it + // since the library can make things that will live arbitrarily long. + mem::forget(lib); + } + } + + rustc_llvm::initialize_available_targets(); + + llvm::LLVMRustSetLLVMOptions(llvm_args.len() as c_int, llvm_args.as_ptr()); +} + +pub fn time_trace_profiler_finish(file_name: &Path) { + unsafe { + let file_name = path_to_c_string(file_name); + llvm::LLVMTimeTraceProfilerFinish(file_name.as_ptr()); + } +} + +// WARNING: the features after applying `to_llvm_features` must be known +// to LLVM or the feature detection code will walk past the end of the feature +// array, leading to crashes. +// +// To find a list of LLVM's names, check llvm-project/llvm/include/llvm/Support/*TargetParser.def +// where the * matches the architecture's name +// Beware to not use the llvm github project for this, but check the git submodule +// found in src/llvm-project +// Though note that Rust can also be build with an external precompiled version of LLVM +// which might lead to failures if the oldest tested / supported LLVM version +// doesn't yet support the relevant intrinsics +pub fn to_llvm_features<'a>(sess: &Session, s: &'a str) -> SmallVec<[&'a str; 2]> { + let arch = if sess.target.arch == "x86_64" { "x86" } else { &*sess.target.arch }; + match (arch, s) { + ("x86", "sse4.2") => { + if get_version() >= (14, 0, 0) { + smallvec!["sse4.2", "crc32"] + } else { + smallvec!["sse4.2"] + } + } + ("x86", "pclmulqdq") => smallvec!["pclmul"], + ("x86", "rdrand") => smallvec!["rdrnd"], + ("x86", "bmi1") => smallvec!["bmi"], + ("x86", "cmpxchg16b") => smallvec!["cx16"], + ("x86", "avx512vaes") => smallvec!["vaes"], + ("x86", "avx512gfni") => smallvec!["gfni"], + ("x86", "avx512vpclmulqdq") => smallvec!["vpclmulqdq"], + ("aarch64", "rcpc2") => smallvec!["rcpc-immo"], + ("aarch64", "dpb") => smallvec!["ccpp"], + ("aarch64", "dpb2") => smallvec!["ccdp"], + ("aarch64", "frintts") => smallvec!["fptoint"], + ("aarch64", "fcma") => smallvec!["complxnum"], + ("aarch64", "pmuv3") => smallvec!["perfmon"], + ("aarch64", "paca") => smallvec!["pauth"], + ("aarch64", "pacg") => smallvec!["pauth"], + // Rust ties fp and neon together. In LLVM neon implicitly enables fp, + // but we manually enable neon when a feature only implicitly enables fp + ("aarch64", "f32mm") => smallvec!["f32mm", "neon"], + ("aarch64", "f64mm") => smallvec!["f64mm", "neon"], + ("aarch64", "fhm") => smallvec!["fp16fml", "neon"], + ("aarch64", "fp16") => smallvec!["fullfp16", "neon"], + ("aarch64", "jsconv") => smallvec!["jsconv", "neon"], + ("aarch64", "sve") => smallvec!["sve", "neon"], + ("aarch64", "sve2") => smallvec!["sve2", "neon"], + ("aarch64", "sve2-aes") => smallvec!["sve2-aes", "neon"], + ("aarch64", "sve2-sm4") => smallvec!["sve2-sm4", "neon"], + ("aarch64", "sve2-sha3") => smallvec!["sve2-sha3", "neon"], + ("aarch64", "sve2-bitperm") => smallvec!["sve2-bitperm", "neon"], + (_, s) => smallvec![s], + } +} + +// Given a map from target_features to whether they are enabled or disabled, +// ensure only valid combinations are allowed. +pub fn check_tied_features( + sess: &Session, + features: &FxHashMap<&str, bool>, +) -> Option<&'static [&'static str]> { + if !features.is_empty() { + for tied in tied_target_features(sess) { + // Tied features must be set to the same value, or not set at all + let mut tied_iter = tied.iter(); + let enabled = features.get(tied_iter.next().unwrap()); + if tied_iter.any(|f| enabled != features.get(f)) { + return Some(tied); + } + } + } + return None; +} + +// Used to generate cfg variables and apply features +// Must express features in the way Rust understands them +pub fn target_features(sess: &Session, allow_unstable: bool) -> Vec { + let target_machine = create_informational_target_machine(sess); + let mut features: Vec = supported_target_features(sess) + .iter() + .filter_map(|&(feature, gate)| { + if sess.is_nightly_build() || allow_unstable || gate.is_none() { + Some(feature) + } else { + None + } + }) + .filter(|feature| { + // check that all features in a given smallvec are enabled + for llvm_feature in to_llvm_features(sess, feature) { + let cstr = SmallCStr::new(llvm_feature); + if !unsafe { llvm::LLVMRustHasFeature(target_machine, cstr.as_ptr()) } { + return false; + } + } + true + }) + .map(|feature| Symbol::intern(feature)) + .collect(); + + // LLVM 14 changed the ABI for i128 arguments to __float/__fix builtins on Win64 + // (see https://reviews.llvm.org/D110413). This unstable target feature is intended for use + // by compiler-builtins, to export the builtins with the expected, LLVM-version-dependent ABI. + // The target feature can be dropped once we no longer support older LLVM versions. + if sess.is_nightly_build() && get_version() >= (14, 0, 0) { + features.push(Symbol::intern("llvm14-builtins-abi")); + } + features +} + +pub fn print_version() { + let (major, minor, patch) = get_version(); + println!("LLVM version: {}.{}.{}", major, minor, patch); +} + +pub fn get_version() -> (u32, u32, u32) { + // Can be called without initializing LLVM + unsafe { + (llvm::LLVMRustVersionMajor(), llvm::LLVMRustVersionMinor(), llvm::LLVMRustVersionPatch()) + } +} + +pub fn print_passes() { + // Can be called without initializing LLVM + unsafe { + llvm::LLVMRustPrintPasses(); + } +} + +fn llvm_target_features(tm: &llvm::TargetMachine) -> Vec<(&str, &str)> { + let len = unsafe { llvm::LLVMRustGetTargetFeaturesCount(tm) }; + let mut ret = Vec::with_capacity(len); + for i in 0..len { + unsafe { + let mut feature = ptr::null(); + let mut desc = ptr::null(); + llvm::LLVMRustGetTargetFeature(tm, i, &mut feature, &mut desc); + if feature.is_null() || desc.is_null() { + bug!("LLVM returned a `null` target feature string"); + } + let feature = CStr::from_ptr(feature).to_str().unwrap_or_else(|e| { + bug!("LLVM returned a non-utf8 feature string: {}", e); + }); + let desc = CStr::from_ptr(desc).to_str().unwrap_or_else(|e| { + bug!("LLVM returned a non-utf8 feature string: {}", e); + }); + ret.push((feature, desc)); + } + } + ret +} + +fn print_target_features(sess: &Session, tm: &llvm::TargetMachine) { + let mut target_features = llvm_target_features(tm); + let mut rustc_target_features = supported_target_features(sess) + .iter() + .filter_map(|(feature, _gate)| { + for llvm_feature in to_llvm_features(sess, *feature) { + // LLVM asserts that these are sorted. LLVM and Rust both use byte comparison for these strings. + match target_features.binary_search_by_key(&llvm_feature, |(f, _d)| f).ok().map( + |index| { + let (_f, desc) = target_features.remove(index); + (*feature, desc) + }, + ) { + Some(v) => return Some(v), + None => {} + } + } + None + }) + .collect::>(); + rustc_target_features.extend_from_slice(&[( + "crt-static", + "Enables C Run-time Libraries to be statically linked", + )]); + let max_feature_len = target_features + .iter() + .chain(rustc_target_features.iter()) + .map(|(feature, _desc)| feature.len()) + .max() + .unwrap_or(0); + + println!("Features supported by rustc for this target:"); + for (feature, desc) in &rustc_target_features { + println!(" {1:0$} - {2}.", max_feature_len, feature, desc); + } + println!("\nCode-generation features supported by LLVM for this target:"); + for (feature, desc) in &target_features { + println!(" {1:0$} - {2}.", max_feature_len, feature, desc); + } + if target_features.is_empty() { + println!(" Target features listing is not supported by this LLVM version."); + } + println!("\nUse +feature to enable a feature, or -feature to disable it."); + println!("For example, rustc -C target-cpu=mycpu -C target-feature=+feature1,-feature2\n"); + println!("Code-generation features cannot be used in cfg or #[target_feature],"); + println!("and may be renamed or removed in a future version of LLVM or rustc.\n"); +} + +pub(crate) fn print(req: PrintRequest, sess: &Session) { + require_inited(); + let tm = create_informational_target_machine(sess); + match req { + PrintRequest::TargetCPUs => unsafe { llvm::LLVMRustPrintTargetCPUs(tm) }, + PrintRequest::TargetFeatures => print_target_features(sess, tm), + _ => bug!("rustc_codegen_llvm can't handle print request: {:?}", req), + } +} + +fn handle_native(name: &str) -> &str { + if name != "native" { + return name; + } + + unsafe { + let mut len = 0; + let ptr = llvm::LLVMRustGetHostCPUName(&mut len); + str::from_utf8(slice::from_raw_parts(ptr as *const u8, len)).unwrap() + } +} + +pub fn target_cpu(sess: &Session) -> &str { + match sess.opts.cg.target_cpu { + Some(ref name) => handle_native(name), + None => handle_native(sess.target.cpu.as_ref()), + } +} + +/// The list of LLVM features computed from CLI flags (`-Ctarget-cpu`, `-Ctarget-feature`, +/// `--target` and similar). +pub(crate) fn global_llvm_features(sess: &Session, diagnostics: bool) -> Vec { + // Features that come earlier are overridden by conflicting features later in the string. + // Typically we'll want more explicit settings to override the implicit ones, so: + // + // * Features from -Ctarget-cpu=*; are overridden by [^1] + // * Features implied by --target; are overridden by + // * Features from -Ctarget-feature; are overridden by + // * function specific features. + // + // [^1]: target-cpu=native is handled here, other target-cpu values are handled implicitly + // through LLVM TargetMachine implementation. + // + // FIXME(nagisa): it isn't clear what's the best interaction between features implied by + // `-Ctarget-cpu` and `--target` are. On one hand, you'd expect CLI arguments to always + // override anything that's implicit, so e.g. when there's no `--target` flag, features implied + // the host target are overridden by `-Ctarget-cpu=*`. On the other hand, what about when both + // `--target` and `-Ctarget-cpu=*` are specified? Both then imply some target features and both + // flags are specified by the user on the CLI. It isn't as clear-cut which order of precedence + // should be taken in cases like these. + let mut features = vec![]; + + // -Ctarget-cpu=native + match sess.opts.cg.target_cpu { + Some(ref s) if s == "native" => { + let features_string = unsafe { + let ptr = llvm::LLVMGetHostCPUFeatures(); + let features_string = if !ptr.is_null() { + CStr::from_ptr(ptr) + .to_str() + .unwrap_or_else(|e| { + bug!("LLVM returned a non-utf8 features string: {}", e); + }) + .to_owned() + } else { + bug!("could not allocate host CPU features, LLVM returned a `null` string"); + }; + + llvm::LLVMDisposeMessage(ptr); + + features_string + }; + features.extend(features_string.split(',').map(String::from)); + } + Some(_) | None => {} + }; + + // Features implied by an implicit or explicit `--target`. + features.extend( + sess.target + .features + .split(',') + .filter(|v| !v.is_empty() && backend_feature_name(v).is_some()) + .map(String::from), + ); + + // -Ctarget-features + let supported_features = supported_target_features(sess); + let mut featsmap = FxHashMap::default(); + let feats = sess + .opts + .cg + .target_feature + .split(',') + .filter_map(|s| { + let enable_disable = match s.chars().next() { + None => return None, + Some(c @ '+' | c @ '-') => c, + Some(_) => { + if diagnostics { + let mut diag = sess.struct_warn(&format!( + "unknown feature specified for `-Ctarget-feature`: `{}`", + s + )); + diag.note("features must begin with a `+` to enable or `-` to disable it"); + diag.emit(); + } + return None; + } + }; + + let feature = backend_feature_name(s)?; + // Warn against use of LLVM specific feature names on the CLI. + if diagnostics && !supported_features.iter().any(|&(v, _)| v == feature) { + let rust_feature = supported_features.iter().find_map(|&(rust_feature, _)| { + let llvm_features = to_llvm_features(sess, rust_feature); + if llvm_features.contains(&feature) && !llvm_features.contains(&rust_feature) { + Some(rust_feature) + } else { + None + } + }); + let mut diag = sess.struct_warn(&format!( + "unknown feature specified for `-Ctarget-feature`: `{}`", + feature + )); + diag.note("it is still passed through to the codegen backend"); + if let Some(rust_feature) = rust_feature { + diag.help(&format!("you might have meant: `{}`", rust_feature)); + } else { + diag.note("consider filing a feature request"); + } + diag.emit(); + } + + if diagnostics { + // FIXME(nagisa): figure out how to not allocate a full hashset here. + featsmap.insert(feature, enable_disable == '+'); + } + + // rustc-specific features do not get passed down to LLVM… + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + // ... otherwise though we run through `to_llvm_features` when + // passing requests down to LLVM. This means that all in-language + // features also work on the command line instead of having two + // different names when the LLVM name and the Rust name differ. + Some( + to_llvm_features(sess, feature) + .into_iter() + .map(move |f| format!("{}{}", enable_disable, f)), + ) + }) + .flatten(); + features.extend(feats); + + if diagnostics && let Some(f) = check_tied_features(sess, &featsmap) { + sess.err(&format!( + "target features {} must all be enabled or disabled together", + f.join(", ") + )); + } + + features +} + +/// Returns a feature name for the given `+feature` or `-feature` string. +/// +/// Only allows features that are backend specific (i.e. not [`RUSTC_SPECIFIC_FEATURES`].) +fn backend_feature_name(s: &str) -> Option<&str> { + // features must start with a `+` or `-`. + let feature = s.strip_prefix(&['+', '-'][..]).unwrap_or_else(|| { + bug!("target feature `{}` must begin with a `+` or `-`", s); + }); + // Rustc-specific feature requests like `+crt-static` or `-crt-static` + // are not passed down to LLVM. + if RUSTC_SPECIFIC_FEATURES.contains(&feature) { + return None; + } + Some(feature) +} + +pub fn tune_cpu(sess: &Session) -> Option<&str> { + let name = sess.opts.unstable_opts.tune_cpu.as_ref()?; + Some(handle_native(name)) +} + +pub(crate) fn should_use_new_llvm_pass_manager(user_opt: &Option, target_arch: &str) -> bool { + // The new pass manager is enabled by default for LLVM >= 13. + // This matches Clang, which also enables it since Clang 13. + + // Since LLVM 15, the legacy pass manager is no longer supported. + if llvm_util::get_version() >= (15, 0, 0) { + return true; + } + + // There are some perf issues with the new pass manager when targeting + // s390x with LLVM 13, so enable the new pass manager only with LLVM 14. + // See https://github.com/rust-lang/rust/issues/89609. + let min_version = if target_arch == "s390x" { 14 } else { 13 }; + user_opt.unwrap_or_else(|| llvm_util::get_version() >= (min_version, 0, 0)) +} diff --git a/compiler/rustc_codegen_llvm/src/mono_item.rs b/compiler/rustc_codegen_llvm/src/mono_item.rs new file mode 100644 index 000000000..6e9428485 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/mono_item.rs @@ -0,0 +1,150 @@ +use crate::attributes; +use crate::base; +use crate::context::CodegenCx; +use crate::llvm; +use crate::type_of::LayoutLlvmExt; +use rustc_codegen_ssa::traits::*; +use rustc_hir::def_id::{DefId, LOCAL_CRATE}; +pub use rustc_middle::mir::mono::MonoItem; +use rustc_middle::mir::mono::{Linkage, Visibility}; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf}; +use rustc_middle::ty::{self, Instance, TypeVisitable}; +use rustc_session::config::CrateType; +use rustc_target::spec::RelocModel; +use tracing::debug; + +impl<'tcx> PreDefineMethods<'tcx> for CodegenCx<'_, 'tcx> { + fn predefine_static( + &self, + def_id: DefId, + linkage: Linkage, + visibility: Visibility, + symbol_name: &str, + ) { + let instance = Instance::mono(self.tcx, def_id); + let ty = instance.ty(self.tcx, ty::ParamEnv::reveal_all()); + let llty = self.layout_of(ty).llvm_type(self); + + let g = self.define_global(symbol_name, llty).unwrap_or_else(|| { + self.sess().span_fatal( + self.tcx.def_span(def_id), + &format!("symbol `{}` is already defined", symbol_name), + ) + }); + + unsafe { + llvm::LLVMRustSetLinkage(g, base::linkage_to_llvm(linkage)); + llvm::LLVMRustSetVisibility(g, base::visibility_to_llvm(visibility)); + if self.should_assume_dso_local(g, false) { + llvm::LLVMRustSetDSOLocal(g, true); + } + } + + self.instances.borrow_mut().insert(instance, g); + } + + fn predefine_fn( + &self, + instance: Instance<'tcx>, + linkage: Linkage, + visibility: Visibility, + symbol_name: &str, + ) { + assert!(!instance.substs.needs_infer()); + + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty()); + let lldecl = self.declare_fn(symbol_name, fn_abi); + unsafe { llvm::LLVMRustSetLinkage(lldecl, base::linkage_to_llvm(linkage)) }; + let attrs = self.tcx.codegen_fn_attrs(instance.def_id()); + base::set_link_section(lldecl, attrs); + if linkage == Linkage::LinkOnceODR || linkage == Linkage::WeakODR { + llvm::SetUniqueComdat(self.llmod, lldecl); + } + + // If we're compiling the compiler-builtins crate, e.g., the equivalent of + // compiler-rt, then we want to implicitly compile everything with hidden + // visibility as we're going to link this object all over the place but + // don't want the symbols to get exported. + if linkage != Linkage::Internal + && linkage != Linkage::Private + && self.tcx.is_compiler_builtins(LOCAL_CRATE) + { + unsafe { + llvm::LLVMRustSetVisibility(lldecl, llvm::Visibility::Hidden); + } + } else { + unsafe { + llvm::LLVMRustSetVisibility(lldecl, base::visibility_to_llvm(visibility)); + } + } + + debug!("predefine_fn: instance = {:?}", instance); + + attributes::from_fn_attrs(self, lldecl, instance); + + unsafe { + if self.should_assume_dso_local(lldecl, false) { + llvm::LLVMRustSetDSOLocal(lldecl, true); + } + } + + self.instances.borrow_mut().insert(instance, lldecl); + } +} + +impl CodegenCx<'_, '_> { + /// Whether a definition or declaration can be assumed to be local to a group of + /// libraries that form a single DSO or executable. + pub(crate) unsafe fn should_assume_dso_local( + &self, + llval: &llvm::Value, + is_declaration: bool, + ) -> bool { + let linkage = llvm::LLVMRustGetLinkage(llval); + let visibility = llvm::LLVMRustGetVisibility(llval); + + if matches!(linkage, llvm::Linkage::InternalLinkage | llvm::Linkage::PrivateLinkage) { + return true; + } + + if visibility != llvm::Visibility::Default && linkage != llvm::Linkage::ExternalWeakLinkage + { + return true; + } + + // Symbols from executables can't really be imported any further. + let all_exe = self.tcx.sess.crate_types().iter().all(|ty| *ty == CrateType::Executable); + let is_declaration_for_linker = + is_declaration || linkage == llvm::Linkage::AvailableExternallyLinkage; + if all_exe && !is_declaration_for_linker { + return true; + } + + // PowerPC64 prefers TOC indirection to avoid copy relocations. + if matches!(&*self.tcx.sess.target.arch, "powerpc64" | "powerpc64le") { + return false; + } + + // Thread-local variables generally don't support copy relocations. + let is_thread_local_var = llvm::LLVMIsAGlobalVariable(llval) + .map(|v| llvm::LLVMIsThreadLocal(v) == llvm::True) + .unwrap_or(false); + if is_thread_local_var { + return false; + } + + // Match clang by only supporting COFF and ELF for now. + if self.tcx.sess.target.is_like_osx { + return false; + } + + // Static relocation model should force copy relocations everywhere. + if self.tcx.sess.relocation_model() == RelocModel::Static { + return true; + } + + // With pie relocation model calls of functions defined in the translation + // unit can use copy relocations. + self.tcx.sess.relocation_model() == RelocModel::Pie && !is_declaration + } +} diff --git a/compiler/rustc_codegen_llvm/src/type_.rs b/compiler/rustc_codegen_llvm/src/type_.rs new file mode 100644 index 000000000..eeb38d4ec --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/type_.rs @@ -0,0 +1,319 @@ +pub use crate::llvm::Type; + +use crate::abi::{FnAbiLlvmExt, LlvmType}; +use crate::common; +use crate::context::CodegenCx; +use crate::llvm; +use crate::llvm::{Bool, False, True}; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; +use rustc_codegen_ssa::common::TypeKind; +use rustc_codegen_ssa::traits::*; +use rustc_data_structures::small_c_str::SmallCStr; +use rustc_middle::bug; +use rustc_middle::ty::layout::TyAndLayout; +use rustc_middle::ty::{self, Ty}; +use rustc_target::abi::call::{CastTarget, FnAbi, Reg}; +use rustc_target::abi::{AddressSpace, Align, Integer, Size}; + +use std::fmt; +use std::ptr; + +use libc::{c_char, c_uint}; + +impl PartialEq for Type { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self, other) + } +} + +impl fmt::Debug for Type { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str( + &llvm::build_string(|s| unsafe { + llvm::LLVMRustWriteTypeToString(self, s); + }) + .expect("non-UTF8 type description from LLVM"), + ) + } +} + +impl<'ll> CodegenCx<'ll, '_> { + pub(crate) fn type_named_struct(&self, name: &str) -> &'ll Type { + let name = SmallCStr::new(name); + unsafe { llvm::LLVMStructCreateNamed(self.llcx, name.as_ptr()) } + } + + pub(crate) fn set_struct_body(&self, ty: &'ll Type, els: &[&'ll Type], packed: bool) { + unsafe { llvm::LLVMStructSetBody(ty, els.as_ptr(), els.len() as c_uint, packed as Bool) } + } + + pub(crate) fn type_void(&self) -> &'ll Type { + unsafe { llvm::LLVMVoidTypeInContext(self.llcx) } + } + + pub(crate) fn type_metadata(&self) -> &'ll Type { + unsafe { llvm::LLVMRustMetadataTypeInContext(self.llcx) } + } + + ///x Creates an integer type with the given number of bits, e.g., i24 + pub(crate) fn type_ix(&self, num_bits: u64) -> &'ll Type { + unsafe { llvm::LLVMIntTypeInContext(self.llcx, num_bits as c_uint) } + } + + pub(crate) fn type_vector(&self, ty: &'ll Type, len: u64) -> &'ll Type { + unsafe { llvm::LLVMVectorType(ty, len as c_uint) } + } + + pub(crate) fn func_params_types(&self, ty: &'ll Type) -> Vec<&'ll Type> { + unsafe { + let n_args = llvm::LLVMCountParamTypes(ty) as usize; + let mut args = Vec::with_capacity(n_args); + llvm::LLVMGetParamTypes(ty, args.as_mut_ptr()); + args.set_len(n_args); + args + } + } + + pub(crate) fn type_bool(&self) -> &'ll Type { + self.type_i8() + } + + pub(crate) fn type_int_from_ty(&self, t: ty::IntTy) -> &'ll Type { + match t { + ty::IntTy::Isize => self.type_isize(), + ty::IntTy::I8 => self.type_i8(), + ty::IntTy::I16 => self.type_i16(), + ty::IntTy::I32 => self.type_i32(), + ty::IntTy::I64 => self.type_i64(), + ty::IntTy::I128 => self.type_i128(), + } + } + + pub(crate) fn type_uint_from_ty(&self, t: ty::UintTy) -> &'ll Type { + match t { + ty::UintTy::Usize => self.type_isize(), + ty::UintTy::U8 => self.type_i8(), + ty::UintTy::U16 => self.type_i16(), + ty::UintTy::U32 => self.type_i32(), + ty::UintTy::U64 => self.type_i64(), + ty::UintTy::U128 => self.type_i128(), + } + } + + pub(crate) fn type_float_from_ty(&self, t: ty::FloatTy) -> &'ll Type { + match t { + ty::FloatTy::F32 => self.type_f32(), + ty::FloatTy::F64 => self.type_f64(), + } + } + + pub(crate) fn type_pointee_for_align(&self, align: Align) -> &'ll Type { + // FIXME(eddyb) We could find a better approximation if ity.align < align. + let ity = Integer::approximate_align(self, align); + self.type_from_integer(ity) + } + + /// Return a LLVM type that has at most the required alignment, + /// and exactly the required size, as a best-effort padding array. + pub(crate) fn type_padding_filler(&self, size: Size, align: Align) -> &'ll Type { + let unit = Integer::approximate_align(self, align); + let size = size.bytes(); + let unit_size = unit.size().bytes(); + assert_eq!(size % unit_size, 0); + self.type_array(self.type_from_integer(unit), size / unit_size) + } + + pub(crate) fn type_variadic_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, True) } + } + + pub(crate) fn type_array(&self, ty: &'ll Type, len: u64) -> &'ll Type { + unsafe { llvm::LLVMRustArrayType(ty, len) } + } +} + +impl<'ll, 'tcx> BaseTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn type_i1(&self) -> &'ll Type { + unsafe { llvm::LLVMInt1TypeInContext(self.llcx) } + } + + fn type_i8(&self) -> &'ll Type { + unsafe { llvm::LLVMInt8TypeInContext(self.llcx) } + } + + fn type_i16(&self) -> &'ll Type { + unsafe { llvm::LLVMInt16TypeInContext(self.llcx) } + } + + fn type_i32(&self) -> &'ll Type { + unsafe { llvm::LLVMInt32TypeInContext(self.llcx) } + } + + fn type_i64(&self) -> &'ll Type { + unsafe { llvm::LLVMInt64TypeInContext(self.llcx) } + } + + fn type_i128(&self) -> &'ll Type { + unsafe { llvm::LLVMIntTypeInContext(self.llcx, 128) } + } + + fn type_isize(&self) -> &'ll Type { + self.isize_ty + } + + fn type_f32(&self) -> &'ll Type { + unsafe { llvm::LLVMFloatTypeInContext(self.llcx) } + } + + fn type_f64(&self) -> &'ll Type { + unsafe { llvm::LLVMDoubleTypeInContext(self.llcx) } + } + + fn type_func(&self, args: &[&'ll Type], ret: &'ll Type) -> &'ll Type { + unsafe { llvm::LLVMFunctionType(ret, args.as_ptr(), args.len() as c_uint, False) } + } + + fn type_struct(&self, els: &[&'ll Type], packed: bool) -> &'ll Type { + unsafe { + llvm::LLVMStructTypeInContext( + self.llcx, + els.as_ptr(), + els.len() as c_uint, + packed as Bool, + ) + } + } + + fn type_kind(&self, ty: &'ll Type) -> TypeKind { + unsafe { llvm::LLVMRustGetTypeKind(ty).to_generic() } + } + + fn type_ptr_to(&self, ty: &'ll Type) -> &'ll Type { + assert_ne!( + self.type_kind(ty), + TypeKind::Function, + "don't call ptr_to on function types, use ptr_to_llvm_type on FnAbi instead or explicitly specify an address space if it makes sense" + ); + ty.ptr_to(AddressSpace::DATA) + } + + fn type_ptr_to_ext(&self, ty: &'ll Type, address_space: AddressSpace) -> &'ll Type { + ty.ptr_to(address_space) + } + + fn element_type(&self, ty: &'ll Type) -> &'ll Type { + match self.type_kind(ty) { + TypeKind::Array | TypeKind::Vector => unsafe { llvm::LLVMGetElementType(ty) }, + TypeKind::Pointer => bug!("element_type is not supported for opaque pointers"), + other => bug!("element_type called on unsupported type {:?}", other), + } + } + + fn vector_length(&self, ty: &'ll Type) -> usize { + unsafe { llvm::LLVMGetVectorSize(ty) as usize } + } + + fn float_width(&self, ty: &'ll Type) -> usize { + match self.type_kind(ty) { + TypeKind::Float => 32, + TypeKind::Double => 64, + TypeKind::X86_FP80 => 80, + TypeKind::FP128 | TypeKind::PPC_FP128 => 128, + _ => bug!("llvm_float_width called on a non-float type"), + } + } + + fn int_width(&self, ty: &'ll Type) -> u64 { + unsafe { llvm::LLVMGetIntTypeWidth(ty) as u64 } + } + + fn val_ty(&self, v: &'ll Value) -> &'ll Type { + common::val_ty(v) + } +} + +impl Type { + pub fn i8_llcx(llcx: &llvm::Context) -> &Type { + unsafe { llvm::LLVMInt8TypeInContext(llcx) } + } + + // Creates an integer type with the given number of bits, e.g., i24 + pub fn ix_llcx(llcx: &llvm::Context, num_bits: u64) -> &Type { + unsafe { llvm::LLVMIntTypeInContext(llcx, num_bits as c_uint) } + } + + pub fn i8p_llcx(llcx: &llvm::Context) -> &Type { + Type::i8_llcx(llcx).ptr_to(AddressSpace::DATA) + } + + fn ptr_to(&self, address_space: AddressSpace) -> &Type { + unsafe { llvm::LLVMPointerType(self, address_space.0) } + } +} + +impl<'ll, 'tcx> LayoutTypeMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { + layout.llvm_type(self) + } + fn immediate_backend_type(&self, layout: TyAndLayout<'tcx>) -> &'ll Type { + layout.immediate_llvm_type(self) + } + fn is_backend_immediate(&self, layout: TyAndLayout<'tcx>) -> bool { + layout.is_llvm_immediate() + } + fn is_backend_scalar_pair(&self, layout: TyAndLayout<'tcx>) -> bool { + layout.is_llvm_scalar_pair() + } + fn backend_field_index(&self, layout: TyAndLayout<'tcx>, index: usize) -> u64 { + layout.llvm_field_index(self, index) + } + fn scalar_pair_element_backend_type( + &self, + layout: TyAndLayout<'tcx>, + index: usize, + immediate: bool, + ) -> &'ll Type { + layout.scalar_pair_element_llvm_type(self, index, immediate) + } + fn cast_backend_type(&self, ty: &CastTarget) -> &'ll Type { + ty.llvm_type(self) + } + fn fn_decl_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + fn_abi.llvm_type(self) + } + fn fn_ptr_backend_type(&self, fn_abi: &FnAbi<'tcx, Ty<'tcx>>) -> &'ll Type { + fn_abi.ptr_to_llvm_type(self) + } + fn reg_backend_type(&self, ty: &Reg) -> &'ll Type { + ty.llvm_type(self) + } +} + +impl<'ll, 'tcx> TypeMembershipMethods<'tcx> for CodegenCx<'ll, 'tcx> { + fn set_type_metadata(&self, function: &'ll Value, typeid: String) { + let typeid_metadata = self.typeid_metadata(typeid); + let v = [self.const_usize(0), typeid_metadata]; + unsafe { + llvm::LLVMGlobalSetMetadata( + function, + llvm::MD_type as c_uint, + llvm::LLVMValueAsMetadata(llvm::LLVMMDNodeInContext( + self.llcx, + v.as_ptr(), + v.len() as c_uint, + )), + ) + } + } + + fn typeid_metadata(&self, typeid: String) -> &'ll Value { + unsafe { + llvm::LLVMMDStringInContext( + self.llcx, + typeid.as_ptr() as *const c_char, + typeid.len() as c_uint, + ) + } + } +} diff --git a/compiler/rustc_codegen_llvm/src/type_of.rs b/compiler/rustc_codegen_llvm/src/type_of.rs new file mode 100644 index 000000000..9f0e6c80b --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/type_of.rs @@ -0,0 +1,418 @@ +use crate::common::*; +use crate::context::TypeLowering; +use crate::llvm_util::get_version; +use crate::type_::Type; +use rustc_codegen_ssa::traits::*; +use rustc_middle::bug; +use rustc_middle::ty::layout::{FnAbiOf, LayoutOf, TyAndLayout}; +use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths}; +use rustc_middle::ty::{self, Ty, TypeVisitable}; +use rustc_target::abi::{Abi, AddressSpace, Align, FieldsShape}; +use rustc_target::abi::{Int, Pointer, F32, F64}; +use rustc_target::abi::{PointeeInfo, Scalar, Size, TyAbiInterface, Variants}; +use smallvec::{smallvec, SmallVec}; +use tracing::debug; + +use std::fmt::Write; + +fn uncached_llvm_type<'a, 'tcx>( + cx: &CodegenCx<'a, 'tcx>, + layout: TyAndLayout<'tcx>, + defer: &mut Option<(&'a Type, TyAndLayout<'tcx>)>, + field_remapping: &mut Option>, +) -> &'a Type { + match layout.abi { + Abi::Scalar(_) => bug!("handled elsewhere"), + Abi::Vector { element, count } => { + let element = layout.scalar_llvm_type_at(cx, element, Size::ZERO); + return cx.type_vector(element, count); + } + Abi::ScalarPair(..) => { + return cx.type_struct( + &[ + layout.scalar_pair_element_llvm_type(cx, 0, false), + layout.scalar_pair_element_llvm_type(cx, 1, false), + ], + false, + ); + } + Abi::Uninhabited | Abi::Aggregate { .. } => {} + } + + let name = match layout.ty.kind() { + // FIXME(eddyb) producing readable type names for trait objects can result + // in problematically distinct types due to HRTB and subtyping (see #47638). + // ty::Dynamic(..) | + ty::Adt(..) | ty::Closure(..) | ty::Foreign(..) | ty::Generator(..) | ty::Str + // For performance reasons we use names only when emitting LLVM IR. Unless we are on + // LLVM < 14, where the use of unnamed types resulted in various issues, e.g., #76213, + // #79564, and #79246. + if get_version() < (14, 0, 0) || !cx.sess().fewer_names() => + { + let mut name = with_no_visible_paths!(with_no_trimmed_paths!(layout.ty.to_string())); + if let (&ty::Adt(def, _), &Variants::Single { index }) = + (layout.ty.kind(), &layout.variants) + { + if def.is_enum() && !def.variants().is_empty() { + write!(&mut name, "::{}", def.variant(index).name).unwrap(); + } + } + if let (&ty::Generator(_, _, _), &Variants::Single { index }) = + (layout.ty.kind(), &layout.variants) + { + write!(&mut name, "::{}", ty::GeneratorSubsts::variant_name(index)).unwrap(); + } + Some(name) + } + // Use identified structure types for ADT. Due to pointee types in LLVM IR their definition + // might be recursive. Other cases are non-recursive and we can use literal structure types. + ty::Adt(..) => Some(String::new()), + _ => None, + }; + + match layout.fields { + FieldsShape::Primitive | FieldsShape::Union(_) => { + let fill = cx.type_padding_filler(layout.size, layout.align.abi); + let packed = false; + match name { + None => cx.type_struct(&[fill], packed), + Some(ref name) => { + let llty = cx.type_named_struct(name); + cx.set_struct_body(llty, &[fill], packed); + llty + } + } + } + FieldsShape::Array { count, .. } => cx.type_array(layout.field(cx, 0).llvm_type(cx), count), + FieldsShape::Arbitrary { .. } => match name { + None => { + let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); + *field_remapping = new_field_remapping; + cx.type_struct(&llfields, packed) + } + Some(ref name) => { + let llty = cx.type_named_struct(name); + *defer = Some((llty, layout)); + llty + } + }, + } +} + +fn struct_llfields<'a, 'tcx>( + cx: &CodegenCx<'a, 'tcx>, + layout: TyAndLayout<'tcx>, +) -> (Vec<&'a Type>, bool, Option>) { + debug!("struct_llfields: {:#?}", layout); + let field_count = layout.fields.count(); + + let mut packed = false; + let mut offset = Size::ZERO; + let mut prev_effective_align = layout.align.abi; + let mut result: Vec<_> = Vec::with_capacity(1 + field_count * 2); + let mut field_remapping = smallvec![0; field_count]; + for i in layout.fields.index_by_increasing_offset() { + let target_offset = layout.fields.offset(i as usize); + let field = layout.field(cx, i); + let effective_field_align = + layout.align.abi.min(field.align.abi).restrict_for_offset(target_offset); + packed |= effective_field_align < field.align.abi; + + debug!( + "struct_llfields: {}: {:?} offset: {:?} target_offset: {:?} \ + effective_field_align: {}", + i, + field, + offset, + target_offset, + effective_field_align.bytes() + ); + assert!(target_offset >= offset); + let padding = target_offset - offset; + if padding != Size::ZERO { + let padding_align = prev_effective_align.min(effective_field_align); + assert_eq!(offset.align_to(padding_align) + padding, target_offset); + result.push(cx.type_padding_filler(padding, padding_align)); + debug!(" padding before: {:?}", padding); + } + field_remapping[i] = result.len() as u32; + result.push(field.llvm_type(cx)); + offset = target_offset + field.size; + prev_effective_align = effective_field_align; + } + let padding_used = result.len() > field_count; + if !layout.is_unsized() && field_count > 0 { + if offset > layout.size { + bug!("layout: {:#?} stride: {:?} offset: {:?}", layout, layout.size, offset); + } + let padding = layout.size - offset; + if padding != Size::ZERO { + let padding_align = prev_effective_align; + assert_eq!(offset.align_to(padding_align) + padding, layout.size); + debug!( + "struct_llfields: pad_bytes: {:?} offset: {:?} stride: {:?}", + padding, offset, layout.size + ); + result.push(cx.type_padding_filler(padding, padding_align)); + } + } else { + debug!("struct_llfields: offset: {:?} stride: {:?}", offset, layout.size); + } + let field_remapping = if padding_used { Some(field_remapping) } else { None }; + (result, packed, field_remapping) +} + +impl<'a, 'tcx> CodegenCx<'a, 'tcx> { + pub fn align_of(&self, ty: Ty<'tcx>) -> Align { + self.layout_of(ty).align.abi + } + + pub fn size_of(&self, ty: Ty<'tcx>) -> Size { + self.layout_of(ty).size + } + + pub fn size_and_align_of(&self, ty: Ty<'tcx>) -> (Size, Align) { + let layout = self.layout_of(ty); + (layout.size, layout.align.abi) + } +} + +pub trait LayoutLlvmExt<'tcx> { + fn is_llvm_immediate(&self) -> bool; + fn is_llvm_scalar_pair(&self) -> bool; + fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; + fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type; + fn scalar_llvm_type_at<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + scalar: Scalar, + offset: Size, + ) -> &'a Type; + fn scalar_pair_element_llvm_type<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + index: usize, + immediate: bool, + ) -> &'a Type; + fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64; + fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option; +} + +impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> { + fn is_llvm_immediate(&self) -> bool { + match self.abi { + Abi::Scalar(_) | Abi::Vector { .. } => true, + Abi::ScalarPair(..) => false, + Abi::Uninhabited | Abi::Aggregate { .. } => self.is_zst(), + } + } + + fn is_llvm_scalar_pair(&self) -> bool { + match self.abi { + Abi::ScalarPair(..) => true, + Abi::Uninhabited | Abi::Scalar(_) | Abi::Vector { .. } | Abi::Aggregate { .. } => false, + } + } + + /// Gets the LLVM type corresponding to a Rust type, i.e., `rustc_middle::ty::Ty`. + /// The pointee type of the pointer in `PlaceRef` is always this type. + /// For sized types, it is also the right LLVM type for an `alloca` + /// containing a value of that type, and most immediates (except `bool`). + /// Unsized types, however, are represented by a "minimal unit", e.g. + /// `[T]` becomes `T`, while `str` and `Trait` turn into `i8` - this + /// is useful for indexing slices, as `&[T]`'s data pointer is `T*`. + /// If the type is an unsized struct, the regular layout is generated, + /// with the inner-most trailing unsized field using the "minimal unit" + /// of that field's type - this is useful for taking the address of + /// that field and ensuring the struct has the right alignment. + fn llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { + if let Abi::Scalar(scalar) = self.abi { + // Use a different cache for scalars because pointers to DSTs + // can be either fat or thin (data pointers of fat pointers). + if let Some(&llty) = cx.scalar_lltypes.borrow().get(&self.ty) { + return llty; + } + let llty = match *self.ty.kind() { + ty::Ref(_, ty, _) | ty::RawPtr(ty::TypeAndMut { ty, .. }) => { + cx.type_ptr_to(cx.layout_of(ty).llvm_type(cx)) + } + ty::Adt(def, _) if def.is_box() => { + cx.type_ptr_to(cx.layout_of(self.ty.boxed_ty()).llvm_type(cx)) + } + ty::FnPtr(sig) => { + cx.fn_ptr_backend_type(cx.fn_abi_of_fn_ptr(sig, ty::List::empty())) + } + _ => self.scalar_llvm_type_at(cx, scalar, Size::ZERO), + }; + cx.scalar_lltypes.borrow_mut().insert(self.ty, llty); + return llty; + } + + // Check the cache. + let variant_index = match self.variants { + Variants::Single { index } => Some(index), + _ => None, + }; + if let Some(llty) = cx.type_lowering.borrow().get(&(self.ty, variant_index)) { + return llty.lltype; + } + + debug!("llvm_type({:#?})", self); + + assert!(!self.ty.has_escaping_bound_vars(), "{:?} has escaping bound vars", self.ty); + + // Make sure lifetimes are erased, to avoid generating distinct LLVM + // types for Rust types that only differ in the choice of lifetimes. + let normal_ty = cx.tcx.erase_regions(self.ty); + + let mut defer = None; + let mut field_remapping = None; + let llty = if self.ty != normal_ty { + let mut layout = cx.layout_of(normal_ty); + if let Some(v) = variant_index { + layout = layout.for_variant(cx, v); + } + layout.llvm_type(cx) + } else { + uncached_llvm_type(cx, *self, &mut defer, &mut field_remapping) + }; + debug!("--> mapped {:#?} to llty={:?}", self, llty); + + cx.type_lowering + .borrow_mut() + .insert((self.ty, variant_index), TypeLowering { lltype: llty, field_remapping }); + + if let Some((llty, layout)) = defer { + let (llfields, packed, new_field_remapping) = struct_llfields(cx, layout); + cx.set_struct_body(llty, &llfields, packed); + cx.type_lowering + .borrow_mut() + .get_mut(&(self.ty, variant_index)) + .unwrap() + .field_remapping = new_field_remapping; + } + llty + } + + fn immediate_llvm_type<'a>(&self, cx: &CodegenCx<'a, 'tcx>) -> &'a Type { + if let Abi::Scalar(scalar) = self.abi { + if scalar.is_bool() { + return cx.type_i1(); + } + } + self.llvm_type(cx) + } + + fn scalar_llvm_type_at<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + scalar: Scalar, + offset: Size, + ) -> &'a Type { + match scalar.primitive() { + Int(i, _) => cx.type_from_integer(i), + F32 => cx.type_f32(), + F64 => cx.type_f64(), + Pointer => { + // If we know the alignment, pick something better than i8. + let (pointee, address_space) = + if let Some(pointee) = self.pointee_info_at(cx, offset) { + (cx.type_pointee_for_align(pointee.align), pointee.address_space) + } else { + (cx.type_i8(), AddressSpace::DATA) + }; + cx.type_ptr_to_ext(pointee, address_space) + } + } + } + + fn scalar_pair_element_llvm_type<'a>( + &self, + cx: &CodegenCx<'a, 'tcx>, + index: usize, + immediate: bool, + ) -> &'a Type { + // HACK(eddyb) special-case fat pointers until LLVM removes + // pointee types, to avoid bitcasting every `OperandRef::deref`. + match self.ty.kind() { + ty::Ref(..) | ty::RawPtr(_) => { + return self.field(cx, index).llvm_type(cx); + } + // only wide pointer boxes are handled as pointers + // thin pointer boxes with scalar allocators are handled by the general logic below + ty::Adt(def, substs) if def.is_box() && cx.layout_of(substs.type_at(1)).is_zst() => { + let ptr_ty = cx.tcx.mk_mut_ptr(self.ty.boxed_ty()); + return cx.layout_of(ptr_ty).scalar_pair_element_llvm_type(cx, index, immediate); + } + _ => {} + } + + let Abi::ScalarPair(a, b) = self.abi else { + bug!("TyAndLayout::scalar_pair_element_llty({:?}): not applicable", self); + }; + let scalar = [a, b][index]; + + // Make sure to return the same type `immediate_llvm_type` would when + // dealing with an immediate pair. This means that `(bool, bool)` is + // effectively represented as `{i8, i8}` in memory and two `i1`s as an + // immediate, just like `bool` is typically `i8` in memory and only `i1` + // when immediate. We need to load/store `bool` as `i8` to avoid + // crippling LLVM optimizations or triggering other LLVM bugs with `i1`. + if immediate && scalar.is_bool() { + return cx.type_i1(); + } + + let offset = if index == 0 { Size::ZERO } else { a.size(cx).align_to(b.align(cx).abi) }; + self.scalar_llvm_type_at(cx, scalar, offset) + } + + fn llvm_field_index<'a>(&self, cx: &CodegenCx<'a, 'tcx>, index: usize) -> u64 { + match self.abi { + Abi::Scalar(_) | Abi::ScalarPair(..) => { + bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) + } + _ => {} + } + match self.fields { + FieldsShape::Primitive | FieldsShape::Union(_) => { + bug!("TyAndLayout::llvm_field_index({:?}): not applicable", self) + } + + FieldsShape::Array { .. } => index as u64, + + FieldsShape::Arbitrary { .. } => { + let variant_index = match self.variants { + Variants::Single { index } => Some(index), + _ => None, + }; + + // Look up llvm field if indexes do not match memory order due to padding. If + // `field_remapping` is `None` no padding was used and the llvm field index + // matches the memory index. + match cx.type_lowering.borrow().get(&(self.ty, variant_index)) { + Some(TypeLowering { field_remapping: Some(ref remap), .. }) => { + remap[index] as u64 + } + Some(_) => self.fields.memory_index(index) as u64, + None => { + bug!("TyAndLayout::llvm_field_index({:?}): type info not found", self) + } + } + } + } + } + + // FIXME(eddyb) this having the same name as `TyAndLayout::pointee_info_at` + // (the inherent method, which is lacking this caching logic) can result in + // the uncached version being called - not wrong, but potentially inefficient. + fn pointee_info_at<'a>(&self, cx: &CodegenCx<'a, 'tcx>, offset: Size) -> Option { + if let Some(&pointee) = cx.pointee_infos.borrow().get(&(self.ty, offset)) { + return pointee; + } + + let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset); + + cx.pointee_infos.borrow_mut().insert((self.ty, offset), result); + result + } +} diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs new file mode 100644 index 000000000..ceb3d5a84 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -0,0 +1,214 @@ +use crate::builder::Builder; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; +use rustc_codegen_ssa::mir::operand::OperandRef; +use rustc_codegen_ssa::{ + common::IntPredicate, + traits::{BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods}, +}; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::Ty; +use rustc_target::abi::{Align, Endian, HasDataLayout, Size}; + +fn round_pointer_up_to_alignment<'ll>( + bx: &mut Builder<'_, 'll, '_>, + addr: &'ll Value, + align: Align, + ptr_ty: &'ll Type, +) -> &'ll Value { + let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize()); + ptr_as_int = bx.add(ptr_as_int, bx.cx().const_i32(align.bytes() as i32 - 1)); + ptr_as_int = bx.and(ptr_as_int, bx.cx().const_i32(-(align.bytes() as i32))); + bx.inttoptr(ptr_as_int, ptr_ty) +} + +fn emit_direct_ptr_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + llty: &'ll Type, + size: Size, + align: Align, + slot_size: Align, + allow_higher_align: bool, +) -> (&'ll Value, Align) { + let va_list_ty = bx.type_i8p(); + let va_list_ptr_ty = bx.type_ptr_to(va_list_ty); + let va_list_addr = if list.layout.llvm_type(bx.cx) != va_list_ptr_ty { + bx.bitcast(list.immediate(), va_list_ptr_ty) + } else { + list.immediate() + }; + + let ptr = bx.load(va_list_ty, va_list_addr, bx.tcx().data_layout.pointer_align.abi); + + let (addr, addr_align) = if allow_higher_align && align > slot_size { + (round_pointer_up_to_alignment(bx, ptr, align, bx.cx().type_i8p()), align) + } else { + (ptr, slot_size) + }; + + let aligned_size = size.align_to(slot_size).bytes() as i32; + let full_direct_size = bx.cx().const_i32(aligned_size); + let next = bx.inbounds_gep(bx.type_i8(), addr, &[full_direct_size]); + bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi); + + if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big { + let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32); + let adjusted = bx.inbounds_gep(bx.type_i8(), addr, &[adjusted_size]); + (bx.bitcast(adjusted, bx.cx().type_ptr_to(llty)), addr_align) + } else { + (bx.bitcast(addr, bx.cx().type_ptr_to(llty)), addr_align) + } +} + +fn emit_ptr_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, + indirect: bool, + slot_size: Align, + allow_higher_align: bool, +) -> &'ll Value { + let layout = bx.cx.layout_of(target_ty); + let (llty, size, align) = if indirect { + ( + bx.cx.layout_of(bx.cx.tcx.mk_imm_ptr(target_ty)).llvm_type(bx.cx), + bx.cx.data_layout().pointer_size, + bx.cx.data_layout().pointer_align, + ) + } else { + (layout.llvm_type(bx.cx), layout.size, layout.align) + }; + let (addr, addr_align) = + emit_direct_ptr_va_arg(bx, list, llty, size, align.abi, slot_size, allow_higher_align); + if indirect { + let tmp_ret = bx.load(llty, addr, addr_align); + bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi) + } else { + bx.load(llty, addr, addr_align) + } +} + +fn emit_aapcs_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of the AAPCS64 calling convention for va_args see + // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst + let va_list_addr = list.immediate(); + let va_list_layout = list.deref(bx.cx).layout; + let va_list_ty = va_list_layout.llvm_type(bx); + let layout = bx.cx.layout_of(target_ty); + + let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg"); + let in_reg = bx.append_sibling_block("va_arg.in_reg"); + let on_stack = bx.append_sibling_block("va_arg.on_stack"); + let end = bx.append_sibling_block("va_arg.end"); + let zero = bx.const_i32(0); + let offset_align = Align::from_bytes(4).unwrap(); + + let gr_type = target_ty.is_any_ptr() || target_ty.is_integral(); + let (reg_off, reg_top_index, slot_size) = if gr_type { + let gr_offs = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); + let nreg = (layout.size.bytes() + 7) / 8; + (gr_offs, va_list_layout.llvm_field_index(bx.cx, 1), nreg * 8) + } else { + let vr_off = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 4)); + let nreg = (layout.size.bytes() + 15) / 16; + (vr_off, va_list_layout.llvm_field_index(bx.cx, 2), nreg * 16) + }; + + // if the offset >= 0 then the value will be on the stack + let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align); + let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero); + bx.cond_br(use_stack, on_stack, maybe_reg); + + // The value at this point might be in a register, but there is a chance that + // it could be on the stack so we have to update the offset and then check + // the offset again. + + bx.switch_to_block(maybe_reg); + if gr_type && layout.align.abi.bytes() > 8 { + reg_off_v = bx.add(reg_off_v, bx.const_i32(15)); + reg_off_v = bx.and(reg_off_v, bx.const_i32(-16)); + } + let new_reg_off_v = bx.add(reg_off_v, bx.const_i32(slot_size as i32)); + + bx.store(new_reg_off_v, reg_off, offset_align); + + // Check to see if we have overflowed the registers as a result of this. + // If we have then we need to use the stack for this value + let use_stack = bx.icmp(IntPredicate::IntSGT, new_reg_off_v, zero); + bx.cond_br(use_stack, on_stack, in_reg); + + bx.switch_to_block(in_reg); + let top_type = bx.type_i8p(); + let top = bx.struct_gep(va_list_ty, va_list_addr, reg_top_index); + let top = bx.load(top_type, top, bx.tcx().data_layout.pointer_align.abi); + + // reg_value = *(@top + reg_off_v); + let mut reg_addr = bx.gep(bx.type_i8(), top, &[reg_off_v]); + if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size { + // On big-endian systems the value is right-aligned in its slot. + let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32); + reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]); + } + let reg_type = layout.llvm_type(bx); + let reg_addr = bx.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type)); + let reg_value = bx.load(reg_type, reg_addr, layout.align.abi); + bx.br(end); + + // On Stack block + bx.switch_to_block(on_stack); + let stack_value = + emit_ptr_va_arg(bx, list, target_ty, false, Align::from_bytes(8).unwrap(), true); + bx.br(end); + + bx.switch_to_block(end); + let val = + bx.phi(layout.immediate_llvm_type(bx), &[reg_value, stack_value], &[in_reg, on_stack]); + + val +} + +pub(super) fn emit_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + addr: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Determine the va_arg implementation to use. The LLVM va_arg instruction + // is lacking in some instances, so we should only use it as a fallback. + let target = &bx.cx.tcx.sess.target; + let arch = &bx.cx.tcx.sess.target.arch; + match &**arch { + // Windows x86 + "x86" if target.is_like_windows => { + emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), false) + } + // Generic x86 + "x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true), + // Windows AArch64 + "aarch64" if target.is_like_windows => { + emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false) + } + // macOS / iOS AArch64 + "aarch64" if target.is_like_osx => { + emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true) + } + "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), + // Windows x86_64 + "x86_64" if target.is_like_windows => { + let target_ty_size = bx.cx.size_of(target_ty).bytes(); + let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); + emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false) + } + // For all other architecture/OS combinations fall back to using + // the LLVM va_arg instruction. + // https://llvm.org/docs/LangRef.html#va-arg-instruction + _ => bx.va_arg(addr.immediate(), bx.cx.layout_of(target_ty).llvm_type(bx.cx)), + } +} diff --git a/compiler/rustc_codegen_llvm/src/value.rs b/compiler/rustc_codegen_llvm/src/value.rs new file mode 100644 index 000000000..1338a2295 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/value.rs @@ -0,0 +1,32 @@ +pub use crate::llvm::Value; + +use crate::llvm; + +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::ptr; + +impl PartialEq for Value { + fn eq(&self, other: &Self) -> bool { + ptr::eq(self, other) + } +} + +impl Eq for Value {} + +impl Hash for Value { + fn hash(&self, hasher: &mut H) { + (self as *const Self).hash(hasher); + } +} + +impl fmt::Debug for Value { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str( + &llvm::build_string(|s| unsafe { + llvm::LLVMRustWriteValueToString(self, s); + }) + .expect("non-UTF8 value description from LLVM"), + ) + } +} -- cgit v1.2.3