use super::operand::{OperandRef, OperandValue}; use super::place::PlaceRef; use super::FunctionCx; use crate::common::IntPredicate; use crate::errors; use crate::errors::InvalidMonomorphization; use crate::glue; use crate::meth; use crate::traits::*; use crate::MemFlags; use rustc_middle::ty::{self, Ty, TyCtxt}; use rustc_span::{sym, Span}; use rustc_target::abi::{ call::{FnAbi, PassMode}, WrappingRange, }; fn copy_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, allow_overlap: bool, volatile: bool, ty: Ty<'tcx>, dst: Bx::Value, src: Bx::Value, count: Bx::Value, ) { let layout = bx.layout_of(ty); let size = layout.size; let align = layout.align.abi; let size = bx.mul(bx.const_usize(size.bytes()), count); let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; if allow_overlap { bx.memmove(dst, align, src, align, size, flags); } else { bx.memcpy(dst, align, src, align, size, flags); } } fn memset_intrinsic<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>( bx: &mut Bx, volatile: bool, ty: Ty<'tcx>, dst: Bx::Value, val: Bx::Value, count: Bx::Value, ) { let layout = bx.layout_of(ty); let size = layout.size; let align = layout.align.abi; let size = bx.mul(bx.const_usize(size.bytes()), count); let flags = if volatile { MemFlags::VOLATILE } else { MemFlags::empty() }; bx.memset(dst, val, size, align, flags); } impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { pub fn codegen_intrinsic_call( bx: &mut Bx, instance: ty::Instance<'tcx>, fn_abi: &FnAbi<'tcx, Ty<'tcx>>, args: &[OperandRef<'tcx, Bx::Value>], llresult: Bx::Value, span: Span, ) { let callee_ty = instance.ty(bx.tcx(), ty::ParamEnv::reveal_all()); let ty::FnDef(def_id, substs) = *callee_ty.kind() else { bug!("expected fn item type, found {}", callee_ty); }; let sig = callee_ty.fn_sig(bx.tcx()); let sig = bx.tcx().normalize_erasing_late_bound_regions(ty::ParamEnv::reveal_all(), sig); let arg_tys = sig.inputs(); let ret_ty = sig.output(); let name = bx.tcx().item_name(def_id); let name_str = name.as_str(); let llret_ty = bx.backend_type(bx.layout_of(ret_ty)); let result = PlaceRef::new_sized(llresult, fn_abi.ret.layout); let llval = match name { sym::abort => { bx.abort(); return; } sym::va_start => bx.va_start(args[0].immediate()), sym::va_end => bx.va_end(args[0].immediate()), sym::size_of_val => { let tp_ty = substs.type_at(0); if let OperandValue::Pair(_, meta) = args[0].val { let (llsize, _) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta)); llsize } else { bx.const_usize(bx.layout_of(tp_ty).size.bytes()) } } sym::min_align_of_val => { let tp_ty = substs.type_at(0); if let OperandValue::Pair(_, meta) = args[0].val { let (_, llalign) = glue::size_and_align_of_dst(bx, tp_ty, Some(meta)); llalign } else { bx.const_usize(bx.layout_of(tp_ty).align.abi.bytes()) } } sym::vtable_size | sym::vtable_align => { let vtable = args[0].immediate(); let idx = match name { sym::vtable_size => ty::COMMON_VTABLE_ENTRIES_SIZE, sym::vtable_align => ty::COMMON_VTABLE_ENTRIES_ALIGN, _ => bug!(), }; let value = meth::VirtualIndex::from_index(idx).get_usize(bx, vtable); match name { // Size is always <= isize::MAX. sym::vtable_size => { let size_bound = bx.data_layout().ptr_sized_integer().signed_max() as u128; bx.range_metadata(value, WrappingRange { start: 0, end: size_bound }); }, // Alignment is always nonzero. sym::vtable_align => bx.range_metadata(value, WrappingRange { start: 1, end: !0 }), _ => {} } value } sym::pref_align_of | sym::needs_drop | sym::type_id | sym::type_name | sym::variant_count => { let value = bx .tcx() .const_eval_instance(ty::ParamEnv::reveal_all(), instance, None) .unwrap(); OperandRef::from_const(bx, value, ret_ty).immediate_or_packed_pair(bx) } sym::offset => { let ty = substs.type_at(0); let layout = bx.layout_of(ty); let ptr = args[0].immediate(); let offset = args[1].immediate(); bx.inbounds_gep(bx.backend_type(layout), ptr, &[offset]) } sym::arith_offset => { let ty = substs.type_at(0); let layout = bx.layout_of(ty); let ptr = args[0].immediate(); let offset = args[1].immediate(); bx.gep(bx.backend_type(layout), ptr, &[offset]) } sym::copy => { copy_intrinsic( bx, true, false, substs.type_at(0), args[1].immediate(), args[0].immediate(), args[2].immediate(), ); return; } sym::write_bytes => { memset_intrinsic( bx, false, substs.type_at(0), args[0].immediate(), args[1].immediate(), args[2].immediate(), ); return; } sym::volatile_copy_nonoverlapping_memory => { copy_intrinsic( bx, false, true, substs.type_at(0), args[0].immediate(), args[1].immediate(), args[2].immediate(), ); return; } sym::volatile_copy_memory => { copy_intrinsic( bx, true, true, substs.type_at(0), args[0].immediate(), args[1].immediate(), args[2].immediate(), ); return; } sym::volatile_set_memory => { memset_intrinsic( bx, true, substs.type_at(0), args[0].immediate(), args[1].immediate(), args[2].immediate(), ); return; } sym::volatile_store => { let dst = args[0].deref(bx.cx()); args[1].val.volatile_store(bx, dst); return; } sym::unaligned_volatile_store => { let dst = args[0].deref(bx.cx()); args[1].val.unaligned_volatile_store(bx, dst); return; } | sym::unchecked_div | sym::unchecked_rem | sym::unchecked_shl | sym::unchecked_shr | sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul | sym::exact_div => { let ty = arg_tys[0]; match int_type_width_signed(ty, bx.tcx()) { Some((_width, signed)) => match name { sym::exact_div => { if signed { bx.exactsdiv(args[0].immediate(), args[1].immediate()) } else { bx.exactudiv(args[0].immediate(), args[1].immediate()) } } sym::unchecked_div => { if signed { bx.sdiv(args[0].immediate(), args[1].immediate()) } else { bx.udiv(args[0].immediate(), args[1].immediate()) } } sym::unchecked_rem => { if signed { bx.srem(args[0].immediate(), args[1].immediate()) } else { bx.urem(args[0].immediate(), args[1].immediate()) } } sym::unchecked_shl => bx.shl(args[0].immediate(), args[1].immediate()), sym::unchecked_shr => { if signed { bx.ashr(args[0].immediate(), args[1].immediate()) } else { bx.lshr(args[0].immediate(), args[1].immediate()) } } sym::unchecked_add => { if signed { bx.unchecked_sadd(args[0].immediate(), args[1].immediate()) } else { bx.unchecked_uadd(args[0].immediate(), args[1].immediate()) } } sym::unchecked_sub => { if signed { bx.unchecked_ssub(args[0].immediate(), args[1].immediate()) } else { bx.unchecked_usub(args[0].immediate(), args[1].immediate()) } } sym::unchecked_mul => { if signed { bx.unchecked_smul(args[0].immediate(), args[1].immediate()) } else { bx.unchecked_umul(args[0].immediate(), args[1].immediate()) } } _ => bug!(), }, None => { bx.tcx().sess.emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty }); return; } } } sym::fadd_fast | sym::fsub_fast | sym::fmul_fast | sym::fdiv_fast | sym::frem_fast => { match float_type_width(arg_tys[0]) { Some(_width) => match name { sym::fadd_fast => bx.fadd_fast(args[0].immediate(), args[1].immediate()), sym::fsub_fast => bx.fsub_fast(args[0].immediate(), args[1].immediate()), sym::fmul_fast => bx.fmul_fast(args[0].immediate(), args[1].immediate()), sym::fdiv_fast => bx.fdiv_fast(args[0].immediate(), args[1].immediate()), sym::frem_fast => bx.frem_fast(args[0].immediate(), args[1].immediate()), _ => bug!(), }, None => { bx.tcx().sess.emit_err(InvalidMonomorphization::BasicFloatType { span, name, ty: arg_tys[0] }); return; } } } sym::float_to_int_unchecked => { if float_type_width(arg_tys[0]).is_none() { bx.tcx().sess.emit_err(InvalidMonomorphization::FloatToIntUnchecked { span, ty: arg_tys[0] }); return; } let Some((_width, signed)) = int_type_width_signed(ret_ty, bx.tcx()) else { bx.tcx().sess.emit_err(InvalidMonomorphization::FloatToIntUnchecked { span, ty: ret_ty }); return; }; if signed { bx.fptosi(args[0].immediate(), llret_ty) } else { bx.fptoui(args[0].immediate(), llret_ty) } } sym::discriminant_value => { if ret_ty.is_integral() { args[0].deref(bx.cx()).codegen_get_discr(bx, ret_ty) } else { span_bug!(span, "Invalid discriminant type for `{:?}`", arg_tys[0]) } } sym::const_allocate => { // returns a null pointer at runtime. bx.const_null(bx.type_i8p()) } sym::const_deallocate => { // nop at runtime. return; } // This requires that atomic intrinsics follow a specific naming pattern: // "atomic_[_]" name if let Some(atomic) = name_str.strip_prefix("atomic_") => { use crate::common::AtomicOrdering::*; use crate::common::{AtomicRmwBinOp, SynchronizationScope}; let Some((instruction, ordering)) = atomic.split_once('_') else { bx.sess().emit_fatal(errors::MissingMemoryOrdering); }; let parse_ordering = |bx: &Bx, s| match s { "unordered" => Unordered, "relaxed" => Relaxed, "acquire" => Acquire, "release" => Release, "acqrel" => AcquireRelease, "seqcst" => SequentiallyConsistent, _ => bx.sess().emit_fatal(errors::UnknownAtomicOrdering), }; let invalid_monomorphization = |ty| { bx.tcx().sess.emit_err(InvalidMonomorphization::BasicIntegerType { span, name, ty }); }; match instruction { "cxchg" | "cxchgweak" => { let Some((success, failure)) = ordering.split_once('_') else { bx.sess().emit_fatal(errors::AtomicCompareExchange); }; let ty = substs.type_at(0); if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { let weak = instruction == "cxchgweak"; let mut dst = args[0].immediate(); let mut cmp = args[1].immediate(); let mut src = args[2].immediate(); if ty.is_unsafe_ptr() { // Some platforms do not support atomic operations on pointers, // so we cast to integer first. let ptr_llty = bx.type_ptr_to(bx.type_isize()); dst = bx.pointercast(dst, ptr_llty); cmp = bx.ptrtoint(cmp, bx.type_isize()); src = bx.ptrtoint(src, bx.type_isize()); } let pair = bx.atomic_cmpxchg(dst, cmp, src, parse_ordering(bx, success), parse_ordering(bx, failure), weak); let val = bx.extract_value(pair, 0); let success = bx.extract_value(pair, 1); let val = bx.from_immediate(val); let success = bx.from_immediate(success); let dest = result.project_field(bx, 0); bx.store(val, dest.llval, dest.align); let dest = result.project_field(bx, 1); bx.store(success, dest.llval, dest.align); return; } else { return invalid_monomorphization(ty); } } "load" => { let ty = substs.type_at(0); if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { let layout = bx.layout_of(ty); let size = layout.size; let mut source = args[0].immediate(); if ty.is_unsafe_ptr() { // Some platforms do not support atomic operations on pointers, // so we cast to integer first... let llty = bx.type_isize(); let ptr_llty = bx.type_ptr_to(llty); source = bx.pointercast(source, ptr_llty); let result = bx.atomic_load(llty, source, parse_ordering(bx, ordering), size); // ... and then cast the result back to a pointer bx.inttoptr(result, bx.backend_type(layout)) } else { bx.atomic_load(bx.backend_type(layout), source, parse_ordering(bx, ordering), size) } } else { return invalid_monomorphization(ty); } } "store" => { let ty = substs.type_at(0); if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { let size = bx.layout_of(ty).size; let mut val = args[1].immediate(); let mut ptr = args[0].immediate(); if ty.is_unsafe_ptr() { // Some platforms do not support atomic operations on pointers, // so we cast to integer first. let ptr_llty = bx.type_ptr_to(bx.type_isize()); ptr = bx.pointercast(ptr, ptr_llty); val = bx.ptrtoint(val, bx.type_isize()); } bx.atomic_store(val, ptr, parse_ordering(bx, ordering), size); return; } else { return invalid_monomorphization(ty); } } "fence" => { bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::CrossThread); return; } "singlethreadfence" => { bx.atomic_fence(parse_ordering(bx, ordering), SynchronizationScope::SingleThread); return; } // These are all AtomicRMW ops op => { let atom_op = match op { "xchg" => AtomicRmwBinOp::AtomicXchg, "xadd" => AtomicRmwBinOp::AtomicAdd, "xsub" => AtomicRmwBinOp::AtomicSub, "and" => AtomicRmwBinOp::AtomicAnd, "nand" => AtomicRmwBinOp::AtomicNand, "or" => AtomicRmwBinOp::AtomicOr, "xor" => AtomicRmwBinOp::AtomicXor, "max" => AtomicRmwBinOp::AtomicMax, "min" => AtomicRmwBinOp::AtomicMin, "umax" => AtomicRmwBinOp::AtomicUMax, "umin" => AtomicRmwBinOp::AtomicUMin, _ => bx.sess().emit_fatal(errors::UnknownAtomicOperation), }; let ty = substs.type_at(0); if int_type_width_signed(ty, bx.tcx()).is_some() || ty.is_unsafe_ptr() { let mut ptr = args[0].immediate(); let mut val = args[1].immediate(); if ty.is_unsafe_ptr() { // Some platforms do not support atomic operations on pointers, // so we cast to integer first. let ptr_llty = bx.type_ptr_to(bx.type_isize()); ptr = bx.pointercast(ptr, ptr_llty); val = bx.ptrtoint(val, bx.type_isize()); } bx.atomic_rmw(atom_op, ptr, val, parse_ordering(bx, ordering)) } else { return invalid_monomorphization(ty); } } } } sym::nontemporal_store => { let dst = args[0].deref(bx.cx()); args[1].val.nontemporal_store(bx, dst); return; } sym::ptr_guaranteed_cmp => { let a = args[0].immediate(); let b = args[1].immediate(); bx.icmp(IntPredicate::IntEQ, a, b) } sym::ptr_offset_from | sym::ptr_offset_from_unsigned => { let ty = substs.type_at(0); let pointee_size = bx.layout_of(ty).size; let a = args[0].immediate(); let b = args[1].immediate(); let a = bx.ptrtoint(a, bx.type_isize()); let b = bx.ptrtoint(b, bx.type_isize()); let pointee_size = bx.const_usize(pointee_size.bytes()); if name == sym::ptr_offset_from { // This is the same sequence that Clang emits for pointer subtraction. // It can be neither `nsw` nor `nuw` because the input is treated as // unsigned but then the output is treated as signed, so neither works. let d = bx.sub(a, b); // this is where the signed magic happens (notice the `s` in `exactsdiv`) bx.exactsdiv(d, pointee_size) } else { // The `_unsigned` version knows the relative ordering of the pointers, // so can use `sub nuw` and `udiv exact` instead of dealing in signed. let d = bx.unchecked_usub(a, b); bx.exactudiv(d, pointee_size) } } _ => { // Need to use backend-specific things in the implementation. bx.codegen_intrinsic_call(instance, fn_abi, args, llresult, span); return; } }; if !fn_abi.ret.is_ignore() { if let PassMode::Cast(ty, _) = &fn_abi.ret.mode { let ptr_llty = bx.type_ptr_to(bx.cast_backend_type(ty)); let ptr = bx.pointercast(result.llval, ptr_llty); bx.store(llval, ptr, result.align); } else { OperandRef::from_immediate_or_packed_pair(bx, llval, result.layout) .val .store(bx, result); } } } } // Returns the width of an int Ty, and if it's signed or not // Returns None if the type is not an integer // FIXME: there’s multiple of this functions, investigate using some of the already existing // stuffs. fn int_type_width_signed(ty: Ty<'_>, tcx: TyCtxt<'_>) -> Option<(u64, bool)> { match ty.kind() { ty::Int(t) => { Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), true)) } ty::Uint(t) => { Some((t.bit_width().unwrap_or(u64::from(tcx.sess.target.pointer_width)), false)) } _ => None, } } // Returns the width of a float Ty // Returns None if the type is not a float fn float_type_width(ty: Ty<'_>) -> Option { match ty.kind() { ty::Float(t) => Some(t.bit_width()), _ => None, } }