diff options
Diffstat (limited to '')
-rw-r--r-- | compiler/rustc_codegen_llvm/src/va_arg.rs | 214 |
1 files changed, 214 insertions, 0 deletions
diff --git a/compiler/rustc_codegen_llvm/src/va_arg.rs b/compiler/rustc_codegen_llvm/src/va_arg.rs new file mode 100644 index 000000000..ceb3d5a84 --- /dev/null +++ b/compiler/rustc_codegen_llvm/src/va_arg.rs @@ -0,0 +1,214 @@ +use crate::builder::Builder; +use crate::type_::Type; +use crate::type_of::LayoutLlvmExt; +use crate::value::Value; +use rustc_codegen_ssa::mir::operand::OperandRef; +use rustc_codegen_ssa::{ + common::IntPredicate, + traits::{BaseTypeMethods, BuilderMethods, ConstMethods, DerivedTypeMethods}, +}; +use rustc_middle::ty::layout::{HasTyCtxt, LayoutOf}; +use rustc_middle::ty::Ty; +use rustc_target::abi::{Align, Endian, HasDataLayout, Size}; + +fn round_pointer_up_to_alignment<'ll>( + bx: &mut Builder<'_, 'll, '_>, + addr: &'ll Value, + align: Align, + ptr_ty: &'ll Type, +) -> &'ll Value { + let mut ptr_as_int = bx.ptrtoint(addr, bx.cx().type_isize()); + ptr_as_int = bx.add(ptr_as_int, bx.cx().const_i32(align.bytes() as i32 - 1)); + ptr_as_int = bx.and(ptr_as_int, bx.cx().const_i32(-(align.bytes() as i32))); + bx.inttoptr(ptr_as_int, ptr_ty) +} + +fn emit_direct_ptr_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + llty: &'ll Type, + size: Size, + align: Align, + slot_size: Align, + allow_higher_align: bool, +) -> (&'ll Value, Align) { + let va_list_ty = bx.type_i8p(); + let va_list_ptr_ty = bx.type_ptr_to(va_list_ty); + let va_list_addr = if list.layout.llvm_type(bx.cx) != va_list_ptr_ty { + bx.bitcast(list.immediate(), va_list_ptr_ty) + } else { + list.immediate() + }; + + let ptr = bx.load(va_list_ty, va_list_addr, bx.tcx().data_layout.pointer_align.abi); + + let (addr, addr_align) = if allow_higher_align && align > slot_size { + (round_pointer_up_to_alignment(bx, ptr, align, bx.cx().type_i8p()), align) + } else { + (ptr, slot_size) + }; + + let aligned_size = size.align_to(slot_size).bytes() as i32; + let full_direct_size = bx.cx().const_i32(aligned_size); + let next = bx.inbounds_gep(bx.type_i8(), addr, &[full_direct_size]); + bx.store(next, va_list_addr, bx.tcx().data_layout.pointer_align.abi); + + if size.bytes() < slot_size.bytes() && bx.tcx().sess.target.endian == Endian::Big { + let adjusted_size = bx.cx().const_i32((slot_size.bytes() - size.bytes()) as i32); + let adjusted = bx.inbounds_gep(bx.type_i8(), addr, &[adjusted_size]); + (bx.bitcast(adjusted, bx.cx().type_ptr_to(llty)), addr_align) + } else { + (bx.bitcast(addr, bx.cx().type_ptr_to(llty)), addr_align) + } +} + +fn emit_ptr_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, + indirect: bool, + slot_size: Align, + allow_higher_align: bool, +) -> &'ll Value { + let layout = bx.cx.layout_of(target_ty); + let (llty, size, align) = if indirect { + ( + bx.cx.layout_of(bx.cx.tcx.mk_imm_ptr(target_ty)).llvm_type(bx.cx), + bx.cx.data_layout().pointer_size, + bx.cx.data_layout().pointer_align, + ) + } else { + (layout.llvm_type(bx.cx), layout.size, layout.align) + }; + let (addr, addr_align) = + emit_direct_ptr_va_arg(bx, list, llty, size, align.abi, slot_size, allow_higher_align); + if indirect { + let tmp_ret = bx.load(llty, addr, addr_align); + bx.load(bx.cx.layout_of(target_ty).llvm_type(bx.cx), tmp_ret, align.abi) + } else { + bx.load(llty, addr, addr_align) + } +} + +fn emit_aapcs_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + list: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Implementation of the AAPCS64 calling convention for va_args see + // https://github.com/ARM-software/abi-aa/blob/master/aapcs64/aapcs64.rst + let va_list_addr = list.immediate(); + let va_list_layout = list.deref(bx.cx).layout; + let va_list_ty = va_list_layout.llvm_type(bx); + let layout = bx.cx.layout_of(target_ty); + + let maybe_reg = bx.append_sibling_block("va_arg.maybe_reg"); + let in_reg = bx.append_sibling_block("va_arg.in_reg"); + let on_stack = bx.append_sibling_block("va_arg.on_stack"); + let end = bx.append_sibling_block("va_arg.end"); + let zero = bx.const_i32(0); + let offset_align = Align::from_bytes(4).unwrap(); + + let gr_type = target_ty.is_any_ptr() || target_ty.is_integral(); + let (reg_off, reg_top_index, slot_size) = if gr_type { + let gr_offs = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 3)); + let nreg = (layout.size.bytes() + 7) / 8; + (gr_offs, va_list_layout.llvm_field_index(bx.cx, 1), nreg * 8) + } else { + let vr_off = + bx.struct_gep(va_list_ty, va_list_addr, va_list_layout.llvm_field_index(bx.cx, 4)); + let nreg = (layout.size.bytes() + 15) / 16; + (vr_off, va_list_layout.llvm_field_index(bx.cx, 2), nreg * 16) + }; + + // if the offset >= 0 then the value will be on the stack + let mut reg_off_v = bx.load(bx.type_i32(), reg_off, offset_align); + let use_stack = bx.icmp(IntPredicate::IntSGE, reg_off_v, zero); + bx.cond_br(use_stack, on_stack, maybe_reg); + + // The value at this point might be in a register, but there is a chance that + // it could be on the stack so we have to update the offset and then check + // the offset again. + + bx.switch_to_block(maybe_reg); + if gr_type && layout.align.abi.bytes() > 8 { + reg_off_v = bx.add(reg_off_v, bx.const_i32(15)); + reg_off_v = bx.and(reg_off_v, bx.const_i32(-16)); + } + let new_reg_off_v = bx.add(reg_off_v, bx.const_i32(slot_size as i32)); + + bx.store(new_reg_off_v, reg_off, offset_align); + + // Check to see if we have overflowed the registers as a result of this. + // If we have then we need to use the stack for this value + let use_stack = bx.icmp(IntPredicate::IntSGT, new_reg_off_v, zero); + bx.cond_br(use_stack, on_stack, in_reg); + + bx.switch_to_block(in_reg); + let top_type = bx.type_i8p(); + let top = bx.struct_gep(va_list_ty, va_list_addr, reg_top_index); + let top = bx.load(top_type, top, bx.tcx().data_layout.pointer_align.abi); + + // reg_value = *(@top + reg_off_v); + let mut reg_addr = bx.gep(bx.type_i8(), top, &[reg_off_v]); + if bx.tcx().sess.target.endian == Endian::Big && layout.size.bytes() != slot_size { + // On big-endian systems the value is right-aligned in its slot. + let offset = bx.const_i32((slot_size - layout.size.bytes()) as i32); + reg_addr = bx.gep(bx.type_i8(), reg_addr, &[offset]); + } + let reg_type = layout.llvm_type(bx); + let reg_addr = bx.bitcast(reg_addr, bx.cx.type_ptr_to(reg_type)); + let reg_value = bx.load(reg_type, reg_addr, layout.align.abi); + bx.br(end); + + // On Stack block + bx.switch_to_block(on_stack); + let stack_value = + emit_ptr_va_arg(bx, list, target_ty, false, Align::from_bytes(8).unwrap(), true); + bx.br(end); + + bx.switch_to_block(end); + let val = + bx.phi(layout.immediate_llvm_type(bx), &[reg_value, stack_value], &[in_reg, on_stack]); + + val +} + +pub(super) fn emit_va_arg<'ll, 'tcx>( + bx: &mut Builder<'_, 'll, 'tcx>, + addr: OperandRef<'tcx, &'ll Value>, + target_ty: Ty<'tcx>, +) -> &'ll Value { + // Determine the va_arg implementation to use. The LLVM va_arg instruction + // is lacking in some instances, so we should only use it as a fallback. + let target = &bx.cx.tcx.sess.target; + let arch = &bx.cx.tcx.sess.target.arch; + match &**arch { + // Windows x86 + "x86" if target.is_like_windows => { + emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), false) + } + // Generic x86 + "x86" => emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(4).unwrap(), true), + // Windows AArch64 + "aarch64" if target.is_like_windows => { + emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), false) + } + // macOS / iOS AArch64 + "aarch64" if target.is_like_osx => { + emit_ptr_va_arg(bx, addr, target_ty, false, Align::from_bytes(8).unwrap(), true) + } + "aarch64" => emit_aapcs_va_arg(bx, addr, target_ty), + // Windows x86_64 + "x86_64" if target.is_like_windows => { + let target_ty_size = bx.cx.size_of(target_ty).bytes(); + let indirect: bool = target_ty_size > 8 || !target_ty_size.is_power_of_two(); + emit_ptr_va_arg(bx, addr, target_ty, indirect, Align::from_bytes(8).unwrap(), false) + } + // For all other architecture/OS combinations fall back to using + // the LLVM va_arg instruction. + // https://llvm.org/docs/LangRef.html#va-arg-instruction + _ => bx.va_arg(addr.immediate(), bx.cx.layout_of(target_ty).llvm_type(bx.cx)), + } +} |