summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_codegen_cranelift/src/inline_asm.rs
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src/inline_asm.rs')
-rw-r--r--compiler/rustc_codegen_cranelift/src/inline_asm.rs167
1 files changed, 64 insertions, 103 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index ce0eecca8..73f4bc7c1 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -3,14 +3,13 @@
use std::fmt::Write;
use rustc_ast::ast::{InlineAsmOptions, InlineAsmTemplatePiece};
-use rustc_middle::mir::InlineAsmOperand;
use rustc_span::sym;
use rustc_target::asm::*;
use target_lexicon::BinaryFormat;
use crate::prelude::*;
-enum CInlineAsmOperand<'tcx> {
+pub(crate) enum CInlineAsmOperand<'tcx> {
In {
reg: InlineAsmRegOrRegClass,
value: Value,
@@ -34,7 +33,7 @@ enum CInlineAsmOperand<'tcx> {
},
}
-pub(crate) fn codegen_inline_asm<'tcx>(
+pub(crate) fn codegen_inline_asm_terminator<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
span: Span,
template: &[InlineAsmTemplatePiece],
@@ -42,8 +41,6 @@ pub(crate) fn codegen_inline_asm<'tcx>(
options: InlineAsmOptions,
destination: Option<mir::BasicBlock>,
) {
- // FIXME add .eh_frame unwind info directives
-
// Used by panic_abort on Windows, but uses a syntax which only happens to work with
// asm!() by accident and breaks with the GNU assembler as well as global_asm!() for
// the LLVM backend.
@@ -135,15 +132,33 @@ pub(crate) fn codegen_inline_asm<'tcx>(
})
.collect::<Vec<_>>();
- let mut inputs = Vec::new();
- let mut outputs = Vec::new();
+ codegen_inline_asm_inner(fx, template, &operands, options);
+
+ match destination {
+ Some(destination) => {
+ let destination_block = fx.get_block(destination);
+ fx.bcx.ins().jump(destination_block, &[]);
+ }
+ None => {
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+ }
+}
+
+pub(crate) fn codegen_inline_asm_inner<'tcx>(
+ fx: &mut FunctionCx<'_, '_, 'tcx>,
+ template: &[InlineAsmTemplatePiece],
+ operands: &[CInlineAsmOperand<'tcx>],
+ options: InlineAsmOptions,
+) {
+ // FIXME add .eh_frame unwind info directives
let mut asm_gen = InlineAssemblyGenerator {
tcx: fx.tcx,
arch: fx.tcx.sess.asm_arch.unwrap(),
enclosing_def_id: fx.instance.def_id(),
template,
- operands: &operands,
+ operands,
options,
registers: Vec::new(),
stack_slots_clobber: Vec::new(),
@@ -165,6 +180,8 @@ pub(crate) fn codegen_inline_asm<'tcx>(
let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
fx.cx.global_asm.push_str(&generated_asm);
+ let mut inputs = Vec::new();
+ let mut outputs = Vec::new();
for (i, operand) in operands.iter().enumerate() {
match operand {
CInlineAsmOperand::In { reg: _, value } => {
@@ -186,16 +203,6 @@ pub(crate) fn codegen_inline_asm<'tcx>(
}
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
-
- match destination {
- Some(destination) => {
- let destination_block = fx.get_block(destination);
- fx.bcx.ins().jump(destination_block, &[]);
- }
- None => {
- fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
- }
- }
}
struct InlineAssemblyGenerator<'a, 'tcx> {
@@ -637,8 +644,21 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
) {
match arch {
InlineAsmArch::X86_64 => {
- write!(generated_asm, " mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
- reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
+ match reg {
+ InlineAsmReg::X86(reg)
+ if reg as u32 >= X86InlineAsmReg::xmm0 as u32
+ && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
+ {
+ // rustc emits x0 rather than xmm0
+ write!(generated_asm, " movups [rbx+0x{:x}], ", offset.bytes()).unwrap();
+ write!(generated_asm, "xmm{}", reg as u32 - X86InlineAsmReg::xmm0 as u32)
+ .unwrap();
+ }
+ _ => {
+ write!(generated_asm, " mov [rbx+0x{:x}], ", offset.bytes()).unwrap();
+ reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
+ }
+ }
generated_asm.push('\n');
}
InlineAsmArch::AArch64 => {
@@ -663,8 +683,24 @@ impl<'tcx> InlineAssemblyGenerator<'_, 'tcx> {
) {
match arch {
InlineAsmArch::X86_64 => {
- generated_asm.push_str(" mov ");
- reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap();
+ match reg {
+ InlineAsmReg::X86(reg)
+ if reg as u32 >= X86InlineAsmReg::xmm0 as u32
+ && reg as u32 <= X86InlineAsmReg::xmm15 as u32 =>
+ {
+ // rustc emits x0 rather than xmm0
+ write!(
+ generated_asm,
+ " movups xmm{}",
+ reg as u32 - X86InlineAsmReg::xmm0 as u32
+ )
+ .unwrap();
+ }
+ _ => {
+ generated_asm.push_str(" mov ");
+ reg.emit(generated_asm, InlineAsmArch::X86_64, None).unwrap()
+ }
+ }
writeln!(generated_asm, ", [rbx+0x{:x}]", offset.bytes()).unwrap();
}
InlineAsmArch::AArch64 => {
@@ -720,7 +756,12 @@ fn call_inline_asm<'tcx>(
fx.bcx.ins().call(inline_asm_func, &[stack_slot_addr]);
for (offset, place) in outputs {
- let ty = fx.clif_type(place.layout().ty).unwrap();
+ let ty = if place.layout().ty.is_simd() {
+ let (lane_count, lane_type) = place.layout().ty.simd_size_and_type(fx.tcx);
+ fx.clif_type(lane_type).unwrap().by(lane_count.try_into().unwrap()).unwrap()
+ } else {
+ fx.clif_type(place.layout().ty).unwrap()
+ };
let value = stack_slot.offset(fx, i32::try_from(offset.bytes()).unwrap().into()).load(
fx,
ty,
@@ -729,83 +770,3 @@ fn call_inline_asm<'tcx>(
place.write_cvalue(fx, CValue::by_val(value, place.layout()));
}
}
-
-pub(crate) fn codegen_xgetbv<'tcx>(
- fx: &mut FunctionCx<'_, '_, 'tcx>,
- xcr_no: Value,
- ret: CPlace<'tcx>,
-) {
- // FIXME add .eh_frame unwind info directives
-
- let operands = vec![
- CInlineAsmOperand::In {
- reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
- value: xcr_no,
- },
- CInlineAsmOperand::Out {
- reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
- late: true,
- place: Some(ret),
- },
- CInlineAsmOperand::Out {
- reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
- late: true,
- place: None,
- },
- ];
- let options = InlineAsmOptions::NOSTACK | InlineAsmOptions::PURE | InlineAsmOptions::NOMEM;
-
- let mut inputs = Vec::new();
- let mut outputs = Vec::new();
-
- let mut asm_gen = InlineAssemblyGenerator {
- tcx: fx.tcx,
- arch: fx.tcx.sess.asm_arch.unwrap(),
- enclosing_def_id: fx.instance.def_id(),
- template: &[InlineAsmTemplatePiece::String(
- "
- xgetbv
- // out = rdx << 32 | rax
- shl rdx, 32
- or rax, rdx
- "
- .to_string(),
- )],
- operands: &operands,
- options,
- registers: Vec::new(),
- stack_slots_clobber: Vec::new(),
- stack_slots_input: Vec::new(),
- stack_slots_output: Vec::new(),
- stack_slot_size: Size::from_bytes(0),
- };
- asm_gen.allocate_registers();
- asm_gen.allocate_stack_slots();
-
- let inline_asm_index = fx.cx.inline_asm_index.get();
- fx.cx.inline_asm_index.set(inline_asm_index + 1);
- let asm_name = format!(
- "__inline_asm_{}_n{}",
- fx.cx.cgu_name.as_str().replace('.', "__").replace('-', "_"),
- inline_asm_index
- );
-
- let generated_asm = asm_gen.generate_asm_wrapper(&asm_name);
- fx.cx.global_asm.push_str(&generated_asm);
-
- for (i, operand) in operands.iter().enumerate() {
- match operand {
- CInlineAsmOperand::In { reg: _, value } => {
- inputs.push((asm_gen.stack_slots_input[i].unwrap(), *value));
- }
- CInlineAsmOperand::Out { reg: _, late: _, place } => {
- if let Some(place) = place {
- outputs.push((asm_gen.stack_slots_output[i].unwrap(), *place));
- }
- }
- _ => unreachable!(),
- }
- }
-
- call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
-}
generated by cgit v1.2.3 (git 2.39.1) at 2024-10-03 03:58:11 +0000