summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_codegen_cranelift/src/intrinsics
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-17 12:20:39 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-04-17 12:20:39 +0000
commit1376c5a617be5c25655d0d7cb63e3beaa5a6e026 (patch)
tree3bb8d61aee02bc7a15eab3f36e3b921afc2075d0 /compiler/rustc_codegen_cranelift/src/intrinsics
parentReleasing progress-linux version 1.69.0+dfsg1-1~progress7.99u1. (diff)
downloadrustc-1376c5a617be5c25655d0d7cb63e3beaa5a6e026.tar.xz
rustc-1376c5a617be5c25655d0d7cb63e3beaa5a6e026.zip
Merging upstream version 1.70.0+dfsg1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src/intrinsics')
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs4
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs161
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs11
3 files changed, 84 insertions, 92 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
index e5c4b244a..0f32d1a25 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs
@@ -179,8 +179,8 @@ fn llvm_add_sub<'tcx>(
// c + carry -> c + first intermediate carry or borrow respectively
let int0 = crate::num::codegen_checked_int_binop(fx, bin_op, a, b);
- let c = int0.value_field(fx, mir::Field::new(0));
- let cb0 = int0.value_field(fx, mir::Field::new(1)).load_scalar(fx);
+ let c = int0.value_field(fx, FieldIdx::new(0));
+ let cb0 = int0.value_field(fx, FieldIdx::new(1)).load_scalar(fx);
// c + carry -> c + second intermediate carry or borrow respectively
let cb_in_as_u64 = fx.bcx.ins().uextend(types::I64, cb_in);
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index e74aabf2f..03f2a65fc 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -23,7 +23,7 @@ pub(crate) use llvm::codegen_llvm_intrinsic_call;
use rustc_middle::ty;
use rustc_middle::ty::layout::{HasParamEnv, ValidityRequirement};
-use rustc_middle::ty::print::with_no_trimmed_paths;
+use rustc_middle::ty::print::{with_no_trimmed_paths, with_no_visible_paths};
use rustc_middle::ty::subst::SubstsRef;
use rustc_span::symbol::{kw, sym, Symbol};
@@ -252,41 +252,45 @@ fn codegen_float_intrinsic_call<'tcx>(
args: &[mir::Operand<'tcx>],
ret: CPlace<'tcx>,
) -> bool {
- let (name, arg_count, ty) = match intrinsic {
- sym::expf32 => ("expf", 1, fx.tcx.types.f32),
- sym::expf64 => ("exp", 1, fx.tcx.types.f64),
- sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32),
- sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64),
- sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32),
- sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64),
- sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32), // compiler-builtins
- sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64), // compiler-builtins
- sym::powf32 => ("powf", 2, fx.tcx.types.f32),
- sym::powf64 => ("pow", 2, fx.tcx.types.f64),
- sym::logf32 => ("logf", 1, fx.tcx.types.f32),
- sym::logf64 => ("log", 1, fx.tcx.types.f64),
- sym::log2f32 => ("log2f", 1, fx.tcx.types.f32),
- sym::log2f64 => ("log2", 1, fx.tcx.types.f64),
- sym::log10f32 => ("log10f", 1, fx.tcx.types.f32),
- sym::log10f64 => ("log10", 1, fx.tcx.types.f64),
- sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32),
- sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64),
- sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32),
- sym::fmaf64 => ("fma", 3, fx.tcx.types.f64),
- sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32),
- sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64),
- sym::floorf32 => ("floorf", 1, fx.tcx.types.f32),
- sym::floorf64 => ("floor", 1, fx.tcx.types.f64),
- sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32),
- sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64),
- sym::truncf32 => ("truncf", 1, fx.tcx.types.f32),
- sym::truncf64 => ("trunc", 1, fx.tcx.types.f64),
- sym::roundf32 => ("roundf", 1, fx.tcx.types.f32),
- sym::roundf64 => ("round", 1, fx.tcx.types.f64),
- sym::sinf32 => ("sinf", 1, fx.tcx.types.f32),
- sym::sinf64 => ("sin", 1, fx.tcx.types.f64),
- sym::cosf32 => ("cosf", 1, fx.tcx.types.f32),
- sym::cosf64 => ("cos", 1, fx.tcx.types.f64),
+ let (name, arg_count, ty, clif_ty) = match intrinsic {
+ sym::expf32 => ("expf", 1, fx.tcx.types.f32, types::F32),
+ sym::expf64 => ("exp", 1, fx.tcx.types.f64, types::F64),
+ sym::exp2f32 => ("exp2f", 1, fx.tcx.types.f32, types::F32),
+ sym::exp2f64 => ("exp2", 1, fx.tcx.types.f64, types::F64),
+ sym::sqrtf32 => ("sqrtf", 1, fx.tcx.types.f32, types::F32),
+ sym::sqrtf64 => ("sqrt", 1, fx.tcx.types.f64, types::F64),
+ sym::powif32 => ("__powisf2", 2, fx.tcx.types.f32, types::F32), // compiler-builtins
+ sym::powif64 => ("__powidf2", 2, fx.tcx.types.f64, types::F64), // compiler-builtins
+ sym::powf32 => ("powf", 2, fx.tcx.types.f32, types::F32),
+ sym::powf64 => ("pow", 2, fx.tcx.types.f64, types::F64),
+ sym::logf32 => ("logf", 1, fx.tcx.types.f32, types::F32),
+ sym::logf64 => ("log", 1, fx.tcx.types.f64, types::F64),
+ sym::log2f32 => ("log2f", 1, fx.tcx.types.f32, types::F32),
+ sym::log2f64 => ("log2", 1, fx.tcx.types.f64, types::F64),
+ sym::log10f32 => ("log10f", 1, fx.tcx.types.f32, types::F32),
+ sym::log10f64 => ("log10", 1, fx.tcx.types.f64, types::F64),
+ sym::fabsf32 => ("fabsf", 1, fx.tcx.types.f32, types::F32),
+ sym::fabsf64 => ("fabs", 1, fx.tcx.types.f64, types::F64),
+ sym::fmaf32 => ("fmaf", 3, fx.tcx.types.f32, types::F32),
+ sym::fmaf64 => ("fma", 3, fx.tcx.types.f64, types::F64),
+ sym::copysignf32 => ("copysignf", 2, fx.tcx.types.f32, types::F32),
+ sym::copysignf64 => ("copysign", 2, fx.tcx.types.f64, types::F64),
+ sym::floorf32 => ("floorf", 1, fx.tcx.types.f32, types::F32),
+ sym::floorf64 => ("floor", 1, fx.tcx.types.f64, types::F64),
+ sym::ceilf32 => ("ceilf", 1, fx.tcx.types.f32, types::F32),
+ sym::ceilf64 => ("ceil", 1, fx.tcx.types.f64, types::F64),
+ sym::truncf32 => ("truncf", 1, fx.tcx.types.f32, types::F32),
+ sym::truncf64 => ("trunc", 1, fx.tcx.types.f64, types::F64),
+ sym::rintf32 => ("rintf", 1, fx.tcx.types.f32, types::F32),
+ sym::rintf64 => ("rint", 1, fx.tcx.types.f64, types::F64),
+ sym::roundf32 => ("roundf", 1, fx.tcx.types.f32, types::F32),
+ sym::roundf64 => ("round", 1, fx.tcx.types.f64, types::F64),
+ sym::roundevenf32 => ("roundevenf", 1, fx.tcx.types.f32, types::F32),
+ sym::roundevenf64 => ("roundeven", 1, fx.tcx.types.f64, types::F64),
+ sym::sinf32 => ("sinf", 1, fx.tcx.types.f32, types::F32),
+ sym::sinf64 => ("sin", 1, fx.tcx.types.f64, types::F64),
+ sym::cosf32 => ("cosf", 1, fx.tcx.types.f32, types::F32),
+ sym::cosf64 => ("cos", 1, fx.tcx.types.f64, types::F64),
_ => return false,
};
@@ -297,15 +301,19 @@ fn codegen_float_intrinsic_call<'tcx>(
let (a, b, c);
let args = match args {
[x] => {
- a = [codegen_operand(fx, x)];
+ a = [codegen_operand(fx, x).load_scalar(fx)];
&a as &[_]
}
[x, y] => {
- b = [codegen_operand(fx, x), codegen_operand(fx, y)];
+ b = [codegen_operand(fx, x).load_scalar(fx), codegen_operand(fx, y).load_scalar(fx)];
&b
}
[x, y, z] => {
- c = [codegen_operand(fx, x), codegen_operand(fx, y), codegen_operand(fx, z)];
+ c = [
+ codegen_operand(fx, x).load_scalar(fx),
+ codegen_operand(fx, y).load_scalar(fx),
+ codegen_operand(fx, z).load_scalar(fx),
+ ];
&c
}
_ => unreachable!(),
@@ -314,15 +322,10 @@ fn codegen_float_intrinsic_call<'tcx>(
let layout = fx.layout_of(ty);
let res = match intrinsic {
sym::fmaf32 | sym::fmaf64 => {
- let a = args[0].load_scalar(fx);
- let b = args[1].load_scalar(fx);
- let c = args[2].load_scalar(fx);
- CValue::by_val(fx.bcx.ins().fma(a, b, c), layout)
+ CValue::by_val(fx.bcx.ins().fma(args[0], args[1], args[2]), layout)
}
sym::copysignf32 | sym::copysignf64 => {
- let a = args[0].load_scalar(fx);
- let b = args[1].load_scalar(fx);
- CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout)
+ CValue::by_val(fx.bcx.ins().fcopysign(args[0], args[1]), layout)
}
sym::fabsf32
| sym::fabsf64
@@ -332,21 +335,29 @@ fn codegen_float_intrinsic_call<'tcx>(
| sym::ceilf64
| sym::truncf32
| sym::truncf64 => {
- let a = args[0].load_scalar(fx);
-
let val = match intrinsic {
- sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a),
- sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a),
- sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a),
- sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a),
+ sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(args[0]),
+ sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(args[0]),
+ sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(args[0]),
+ sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(args[0]),
_ => unreachable!(),
};
CValue::by_val(val, layout)
}
+
// These intrinsics aren't supported natively by Cranelift.
// Lower them to a libcall.
- _ => fx.easy_call(name, &args, ty),
+ sym::powif32 | sym::powif64 => {
+ let input_tys: Vec<_> = vec![AbiParam::new(clif_ty), AbiParam::new(types::I32)];
+ let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
+ CValue::by_val(ret_val, fx.layout_of(ty))
+ }
+ _ => {
+ let input_tys: Vec<_> = args.iter().map(|_| AbiParam::new(clif_ty)).collect();
+ let ret_val = fx.lib_call(name, input_tys, vec![AbiParam::new(clif_ty)], &args)[0];
+ CValue::by_val(ret_val, fx.layout_of(ty))
+ }
};
ret.write_cvalue(fx, res);
@@ -381,7 +392,7 @@ fn codegen_regular_intrinsic_call<'tcx>(
fx.bcx.ins().debugtrap();
}
- sym::copy | sym::copy_nonoverlapping => {
+ sym::copy => {
intrinsic_args!(fx, args => (src, dst, count); intrinsic);
let src = src.load_scalar(fx);
let dst = dst.load_scalar(fx);
@@ -393,13 +404,8 @@ fn codegen_regular_intrinsic_call<'tcx>(
let byte_amount =
if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
- if intrinsic == sym::copy_nonoverlapping {
- // FIXME emit_small_memcpy
- fx.bcx.call_memcpy(fx.target_config, dst, src, byte_amount);
- } else {
- // FIXME emit_small_memmove
- fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
- }
+ // FIXME emit_small_memmove
+ fx.bcx.call_memmove(fx.target_config, dst, src, byte_amount);
}
sym::volatile_copy_memory | sym::volatile_copy_nonoverlapping_memory => {
// NOTE: the volatile variants have src and dst swapped
@@ -551,16 +557,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
fx.bcx.ins().band(ptr, mask);
}
- sym::transmute => {
- intrinsic_args!(fx, args => (from); intrinsic);
-
- if ret.layout().abi.is_uninhabited() {
- crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
- return;
- }
-
- ret.write_cvalue_transmute(fx, from);
- }
sym::write_bytes | sym::volatile_set_memory => {
intrinsic_args!(fx, args => (dst, val, count); intrinsic);
let val = val.load_scalar(fx);
@@ -639,26 +635,25 @@ fn codegen_regular_intrinsic_call<'tcx>(
if do_panic {
let layout = fx.layout_of(ty);
-
- with_no_trimmed_paths!({
- crate::base::codegen_panic_nounwind(
- fx,
- &if layout.abi.is_uninhabited() {
- format!("attempted to instantiate uninhabited type `{}`", layout.ty)
- } else if requirement == ValidityRequirement::Zero {
+ let msg_str = with_no_visible_paths!({
+ with_no_trimmed_paths!({
+ if layout.abi.is_uninhabited() {
+ // Use this error even for the other intrinsics as it is more precise.
+ format!("attempted to instantiate uninhabited type `{}`", ty)
+ } else if intrinsic == sym::assert_zero_valid {
format!(
"attempted to zero-initialize type `{}`, which is invalid",
- layout.ty
+ ty
)
} else {
format!(
"attempted to leave type `{}` uninitialized, which is invalid",
- layout.ty
+ ty
)
- },
- source_info,
- )
+ }
+ })
});
+ crate::base::codegen_panic_nounwind(fx, &msg_str, source_info);
return;
}
}
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index a1d63acfb..6f54a8d49 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -253,7 +253,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
}
ret.write_cvalue(fx, base);
- let ret_lane = ret.place_field(fx, mir::Field::new(idx.try_into().unwrap()));
+ let ret_lane = ret.place_field(fx, FieldIdx::new(idx.try_into().unwrap()));
ret_lane.write_cvalue(fx, val);
}
@@ -279,9 +279,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
let trap_block = fx.bcx.create_block();
let true_ = fx.bcx.ins().iconst(types::I8, 1);
- fx.bcx.ins().brnz(true_, trap_block, &[]);
let ret_block = fx.get_block(target);
- fx.bcx.ins().jump(ret_block, &[]);
+ fx.bcx.ins().brif(true_, trap_block, &[], ret_block, &[]);
fx.bcx.switch_to_block(trap_block);
crate::trap::trap_unimplemented(
fx,
@@ -825,8 +824,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let next = fx.bcx.create_block();
let res_lane = fx.bcx.append_block_param(next, lane_clif_ty);
- fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
- fx.bcx.ins().jump(if_disabled, &[]);
+ fx.bcx.ins().brif(mask_lane, if_enabled, &[], if_disabled, &[]);
fx.bcx.seal_block(if_enabled);
fx.bcx.seal_block(if_disabled);
@@ -864,8 +862,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let if_enabled = fx.bcx.create_block();
let next = fx.bcx.create_block();
- fx.bcx.ins().brnz(mask_lane, if_enabled, &[]);
- fx.bcx.ins().jump(next, &[]);
+ fx.bcx.ins().brif(mask_lane, if_enabled, &[], next, &[]);
fx.bcx.seal_block(if_enabled);
fx.bcx.switch_to_block(if_enabled);