summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs166
1 files changed, 145 insertions, 21 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index 30e3d1125..51fce8c85 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -2,6 +2,7 @@
use rustc_middle::ty::subst::SubstsRef;
use rustc_span::Symbol;
+use rustc_target::abi::Endian;
use super::*;
use crate::prelude::*;
@@ -14,7 +15,7 @@ fn report_simd_type_validation_error(
) {
fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
// Prevent verifier error
- crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
pub(super) fn codegen_simd_intrinsic_call<'tcx>(
@@ -26,7 +27,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
span: Span,
) {
match intrinsic {
- sym::simd_cast => {
+ sym::simd_as | sym::simd_cast => {
intrinsic_args!(fx, args => (a); intrinsic);
if !a.layout().ty.is_simd() {
@@ -157,11 +158,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
),
);
// Prevent verifier error
- crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
return;
}
}
} else {
+ // FIXME remove this case
intrinsic.as_str()["simd_shuffle".len()..].parse().unwrap()
};
@@ -186,7 +188,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let size = Size::from_bytes(
4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */
);
- alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
+ alloc
+ .inner()
+ .get_bytes_strip_provenance(fx, alloc_range(offset, size))
+ .unwrap()
}
_ => unreachable!("{:?}", idx_const),
};
@@ -274,12 +279,17 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
idx_const
} else {
fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
- let res = crate::trap::trap_unimplemented_ret_value(
+ let trap_block = fx.bcx.create_block();
+ let dummy_block = fx.bcx.create_block();
+ let true_ = fx.bcx.ins().iconst(types::I8, 1);
+ fx.bcx.ins().brnz(true_, trap_block, &[]);
+ fx.bcx.ins().jump(dummy_block, &[]);
+ fx.bcx.switch_to_block(trap_block);
+ crate::trap::trap_unimplemented(
fx,
- ret.layout(),
"Index argument for `simd_extract` is not a constant",
);
- ret.write_cvalue(fx, res);
+ fx.bcx.switch_to_block(dummy_block);
return;
};
@@ -392,21 +402,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let layout = a.layout();
let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let res_lane_layout = fx.layout_of(lane_ty);
for lane in 0..lane_count {
- let a_lane = a.value_lane(fx, lane);
- let b_lane = b.value_lane(fx, lane);
- let c_lane = c.value_lane(fx, lane);
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+ let b_lane = b.value_lane(fx, lane).load_scalar(fx);
+ let c_lane = c.value_lane(fx, lane).load_scalar(fx);
- let res_lane = match lane_ty.kind() {
- ty::Float(FloatTy::F32) => {
- fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty)
- }
- ty::Float(FloatTy::F64) => {
- fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty)
- }
- _ => unreachable!(),
- };
+ let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane);
+ let res_lane = CValue::by_val(res_lane, res_lane_layout);
ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
}
@@ -648,8 +652,128 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
}
}
- // simd_saturating_*
- // simd_bitmask
+ sym::simd_select_bitmask => {
+ intrinsic_args!(fx, args => (m, a, b); intrinsic);
+
+ if !a.layout().ty.is_simd() {
+ report_simd_type_validation_error(fx, intrinsic, span, a.layout().ty);
+ return;
+ }
+ assert_eq!(a.layout(), b.layout());
+
+ let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_layout = fx.layout_of(lane_ty);
+
+ let m = m.load_scalar(fx);
+
+ for lane in 0..lane_count {
+ let m_lane = fx.bcx.ins().ushr_imm(m, u64::from(lane) as i64);
+ let m_lane = fx.bcx.ins().band_imm(m_lane, 1);
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+ let b_lane = b.value_lane(fx, lane).load_scalar(fx);
+
+ let m_lane = fx.bcx.ins().icmp_imm(IntCC::Equal, m_lane, 0);
+ let res_lane =
+ CValue::by_val(fx.bcx.ins().select(m_lane, b_lane, a_lane), lane_layout);
+
+ ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
+ }
+ }
+
+ sym::simd_bitmask => {
+ intrinsic_args!(fx, args => (a); intrinsic);
+
+ let (lane_count, lane_ty) = a.layout().ty.simd_size_and_type(fx.tcx);
+ let lane_clif_ty = fx.clif_type(lane_ty).unwrap();
+
+ // The `fn simd_bitmask(vector) -> unsigned integer` intrinsic takes a
+ // vector mask and returns the most significant bit (MSB) of each lane in the form
+ // of either:
+ // * an unsigned integer
+ // * an array of `u8`
+ // If the vector has less than 8 lanes, a u8 is returned with zeroed trailing bits.
+ //
+ // The bit order of the result depends on the byte endianness, LSB-first for little
+ // endian and MSB-first for big endian.
+ let expected_int_bits = lane_count.max(8);
+ let expected_bytes = expected_int_bits / 8 + ((expected_int_bits % 8 > 0) as u64);
+
+ match lane_ty.kind() {
+ ty::Int(_) | ty::Uint(_) => {}
+ _ => {
+ fx.tcx.sess.span_fatal(
+ span,
+ &format!(
+ "invalid monomorphization of `simd_bitmask` intrinsic: \
+ vector argument `{}`'s element type `{}`, expected integer element \
+ type",
+ a.layout().ty,
+ lane_ty
+ ),
+ );
+ }
+ }
+
+ let res_type =
+ Type::int_with_byte_size(u16::try_from(expected_bytes).unwrap()).unwrap();
+ let mut res = fx.bcx.ins().iconst(res_type, 0);
+
+ let lanes = match fx.tcx.sess.target.endian {
+ Endian::Big => Box::new(0..lane_count) as Box<dyn Iterator<Item = u64>>,
+ Endian::Little => Box::new((0..lane_count).rev()) as Box<dyn Iterator<Item = u64>>,
+ };
+ for lane in lanes {
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+
+ // extract sign bit of an int
+ let a_lane_sign = fx.bcx.ins().ushr_imm(a_lane, i64::from(lane_clif_ty.bits() - 1));
+
+ // shift sign bit into result
+ let a_lane_sign = clif_intcast(fx, a_lane_sign, res_type, false);
+ res = fx.bcx.ins().ishl_imm(res, 1);
+ res = fx.bcx.ins().bor(res, a_lane_sign);
+ }
+
+ match ret.layout().ty.kind() {
+ ty::Uint(i) if i.bit_width() == Some(expected_int_bits) => {}
+ ty::Array(elem, len)
+ if matches!(elem.kind(), ty::Uint(ty::UintTy::U8))
+ && len.try_eval_usize(fx.tcx, ty::ParamEnv::reveal_all())
+ == Some(expected_bytes) => {}
+ _ => {
+ fx.tcx.sess.span_fatal(
+ span,
+ &format!(
+ "invalid monomorphization of `simd_bitmask` intrinsic: \
+ cannot return `{}`, expected `u{}` or `[u8; {}]`",
+ ret.layout().ty,
+ expected_int_bits,
+ expected_bytes
+ ),
+ );
+ }
+ }
+
+ let res = CValue::by_val(res, ret.layout());
+ ret.write_cvalue(fx, res);
+ }
+
+ sym::simd_saturating_add | sym::simd_saturating_sub => {
+ intrinsic_args!(fx, args => (x, y); intrinsic);
+
+ let bin_op = match intrinsic {
+ sym::simd_saturating_add => BinOp::Add,
+ sym::simd_saturating_sub => BinOp::Sub,
+ _ => unreachable!(),
+ };
+
+ // FIXME use vector instructions when possible
+ simd_pair_for_each_lane_typed(fx, x, y, ret, &|fx, x_lane, y_lane| {
+ crate::num::codegen_saturating_int_binop(fx, bin_op, x_lane, y_lane)
+ });
+ }
+
+ // simd_arith_offset
// simd_scatter
// simd_gather
_ => {
generated by cgit v1.2.3 (git 2.39.1) at 2024-07-09 15:46:03 +0000