diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift')
19 files changed, 163 insertions, 421 deletions
diff --git a/compiler/rustc_codegen_cranelift/docs/usage.md b/compiler/rustc_codegen_cranelift/docs/usage.md index c6210f958..135a51ce3 100644 --- a/compiler/rustc_codegen_cranelift/docs/usage.md +++ b/compiler/rustc_codegen_cranelift/docs/usage.md @@ -54,7 +54,7 @@ These are a few functions that allow you to easily run rust code from the shell ```bash function jit_naked() { - echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-features -Cllvm-args=mode=jit -Cprefer-dynamic + echo "$@" | $cg_clif_dir/dist/rustc-clif - -Zunstable-options -Cllvm-args=mode=jit-lazy -Cprefer-dynamic } function jit() { diff --git a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml index fa175edca..5b79d6569 100644 --- a/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml +++ b/compiler/rustc_codegen_cranelift/patches/stdlib-lock.toml @@ -4,9 +4,9 @@ version = 3 [[package]] name = "addr2line" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4fa78e18c64fce05e902adecd7a5eed15a5e0a3439f7b0e169f0252214865e3" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" dependencies = [ "compiler_builtins", "gimli", @@ -140,9 +140,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.27.2" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad0a93d233ebf96623465aad4046a8d3aa4da22d4f4beba5388838c8a434bbb4" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" dependencies = [ "compiler_builtins", "rustc-std-workspace-alloc", @@ -205,9 +205,9 @@ dependencies = [ [[package]] name = "object" -version = "0.31.1" +version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bda667d9f2b5051b8833f59f3bf748b28ef54f850f4fcb389a252aa383866d1" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" dependencies = [ "compiler_builtins", "memchr", diff --git a/compiler/rustc_codegen_cranelift/rust-toolchain b/compiler/rustc_codegen_cranelift/rust-toolchain index 5689bdee6..2cc5d7777 100644 --- a/compiler/rustc_codegen_cranelift/rust-toolchain +++ b/compiler/rustc_codegen_cranelift/rust-toolchain @@ -1,3 +1,3 @@ [toolchain] -channel = "nightly-2023-08-08" +channel = "nightly-2023-09-06" components = ["rust-src", "rustc-dev", "llvm-tools"] diff --git a/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs b/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs index f782671fe..03912b18e 100755 --- a/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs +++ b/compiler/rustc_codegen_cranelift/scripts/filter_profile.rs @@ -100,9 +100,9 @@ fn main() -> Result<(), Box<dyn std::error::Error>> { stack = &stack[..index + ENCODE_METADATA.len()]; } - const SUBST_AND_NORMALIZE_ERASING_REGIONS: &str = "rustc_middle::ty::normalize_erasing_regions::<impl rustc_middle::ty::context::TyCtxt>::subst_and_normalize_erasing_regions"; - if let Some(index) = stack.find(SUBST_AND_NORMALIZE_ERASING_REGIONS) { - stack = &stack[..index + SUBST_AND_NORMALIZE_ERASING_REGIONS.len()]; + const INSTANTIATE_AND_NORMALIZE_ERASING_REGIONS: &str = "rustc_middle::ty::normalize_erasing_regions::<impl rustc_middle::ty::context::TyCtxt>::instantiate_and_normalize_erasing_regions"; + if let Some(index) = stack.find(INSTANTIATE_AND_NORMALIZE_ERASING_REGIONS) { + stack = &stack[..index + INSTANTIATE_AND_NORMALIZE_ERASING_REGIONS.len()]; } const NORMALIZE_ERASING_LATE_BOUND_REGIONS: &str = "rustc_middle::ty::normalize_erasing_regions::<impl rustc_middle::ty::context::TyCtxt>::normalize_erasing_late_bound_regions"; diff --git a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh index c163b8543..3fc462a39 100755 --- a/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh +++ b/compiler/rustc_codegen_cranelift/scripts/test_rustc_tests.sh @@ -45,6 +45,7 @@ rm tests/ui/proc-macro/quote-debug.rs rm tests/ui/proc-macro/no-missing-docs.rs rm tests/ui/rust-2018/proc-macro-crate-in-paths.rs rm tests/ui/proc-macro/allowed-signatures.rs +rm tests/ui/proc-macro/no-mangle-in-proc-macro-issue-111888.rs # vendor intrinsics rm tests/ui/sse2.rs # cpuid not supported, so sse2 not detected @@ -114,6 +115,7 @@ rm tests/ui/mir/mir_misc_casts.rs # depends on deduplication of constants rm tests/ui/mir/mir_raw_fat_ptr.rs # same rm tests/ui/consts/issue-33537.rs # same rm tests/ui/layout/valid_range_oob.rs # different ICE message +rm tests/ui/const-generics/generic_const_exprs/issue-80742.rs # gives error instead of ICE with cg_clif rm tests/ui/consts/issue-miri-1910.rs # different error message rm tests/ui/consts/offset_ub.rs # same diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index b7f56a298..5d775b9b5 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -39,7 +39,7 @@ fn clif_sig_from_fn_abi<'tcx>( pub(crate) fn conv_to_call_conv(sess: &Session, c: Conv, default_call_conv: CallConv) -> CallConv { match c { Conv::Rust | Conv::C => default_call_conv, - Conv::RustCold => CallConv::Cold, + Conv::Cold | Conv::PreserveMost | Conv::PreserveAll => CallConv::Cold, Conv::X86_64SysV => CallConv::SystemV, Conv::X86_64Win64 => CallConv::WindowsFastcall, diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs index d847e524f..0d16da480 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs @@ -100,11 +100,11 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(ref cast, pad_i32) => { + PassMode::Cast { ref cast, pad_i32 } => { assert!(!pad_i32, "padding support not yet implemented"); cast_target_to_abi_params(cast) } - PassMode::Indirect { attrs, extra_attrs: None, on_stack } => { + PassMode::Indirect { attrs, meta_attrs: None, on_stack } => { if on_stack { // Abi requires aligning struct size to pointer size let size = self.layout.size.align_to(tcx.data_layout.pointer_align.abi); @@ -117,11 +117,11 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { smallvec![apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs)] } } - PassMode::Indirect { attrs, extra_attrs: Some(extra_attrs), on_stack } => { + PassMode::Indirect { attrs, meta_attrs: Some(meta_attrs), on_stack } => { assert!(!on_stack); smallvec![ apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), attrs), - apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), extra_attrs), + apply_arg_attrs_to_abi_param(AbiParam::new(pointer_ty(tcx)), meta_attrs), ] } } @@ -148,14 +148,14 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(ref cast, _) => { + PassMode::Cast { ref cast, .. } => { (None, cast_target_to_abi_params(cast).into_iter().collect()) } - PassMode::Indirect { attrs: _, extra_attrs: None, on_stack } => { + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack } => { assert!(!on_stack); (Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![]) } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { unreachable!("unsized return value") } } @@ -229,7 +229,7 @@ pub(super) fn adjust_arg_for_abi<'tcx>( let (a, b) = arg.load_scalar_pair(fx); smallvec![a, b] } - PassMode::Cast(ref cast, _) => to_casted_value(fx, arg, cast), + PassMode::Cast { ref cast, .. } => to_casted_value(fx, arg, cast), PassMode::Indirect { .. } => { if is_owned { match arg.force_stack(fx) { @@ -287,14 +287,14 @@ pub(super) fn cvalue_for_param<'tcx>( assert_eq!(block_params.len(), 2, "{:?}", block_params); Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout)) } - PassMode::Cast(ref cast, _) => { + PassMode::Cast { ref cast, .. } => { Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) } - PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { assert_eq!(block_params.len(), 1, "{:?}", block_params); Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { assert_eq!(block_params.len(), 2, "{:?}", block_params); Some(CValue::by_ref_unsized( Pointer::new(block_params[0]), diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs index 14e54d5ee..646fb4a3c 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs @@ -13,7 +13,7 @@ pub(super) fn codegen_return_param<'tcx>( block_params_iter: &mut impl Iterator<Item = Value>, ) -> CPlace<'tcx> { let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode { - PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => { + PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => { let is_ssa = ssa_analyzed[RETURN_PLACE].is_ssa(fx, fx.fn_abi.as_ref().unwrap().ret.layout.ty); ( @@ -26,7 +26,7 @@ pub(super) fn codegen_return_param<'tcx>( smallvec![], ) } - PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { let ret_param = block_params_iter.next().unwrap(); assert_eq!(fx.bcx.func.dfg.value_type(ret_param), fx.pointer_type); ( @@ -34,7 +34,7 @@ pub(super) fn codegen_return_param<'tcx>( smallvec![ret_param], ) } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { unreachable!("unsized return value") } }; @@ -62,7 +62,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( ) { let (ret_temp_place, return_ptr) = match ret_arg_abi.mode { PassMode::Ignore => (None, None), - PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { if let Some(ret_ptr) = ret_place.try_to_ptr() { // This is an optimization to prevent unnecessary copies of the return value when // the return place is already a memory place as opposed to a register. @@ -73,10 +73,10 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( (Some(place), Some(place.to_ptr().get_addr(fx))) } } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { unreachable!("unsized return value") } - PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => (None, None), + PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast { .. } => (None, None), }; let call_inst = f(fx, return_ptr); @@ -93,21 +93,21 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( ret_place .write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout)); } - PassMode::Cast(ref cast, _) => { + PassMode::Cast { ref cast, .. } => { let results = fx.bcx.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>(); let result = super::pass_mode::from_casted_value(fx, &results, ret_place.layout(), cast); ret_place.write_cvalue(fx, result); } - PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { if let Some(ret_temp_place) = ret_temp_place { // If ret_temp_place is None, it is not necessary to copy the return value. let ret_temp_value = ret_temp_place.to_cvalue(fx); ret_place.write_cvalue(fx, ret_temp_value); } } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { unreachable!("unsized return value") } } @@ -116,10 +116,10 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( /// Codegen a return instruction with the right return value(s) if any. pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) { match fx.fn_abi.as_ref().unwrap().ret.mode { - PassMode::Ignore | PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { + PassMode::Ignore | PassMode::Indirect { attrs: _, meta_attrs: None, on_stack: _ } => { fx.bcx.ins().return_(&[]); } - PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { + PassMode::Indirect { attrs: _, meta_attrs: Some(_), on_stack: _ } => { unreachable!("unsized return value") } PassMode::Direct(_) => { @@ -132,7 +132,7 @@ pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) { let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); } - PassMode::Cast(ref cast, _) => { + PassMode::Cast { ref cast, .. } => { let place = fx.get_local_place(RETURN_PLACE); let ret_val = place.to_cvalue(fx); let ret_vals = super::pass_mode::to_casted_value(fx, ret_val, cast); diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 522dd7189..0a451dad9 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -250,7 +250,10 @@ pub(crate) fn verify_func( } fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { - if !crate::constant::check_constants(fx) { + if let Err(err) = + fx.mir.post_mono_checks(fx.tcx, ty::ParamEnv::reveal_all(), |c| Ok(fx.monomorphize(c))) + { + err.emit_err(fx.tcx); fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); // compilation should have been aborted @@ -474,10 +477,10 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { *destination, ); } - TerminatorKind::Terminate => { - codegen_panic_cannot_unwind(fx, source_info); + TerminatorKind::UnwindTerminate(reason) => { + codegen_unwind_terminate(fx, source_info, *reason); } - TerminatorKind::Resume => { + TerminatorKind::UnwindResume => { // FIXME implement unwinding fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); } @@ -723,11 +726,8 @@ fn codegen_stmt<'tcx>( } Rvalue::Repeat(ref operand, times) => { let operand = codegen_operand(fx, operand); - let times = fx - .monomorphize(times) - .eval(fx.tcx, ParamEnv::reveal_all()) - .try_to_bits(fx.tcx.data_layout.pointer_size) - .unwrap(); + let times = + fx.monomorphize(times).eval_target_usize(fx.tcx, ParamEnv::reveal_all()); if operand.layout().size.bytes() == 0 { // Do nothing for ZST's } else if fx.clif_type(operand.layout().ty) == Some(types::I8) { @@ -875,7 +875,8 @@ pub(crate) fn codegen_place<'tcx>( PlaceElem::Deref => { cplace = cplace.place_deref(fx); } - PlaceElem::OpaqueCast(ty) => cplace = cplace.place_opaque_cast(fx, ty), + PlaceElem::OpaqueCast(ty) => bug!("encountered OpaqueCast({ty}) in codegen"), + PlaceElem::Subtype(ty) => cplace = cplace.place_transmute_type(fx, fx.monomorphize(ty)), PlaceElem::Field(field, _ty) => { cplace = cplace.place_field(fx, field); } @@ -971,13 +972,14 @@ pub(crate) fn codegen_panic_nounwind<'tcx>( codegen_panic_inner(fx, rustc_hir::LangItem::PanicNounwind, &args, source_info.span); } -pub(crate) fn codegen_panic_cannot_unwind<'tcx>( +pub(crate) fn codegen_unwind_terminate<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, source_info: mir::SourceInfo, + reason: UnwindTerminateReason, ) { let args = []; - codegen_panic_inner(fx, rustc_hir::LangItem::PanicCannotUnwind, &args, source_info.span); + codegen_panic_inner(fx, reason.lang_item(), &args, source_info.span); } fn codegen_panic_inner<'tcx>( diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index 3081dcfa2..359b430b4 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -359,7 +359,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { where T: TypeFoldable<TyCtxt<'tcx>> + Copy, { - self.instance.subst_mir_and_normalize_erasing_regions( + self.instance.instantiate_mir_and_normalize_erasing_regions( self.tcx, ty::ParamEnv::reveal_all(), ty::EarlyBinder::bind(value), @@ -480,7 +480,7 @@ impl<'tcx> LayoutOfHelpers<'tcx> for RevealAllLayoutCx<'tcx> { if let LayoutError::SizeOverflow(_) | LayoutError::ReferencesError(_) = err { self.0.sess.span_fatal(span, err.to_string()) } else { - span_bug!(span, "failed to get layout for `{}`: {}", ty, err) + self.0.sess.span_fatal(span, format!("failed to get layout for `{}`: {}", ty, err)) } } } diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index c31535742..14b10ed8b 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -2,9 +2,8 @@ use rustc_data_structures::fx::{FxHashMap, FxHashSet}; use rustc_middle::middle::codegen_fn_attrs::CodegenFnAttrFlags; -use rustc_middle::mir::interpret::{ - read_target_uint, AllocId, ConstAllocation, ConstValue, ErrorHandled, GlobalAlloc, Scalar, -}; +use rustc_middle::mir::interpret::{read_target_uint, AllocId, GlobalAlloc, Scalar}; +use rustc_middle::mir::ConstValue; use cranelift_module::*; @@ -33,16 +32,6 @@ impl ConstantCx { } } -pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool { - let mut all_constants_ok = true; - for constant in &fx.mir.required_consts { - if eval_mir_constant(fx, constant).is_none() { - all_constants_ok = false; - } - } - all_constants_ok -} - pub(crate) fn codegen_static(tcx: TyCtxt<'_>, module: &mut dyn Module, def_id: DefId) { let mut constants_cx = ConstantCx::new(); constants_cx.todo.push(TodoItem::Static(def_id)); @@ -75,53 +64,21 @@ pub(crate) fn codegen_tls_ref<'tcx>( pub(crate) fn eval_mir_constant<'tcx>( fx: &FunctionCx<'_, '_, 'tcx>, - constant: &Constant<'tcx>, -) -> Option<(ConstValue<'tcx>, Ty<'tcx>)> { - let constant_kind = fx.monomorphize(constant.literal); - let uv = match constant_kind { - ConstantKind::Ty(const_) => match const_.kind() { - ty::ConstKind::Unevaluated(uv) => uv.expand(), - ty::ConstKind::Value(val) => { - return Some((fx.tcx.valtree_to_const_val((const_.ty(), val)), const_.ty())); - } - err => span_bug!( - constant.span, - "encountered bad ConstKind after monomorphizing: {:?}", - err - ), - }, - ConstantKind::Unevaluated(mir::UnevaluatedConst { def, .. }, _) - if fx.tcx.is_static(def) => - { - span_bug!(constant.span, "MIR constant refers to static"); - } - ConstantKind::Unevaluated(uv, _) => uv, - ConstantKind::Val(val, _) => return Some((val, constant_kind.ty())), - }; - - let val = fx - .tcx - .const_eval_resolve(ty::ParamEnv::reveal_all(), uv, None) - .map_err(|err| match err { - ErrorHandled::Reported(_) => { - fx.tcx.sess.span_err(constant.span, "erroneous constant encountered"); - } - ErrorHandled::TooGeneric => { - span_bug!(constant.span, "codegen encountered polymorphic constant: {:?}", err); - } - }) - .ok(); - val.map(|val| (val, constant_kind.ty())) + constant: &ConstOperand<'tcx>, +) -> (ConstValue<'tcx>, Ty<'tcx>) { + let cv = fx.monomorphize(constant.const_); + // This cannot fail because we checked all required_consts in advance. + let val = cv + .eval(fx.tcx, ty::ParamEnv::reveal_all(), Some(constant.span)) + .expect("erroneous constant not captured by required_consts"); + (val, cv.ty()) } pub(crate) fn codegen_constant_operand<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, - constant: &Constant<'tcx>, + constant: &ConstOperand<'tcx>, ) -> CValue<'tcx> { - let (const_val, ty) = eval_mir_constant(fx, constant).unwrap_or_else(|| { - span_bug!(constant.span, "erroneous constant not captured by required_consts") - }); - + let (const_val, ty) = eval_mir_constant(fx, constant); codegen_const_value(fx, const_val, ty) } @@ -138,7 +95,7 @@ pub(crate) fn codegen_const_value<'tcx>( } match const_val { - ConstValue::ZeroSized => unreachable!(), // we already handles ZST above + ConstValue::ZeroSized => unreachable!(), // we already handled ZST above ConstValue::Scalar(x) => match x { Scalar::Int(int) => { if fx.clif_type(layout.ty).is_some() { @@ -222,19 +179,16 @@ pub(crate) fn codegen_const_value<'tcx>( CValue::by_val(val, layout) } }, - ConstValue::ByRef { alloc, offset } => CValue::by_ref( - pointer_for_allocation(fx, alloc) + ConstValue::Indirect { alloc_id, offset } => CValue::by_ref( + pointer_for_allocation(fx, alloc_id) .offset_i64(fx, i64::try_from(offset.bytes()).unwrap()), layout, ), - ConstValue::Slice { data, start, end } => { - let ptr = pointer_for_allocation(fx, data) - .offset_i64(fx, i64::try_from(start).unwrap()) - .get_addr(fx); - let len = fx - .bcx - .ins() - .iconst(fx.pointer_type, i64::try_from(end.checked_sub(start).unwrap()).unwrap()); + ConstValue::Slice { data, meta } => { + let alloc_id = fx.tcx.reserve_and_set_memory_alloc(data); + let ptr = pointer_for_allocation(fx, alloc_id).get_addr(fx); + // FIXME: the `try_from` here can actually fail, e.g. for very long ZST slices. + let len = fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(meta).unwrap()); CValue::by_val_pair(ptr, len, layout) } } @@ -242,9 +196,9 @@ pub(crate) fn codegen_const_value<'tcx>( fn pointer_for_allocation<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, - alloc: ConstAllocation<'tcx>, + alloc_id: AllocId, ) -> crate::pointer::Pointer { - let alloc_id = fx.tcx.create_memory_alloc(alloc); + let alloc = fx.tcx.global_alloc(alloc_id).unwrap_memory(); let data_id = data_id_for_alloc_id( &mut fx.constants_cx, &mut *fx.module, @@ -375,6 +329,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant unreachable!() } }; + // FIXME: should we have a cache so we don't do this multiple times for the same `ConstAllocation`? let data_id = *cx.anon_allocs.entry(alloc_id).or_insert_with(|| { module.declare_anonymous_data(alloc.inner().mutability.is_mut(), false).unwrap() }); @@ -479,7 +434,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( operand: &Operand<'tcx>, ) -> Option<ConstValue<'tcx>> { match operand { - Operand::Constant(const_) => Some(eval_mir_constant(fx, const_).unwrap().0), + Operand::Constant(const_) => Some(eval_mir_constant(fx, const_).0), // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored // inside a temporary before being passed to the intrinsic requiring the const argument. // This code tries to find a single constant defining definition of the referenced local. @@ -550,8 +505,8 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( match &bb_data.terminator().kind { TerminatorKind::Goto { .. } | TerminatorKind::SwitchInt { .. } - | TerminatorKind::Resume - | TerminatorKind::Terminate + | TerminatorKind::UnwindResume + | TerminatorKind::UnwindTerminate(_) | TerminatorKind::Return | TerminatorKind::Unreachable | TerminatorKind::Drop { .. } diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs index 50bc7a127..b19b935a0 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs @@ -81,13 +81,10 @@ impl DebugContext { match tcx.sess.source_map().lookup_line(span.lo()) { Ok(SourceFileAndLine { sf: file, line }) => { - let line_pos = file.lines(|lines| lines[line]); + let line_pos = file.lines()[line]; + let col = file.relative_position(span.lo()) - line_pos; - ( - file, - u64::try_from(line).unwrap() + 1, - u64::from((span.lo() - line_pos).to_u32()) + 1, - ) + (file, u64::try_from(line).unwrap() + 1, u64::from(col.to_u32()) + 1) } Err(file) => (file, 0, 0), } diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index d143bcc96..3e9383095 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -269,7 +269,7 @@ fn module_codegen( ), ) -> OngoingModuleCodegen { let (cgu_name, mut cx, mut module, codegened_functions) = - tcx.prof.verbose_generic_activity_with_arg("codegen cgu", cgu_name.as_str()).run(|| { + tcx.prof.generic_activity_with_arg("codegen cgu", cgu_name.as_str()).run(|| { let cgu = tcx.codegen_unit(cgu_name); let mono_items = cgu.items_in_deterministic_order(tcx); @@ -322,35 +322,24 @@ fn module_codegen( }); OngoingModuleCodegen::Async(std::thread::spawn(move || { - cx.profiler.clone().verbose_generic_activity_with_arg("compile functions", &*cgu_name).run( - || { - cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( - cx.profiler.clone(), - ))); - - let mut cached_context = Context::new(); - for codegened_func in codegened_functions { - crate::base::compile_fn( - &mut cx, - &mut cached_context, - &mut module, - codegened_func, - ); - } - }, - ); + cx.profiler.clone().generic_activity_with_arg("compile functions", &*cgu_name).run(|| { + cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( + cx.profiler.clone(), + ))); + + let mut cached_context = Context::new(); + for codegened_func in codegened_functions { + crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func); + } + }); - let global_asm_object_file = cx - .profiler - .verbose_generic_activity_with_arg("compile assembly", &*cgu_name) - .run(|| { + let global_asm_object_file = + cx.profiler.generic_activity_with_arg("compile assembly", &*cgu_name).run(|| { crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm) })?; - let codegen_result = cx - .profiler - .verbose_generic_activity_with_arg("write object file", &*cgu_name) - .run(|| { + let codegen_result = + cx.profiler.generic_activity_with_arg("write object file", &*cgu_name).run(|| { emit_cgu( &global_asm_config.output_filenames, &cx.profiler, diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index 518e3da07..50bbf8105 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -242,8 +242,7 @@ pub(crate) fn codegen_inline_asm<'tcx>( } } InlineAsmOperand::Const { ref value } => { - let (const_value, ty) = crate::constant::eval_mir_constant(fx, value) - .unwrap_or_else(|| span_bug!(span, "asm const cannot be resolved")); + let (const_value, ty) = crate::constant::eval_mir_constant(fx, value); let value = rustc_codegen_ssa::common::asm_const_to_str( fx.tcx, span, @@ -253,8 +252,8 @@ pub(crate) fn codegen_inline_asm<'tcx>( CInlineAsmOperand::Const { value } } InlineAsmOperand::SymFn { ref value } => { - let literal = fx.monomorphize(value.literal); - if let ty::FnDef(def_id, args) = *literal.ty().kind() { + let const_ = fx.monomorphize(value.const_); + if let ty::FnDef(def_id, args) = *const_.ty().kind() { let instance = ty::Instance::resolve_for_fn_ptr( fx.tcx, ty::ParamEnv::reveal_all(), diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs index fdd27a454..e62de6b61 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs @@ -177,244 +177,6 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( bool_to_zero_or_max_uint(fx, res_lane_ty, res_lane) }); } - "llvm.x86.sse2.psrli.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.psrli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.psrai.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.psrai.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.pslli.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.pslli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.psrli.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.psrli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.psrai.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.psrai.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.pslli.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.sse2.pslli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx.psrli.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx.psrai.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrai.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.psrli.q" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrli.q imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 64 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.sse2.pslli.q" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.pslli.q imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 64 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx.pslli.d" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.pslli.d imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 32 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx2.psrli.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrli.w imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().ushr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx2.psrai.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.psrai.w imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().sshr_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } - "llvm.x86.avx2.pslli.w" => { - let (a, imm8) = match args { - [a, imm8] => (a, imm8), - _ => bug!("wrong number of args for intrinsic {intrinsic}"), - }; - let a = codegen_operand(fx, a); - let imm8 = crate::constant::mir_operand_get_const_val(fx, imm8) - .expect("llvm.x86.avx.pslli.w imm8 not const"); - - simd_for_each_lane(fx, a, ret, &|fx, _lane_ty, _res_lane_ty, lane| match imm8 - .try_to_bits(Size::from_bytes(4)) - .unwrap_or_else(|| panic!("imm8 not scalar: {:?}", imm8)) - { - imm8 if imm8 < 16 => fx.bcx.ins().ishl_imm(lane, i64::from(imm8 as u8)), - _ => fx.bcx.ins().iconst(types::I32, 0), - }); - } "llvm.x86.ssse3.pshuf.b.128" | "llvm.x86.avx2.pshuf.b" => { let (a, b) = match args { [a, b] => (a, b), @@ -506,14 +268,6 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( ret.place_lane(fx, 2).to_ptr().store(fx, res_2, MemFlags::trusted()); ret.place_lane(fx, 3).to_ptr().store(fx, res_3, MemFlags::trusted()); } - "llvm.x86.sse2.storeu.dq" | "llvm.x86.sse2.storeu.pd" => { - intrinsic_args!(fx, args => (mem_addr, a); intrinsic); - let mem_addr = mem_addr.load_scalar(fx); - - // FIXME correctly handle the unalignment - let dest = CPlace::for_ptr(Pointer::new(mem_addr), a.layout()); - dest.write_cvalue(fx, a); - } "llvm.x86.ssse3.pabs.b.128" | "llvm.x86.ssse3.pabs.w.128" | "llvm.x86.ssse3.pabs.d.128" => { let a = match args { [a] => a, @@ -571,8 +325,6 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( // llvm.x86.avx2.vperm2i128 // llvm.x86.ssse3.pshuf.b.128 // llvm.x86.avx2.pshuf.b -// llvm.x86.avx2.psrli.w -// llvm.x86.sse2.psrli.w fn llvm_add_sub<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 9863e40b5..6efbe1498 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -21,7 +21,7 @@ fn report_simd_type_validation_error( pub(super) fn codegen_simd_intrinsic_call<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, intrinsic: Symbol, - _args: GenericArgsRef<'tcx>, + generic_args: GenericArgsRef<'tcx>, args: &[mir::Operand<'tcx>], ret: CPlace<'tcx>, target: BasicBlock, @@ -117,6 +117,54 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( }); } + // simd_shuffle_generic<T, U, const I: &[u32]>(x: T, y: T) -> U + sym::simd_shuffle_generic => { + let [x, y] = args else { + bug!("wrong number of args for intrinsic {intrinsic}"); + }; + let x = codegen_operand(fx, x); + let y = codegen_operand(fx, y); + + if !x.layout().ty.is_simd() { + report_simd_type_validation_error(fx, intrinsic, span, x.layout().ty); + return; + } + + let idx = generic_args[2] + .expect_const() + .eval(fx.tcx, ty::ParamEnv::reveal_all(), Some(span)) + .unwrap() + .unwrap_branch(); + + assert_eq!(x.layout(), y.layout()); + let layout = x.layout(); + + let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let (ret_lane_count, ret_lane_ty) = ret.layout().ty.simd_size_and_type(fx.tcx); + + assert_eq!(lane_ty, ret_lane_ty); + assert_eq!(idx.len() as u64, ret_lane_count); + + let total_len = lane_count * 2; + + let indexes = + idx.iter().map(|idx| idx.unwrap_leaf().try_to_u16().unwrap()).collect::<Vec<u16>>(); + + for &idx in &indexes { + assert!(u64::from(idx) < total_len, "idx {} out of range 0..{}", idx, total_len); + } + + for (out_idx, in_idx) in indexes.into_iter().enumerate() { + let in_lane = if u64::from(in_idx) < lane_count { + x.value_lane(fx, in_idx.into()) + } else { + y.value_lane(fx, u64::from(in_idx) - lane_count) + }; + let out_lane = ret.place_lane(fx, u64::try_from(out_idx).unwrap()); + out_lane.write_cvalue(fx, in_lane); + } + } + // simd_shuffle<T, I, U>(x: T, y: T, idx: I) -> U sym::simd_shuffle => { let (x, y, idx) = match args { @@ -172,7 +220,8 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( .expect("simd_shuffle idx not const"); let idx_bytes = match idx_const { - ConstValue::ByRef { alloc, offset } => { + ConstValue::Indirect { alloc_id, offset } => { + let alloc = fx.tcx.global_alloc(alloc_id).unwrap_memory(); let size = Size::from_bytes( 4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */ ); diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs index 6aeba13f6..c6133f2b3 100644 --- a/compiler/rustc_codegen_cranelift/src/unsize.rs +++ b/compiler/rustc_codegen_cranelift/src/unsize.rs @@ -88,7 +88,8 @@ fn unsize_ptr<'tcx>( let src_f = src_layout.field(fx, i); assert_eq!(src_layout.fields.offset(i).bytes(), 0); assert_eq!(dst_layout.fields.offset(i).bytes(), 0); - if src_f.is_zst() { + if src_f.is_1zst() { + // We are looking for the one non-1-ZST field; this is not it. continue; } assert_eq!(src_layout.size, src_f.size); @@ -151,6 +152,7 @@ pub(crate) fn coerce_unsized_into<'tcx>( let dst_f = dst.place_field(fx, FieldIdx::new(i)); if dst_f.layout().is_zst() { + // No data here, nothing to copy/coerce. continue; } diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index ff95141ce..45893a4f3 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -674,7 +674,9 @@ impl<'tcx> CPlace<'tcx> { } } - pub(crate) fn place_opaque_cast( + /// Used for `ProjectionElem::Subtype`, `ty` has to be monomorphized before + /// passed on. + pub(crate) fn place_transmute_type( self, fx: &mut FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>, diff --git a/compiler/rustc_codegen_cranelift/src/vtable.rs b/compiler/rustc_codegen_cranelift/src/vtable.rs index b309695c1..41ea0b122 100644 --- a/compiler/rustc_codegen_cranelift/src/vtable.rs +++ b/compiler/rustc_codegen_cranelift/src/vtable.rs @@ -48,19 +48,12 @@ pub(crate) fn get_ptr_and_method_ref<'tcx>( ) -> (Pointer, Value) { let (ptr, vtable) = 'block: { if let Abi::Scalar(_) = arg.layout().abi { - 'descend_newtypes: while !arg.layout().ty.is_unsafe_ptr() && !arg.layout().ty.is_ref() { - for i in 0..arg.layout().fields.count() { - let field = arg.value_field(fx, FieldIdx::new(i)); - if !field.layout().is_zst() { - // we found the one non-zero-sized field that is allowed - // now find *its* non-zero-sized field, or stop if it's a - // pointer - arg = field; - continue 'descend_newtypes; - } - } - - bug!("receiver has no non-zero-sized fields {:?}", arg); + while !arg.layout().ty.is_unsafe_ptr() && !arg.layout().ty.is_ref() { + let (idx, _) = arg + .layout() + .non_1zst_field(fx) + .expect("not exactly one non-1-ZST field in a `DispatchFromDyn` type"); + arg = arg.value_field(fx, FieldIdx::new(idx)); } } |