diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src')
26 files changed, 455 insertions, 535 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs index abf63e33c..364503fd3 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs @@ -6,8 +6,6 @@ use std::borrow::Cow; use rustc_middle::mir; use rustc_target::abi::call::PassMode; -use cranelift_codegen::entity::EntityRef; - use crate::prelude::*; pub(super) fn add_args_header_comment(fx: &mut FunctionCx<'_, '_, '_>) { @@ -91,35 +89,7 @@ pub(super) fn add_local_place_comments<'tcx>( largest_niche: _, } = layout.0.0; - let (kind, extra) = match *place.inner() { - CPlaceInner::Var(place_local, var) => { - assert_eq!(local, place_local); - ("ssa", Cow::Owned(format!(",var={}", var.index()))) - } - CPlaceInner::VarPair(place_local, var1, var2) => { - assert_eq!(local, place_local); - ("ssa", Cow::Owned(format!("var=({}, {})", var1.index(), var2.index()))) - } - CPlaceInner::VarLane(_local, _var, _lane) => unreachable!(), - CPlaceInner::Addr(ptr, meta) => { - let meta = if let Some(meta) = meta { - Cow::Owned(format!("meta={}", meta)) - } else { - Cow::Borrowed("") - }; - match ptr.debug_base_and_offset() { - (crate::pointer::PointerBase::Addr(addr), offset) => { - ("reuse", format!("storage={}{}{}", addr, offset, meta).into()) - } - (crate::pointer::PointerBase::Stack(stack_slot), offset) => { - ("stack", format!("storage={}{}{}", stack_slot, offset, meta).into()) - } - (crate::pointer::PointerBase::Dangling(align), offset) => { - ("zst", format!("align={},offset={}", align.bytes(), offset).into()) - } - } - } - }; + let (kind, extra) = place.debug_comment(); fx.add_global_comment(format!( "{:<5} {:5} {:30} {:4}b {}, {}{}{}", diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index 91c085d3d..84e09cf0a 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -70,7 +70,7 @@ pub(crate) fn get_function_sig<'tcx>( default_call_conv: CallConv, inst: Instance<'tcx>, ) -> Signature { - assert!(!inst.substs.needs_infer()); + assert!(!inst.substs.has_infer()); clif_sig_from_fn_abi( tcx, default_call_conv, @@ -88,10 +88,10 @@ pub(crate) fn import_function<'tcx>( let sig = get_function_sig(tcx, module.target_config().default_call_conv, inst); match module.declare_function(name, Linkage::Import, &sig) { Ok(func_id) => func_id, - Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(&format!( + Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(format!( "attempt to declare `{name}` as function, but it was already declared as static" )), - Err(ModuleError::IncompatibleSignature(_, prev_sig, new_sig)) => tcx.sess.fatal(&format!( + Err(ModuleError::IncompatibleSignature(_, prev_sig, new_sig)) => tcx.sess.fatal(format!( "attempt to declare `{name}` with signature {new_sig:?}, \ but it was already declared with signature {prev_sig:?}" )), @@ -432,11 +432,9 @@ pub(crate) fn codegen_terminator_call<'tcx>( let is_cold = if fn_sig.abi() == Abi::RustCold { true } else { - instance - .map(|inst| { - fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD) - }) - .unwrap_or(false) + instance.is_some_and(|inst| { + fx.tcx.codegen_fn_attrs(inst.def_id()).flags.contains(CodegenFnAttrFlags::COLD) + }) }; if is_cold { fx.bcx.set_cold_block(fx.bcx.current_block().unwrap()); @@ -470,7 +468,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( }; // Pass the caller location for `#[track_caller]`. - if instance.map(|inst| inst.def.requires_caller_location(fx.tcx)).unwrap_or(false) { + if instance.is_some_and(|inst| inst.def.requires_caller_location(fx.tcx)) { let caller_location = fx.get_caller_location(source_info); args.push(CallArgument { value: caller_location, is_owned: false }); } @@ -548,7 +546,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( if !matches!(fn_sig.abi(), Abi::C { .. }) { fx.tcx.sess.span_fatal( source_info.span, - &format!("Variadic call for non-C abi {:?}", fn_sig.abi()), + format!("Variadic call for non-C abi {:?}", fn_sig.abi()), ); } let sig_ref = fx.bcx.func.dfg.call_signature(call_inst).unwrap(); @@ -560,7 +558,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( // FIXME set %al to upperbound on float args once floats are supported fx.tcx.sess.span_fatal( source_info.span, - &format!("Non int ty {:?} for variadic call", ty), + format!("Non int ty {:?} for variadic call", ty), ); } AbiParam::new(ty) @@ -605,9 +603,9 @@ pub(crate) fn codegen_drop<'tcx>( // | ... | // \-------/ // - let (ptr, vtable) = drop_place.to_ptr_maybe_unsized(); + let (ptr, vtable) = drop_place.to_ptr_unsized(); let ptr = ptr.get_addr(fx); - let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable.unwrap()); + let drop_fn = crate::vtable::drop_fn_of_obj(fx, vtable); // FIXME(eddyb) perhaps move some of this logic into // `Instance::resolve_drop_in_place`? diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs index e5ad31eb9..d847e524f 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs @@ -84,7 +84,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { attrs )], Abi::Vector { .. } => { - let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout); smallvec![AbiParam::new(vector_ty)] } _ => unreachable!("{:?}", self.layout.abi), @@ -135,7 +135,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { (None, vec![AbiParam::new(scalar_to_clif_type(tcx, scalar))]) } Abi::Vector { .. } => { - let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout).unwrap(); + let vector_ty = crate::intrinsics::clif_vector_type(tcx, self.layout); (None, vec![AbiParam::new(vector_ty)]) } _ => unreachable!("{:?}", self.layout.abi), diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs index 6d3e8eda2..14e54d5ee 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs @@ -9,7 +9,7 @@ use smallvec::{smallvec, SmallVec}; /// this adds an extra parameter pointing to where the return value needs to be stored. pub(super) fn codegen_return_param<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, - ssa_analyzed: &rustc_index::vec::IndexVec<Local, crate::analyze::SsaKind>, + ssa_analyzed: &rustc_index::IndexSlice<Local, crate::analyze::SsaKind>, block_params_iter: &mut impl Iterator<Item = Value>, ) -> CPlace<'tcx> { let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode { @@ -63,11 +63,11 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( let (ret_temp_place, return_ptr) = match ret_arg_abi.mode { PassMode::Ignore => (None, None), PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { - if matches!(ret_place.inner(), CPlaceInner::Addr(_, None)) { + if let Some(ret_ptr) = ret_place.try_to_ptr() { // This is an optimization to prevent unnecessary copies of the return value when // the return place is already a memory place as opposed to a register. // This match arm can be safely removed. - (None, Some(ret_place.to_ptr().get_addr(fx))) + (None, Some(ret_ptr.get_addr(fx))) } else { let place = CPlace::new_stack_slot(fx, ret_arg_abi.layout); (Some(place), Some(place.to_ptr().get_addr(fx))) diff --git a/compiler/rustc_codegen_cranelift/src/allocator.rs b/compiler/rustc_codegen_cranelift/src/allocator.rs index 2c246ceb3..d4b1ae2b6 100644 --- a/compiler/rustc_codegen_cranelift/src/allocator.rs +++ b/compiler/rustc_codegen_cranelift/src/allocator.rs @@ -3,10 +3,12 @@ use crate::prelude::*; -use rustc_ast::expand::allocator::{AllocatorKind, AllocatorTy, ALLOCATOR_METHODS}; +use rustc_ast::expand::allocator::{ + alloc_error_handler_name, default_fn_name, global_fn_name, AllocatorKind, AllocatorTy, + ALLOCATOR_METHODS, NO_ALLOC_SHIM_IS_UNSTABLE, +}; use rustc_codegen_ssa::base::allocator_kind_for_codegen; use rustc_session::config::OomStrategy; -use rustc_span::symbol::sym; /// Returns whether an allocator shim was created pub(crate) fn codegen( @@ -34,41 +36,43 @@ fn codegen_inner( ) { let usize_ty = module.target_config().pointer_type(); - for method in ALLOCATOR_METHODS { - let mut arg_tys = Vec::with_capacity(method.inputs.len()); - for ty in method.inputs.iter() { - match *ty { - AllocatorTy::Layout => { - arg_tys.push(usize_ty); // size - arg_tys.push(usize_ty); // align - } - AllocatorTy::Ptr => arg_tys.push(usize_ty), - AllocatorTy::Usize => arg_tys.push(usize_ty), + if kind == AllocatorKind::Default { + for method in ALLOCATOR_METHODS { + let mut arg_tys = Vec::with_capacity(method.inputs.len()); + for ty in method.inputs.iter() { + match *ty { + AllocatorTy::Layout => { + arg_tys.push(usize_ty); // size + arg_tys.push(usize_ty); // align + } + AllocatorTy::Ptr => arg_tys.push(usize_ty), + AllocatorTy::Usize => arg_tys.push(usize_ty), - AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + AllocatorTy::ResultPtr | AllocatorTy::Unit => panic!("invalid allocator arg"), + } } - } - let output = match method.output { - AllocatorTy::ResultPtr => Some(usize_ty), - AllocatorTy::Unit => None, + let output = match method.output { + AllocatorTy::ResultPtr => Some(usize_ty), + AllocatorTy::Unit => None, - AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { - panic!("invalid allocator output") - } - }; + AllocatorTy::Layout | AllocatorTy::Usize | AllocatorTy::Ptr => { + panic!("invalid allocator output") + } + }; - let sig = Signature { - call_conv: module.target_config().default_call_conv, - params: arg_tys.iter().cloned().map(AbiParam::new).collect(), - returns: output.into_iter().map(AbiParam::new).collect(), - }; - crate::common::create_wrapper_function( - module, - unwind_context, - sig, - &format!("__rust_{}", method.name), - &kind.fn_name(method.name), - ); + let sig = Signature { + call_conv: module.target_config().default_call_conv, + params: arg_tys.iter().cloned().map(AbiParam::new).collect(), + returns: output.into_iter().map(AbiParam::new).collect(), + }; + crate::common::create_wrapper_function( + module, + unwind_context, + sig, + &global_fn_name(method.name), + &default_fn_name(method.name), + ); + } } let sig = Signature { @@ -81,7 +85,7 @@ fn codegen_inner( unwind_context, sig, "__rust_alloc_error_handler", - &alloc_error_handler_kind.fn_name(sym::oom), + &alloc_error_handler_name(alloc_error_handler_kind), ); let data_id = module.declare_data(OomStrategy::SYMBOL, Linkage::Export, false, false).unwrap(); @@ -90,4 +94,11 @@ fn codegen_inner( let val = oom_strategy.should_panic(); data_ctx.define(Box::new([val])); module.define_data(data_id, &data_ctx).unwrap(); + + let data_id = + module.declare_data(NO_ALLOC_SHIM_IS_UNSTABLE, Linkage::Export, false, false).unwrap(); + let mut data_ctx = DataContext::new(); + data_ctx.set_align(1); + data_ctx.define(Box::new([0])); + module.define_data(data_id, &data_ctx).unwrap(); } diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs index 54d5c1c2a..359d581c1 100644 --- a/compiler/rustc_codegen_cranelift/src/analyze.rs +++ b/compiler/rustc_codegen_cranelift/src/analyze.rs @@ -2,7 +2,7 @@ use crate::prelude::*; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::mir::StatementKind::*; use rustc_middle::ty::Ty; diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index f5301f9f7..fcfa0b862 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -1,7 +1,7 @@ //! Codegen of a single function use rustc_ast::InlineAsmOptions; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; @@ -28,7 +28,7 @@ pub(crate) fn codegen_fn<'tcx>( module: &mut dyn Module, instance: Instance<'tcx>, ) -> CodegenedFunction { - debug_assert!(!instance.substs.needs_infer()); + debug_assert!(!instance.substs.has_infer()); let symbol_name = tcx.symbol_name(instance).name.to_string(); let _timer = tcx.prof.generic_activity_with_arg("codegen fn", &*symbol_name); @@ -141,16 +141,6 @@ pub(crate) fn compile_fn( context.clear(); context.func = codegened_func.func; - // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` - // instruction, which doesn't have an encoding. - context.compute_cfg(); - context.compute_domtree(); - context.eliminate_unreachable_code(module.isa()).unwrap(); - context.dce(module.isa()).unwrap(); - // Some Cranelift optimizations expect the domtree to not yet be computed and as such don't - // invalidate it when it would change. - context.domtree.clear(); - #[cfg(any())] // This is never true let _clif_guard = { use std::fmt::Write; @@ -182,27 +172,6 @@ pub(crate) fn compile_fn( cx.profiler.generic_activity("define function").run(|| { context.want_disasm = cx.should_write_ir; module.define_function(codegened_func.func_id, context).unwrap(); - - if cx.profiler.enabled() { - let mut recording_args = false; - cx.profiler - .generic_activity_with_arg_recorder( - "define function (clif pass timings)", - |recorder| { - let pass_times = cranelift_codegen::timing::take_current(); - // Replace newlines with | as measureme doesn't allow control characters like - // newlines inside strings. - recorder.record_arg(format!("{}", pass_times).replace('\n', " | ")); - recording_args = true; - }, - ) - .run(|| { - if recording_args { - // Wait a tiny bit to ensure chrome's profiler doesn't hide the event - std::thread::sleep(std::time::Duration::from_nanos(2)) - } - }); - } }); if cx.should_write_ir { @@ -216,7 +185,7 @@ pub(crate) fn compile_fn( &clif_comments, ); - if let Some(disasm) = &context.compiled_code().unwrap().disasm { + if let Some(disasm) = &context.compiled_code().unwrap().vcode { crate::pretty_clif::write_ir_file( &cx.output_filenames, &format!("{}.vcode", codegened_func.symbol_name), @@ -251,13 +220,13 @@ pub(crate) fn verify_func( match cranelift_codegen::verify_function(&func, &flags) { Ok(_) => {} Err(err) => { - tcx.sess.err(&format!("{:?}", err)); + tcx.sess.err(format!("{:?}", err)); let pretty_error = cranelift_codegen::print_errors::pretty_verifier_error( &func, Some(Box::new(writer)), err, ); - tcx.sess.fatal(&format!("cranelift verify error:\n{}", pretty_error)); + tcx.sess.fatal(format!("cranelift verify error:\n{}", pretty_error)); } } }); @@ -366,7 +335,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { fx.bcx.switch_to_block(failure); fx.bcx.ins().nop(); - match msg { + match &**msg { AssertKind::BoundsCheck { ref len, ref index } => { let len = codegen_operand(fx, len).load_scalar(fx); let index = codegen_operand(fx, index).load_scalar(fx); @@ -504,7 +473,7 @@ fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { | TerminatorKind::GeneratorDrop => { bug!("shouldn't exist at codegen {:?}", bb_data.terminator()); } - TerminatorKind::Drop { place, target, unwind: _ } => { + TerminatorKind::Drop { place, target, unwind: _, replace: _ } => { let drop_place = codegen_place(fx, *place); crate::abi::codegen_drop(fx, source_info, drop_place); @@ -524,13 +493,14 @@ fn codegen_stmt<'tcx>( fx.set_debug_loc(stmt.source_info); - #[cfg(any())] // This is never true match &stmt.kind { StatementKind::StorageLive(..) | StatementKind::StorageDead(..) => {} // Those are not very useful _ => { if fx.clif_comments.enabled() { let inst = fx.bcx.func.layout.last_inst(cur_block).unwrap(); - fx.add_comment(inst, format!("{:?}", stmt)); + with_no_trimmed_paths!({ + fx.add_comment(inst, format!("{:?}", stmt)); + }); } } } @@ -660,11 +630,11 @@ fn codegen_stmt<'tcx>( let to_ty = fx.monomorphize(to_ty); fn is_fat_ptr<'tcx>(fx: &FunctionCx<'_, '_, 'tcx>, ty: Ty<'tcx>) -> bool { - ty.builtin_deref(true) - .map(|ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| { + ty.builtin_deref(true).is_some_and( + |ty::TypeAndMut { ty: pointee_ty, mutbl: _ }| { has_ptr_meta(fx.tcx, pointee_ty) - }) - .unwrap_or(false) + }, + ) } if is_fat_ptr(fx, from_ty) { @@ -715,11 +685,11 @@ fn codegen_stmt<'tcx>( } Rvalue::Cast(CastKind::Pointer(PointerCast::Unsize), ref operand, _to_ty) => { let operand = codegen_operand(fx, operand); - operand.unsize_value(fx, lval); + crate::unsize::coerce_unsized_into(fx, operand, lval); } Rvalue::Cast(CastKind::DynStar, ref operand, _) => { let operand = codegen_operand(fx, operand); - operand.coerce_dyn_star(fx, lval); + crate::unsize::coerce_dyn_star(fx, operand, lval); } Rvalue::Cast(CastKind::Transmute, ref operand, _to_ty) => { let operand = codegen_operand(fx, operand); @@ -781,14 +751,20 @@ fn codegen_stmt<'tcx>( let operand = operand.load_scalar(fx); lval.write_cvalue(fx, CValue::by_val(operand, box_layout)); } - Rvalue::NullaryOp(null_op, ty) => { + Rvalue::NullaryOp(ref null_op, ty) => { assert!(lval.layout().ty.is_sized(fx.tcx, ParamEnv::reveal_all())); let layout = fx.layout_of(fx.monomorphize(ty)); let val = match null_op { NullOp::SizeOf => layout.size.bytes(), NullOp::AlignOf => layout.align.abi.bytes(), + NullOp::OffsetOf(fields) => { + layout.offset_of_subfield(fx, fields.iter().map(|f| f.index())).bytes() + } }; - let val = CValue::const_val(fx, fx.layout_of(fx.tcx.types.usize), val.into()); + let val = CValue::by_val( + fx.bcx.ins().iconst(fx.pointer_type, i64::try_from(val).unwrap()), + fx.layout_of(fx.tcx.types.usize), + ); lval.write_cvalue(fx, val); } Rvalue::Aggregate(ref kind, ref operands) => { @@ -863,9 +839,7 @@ fn codegen_array_len<'tcx>(fx: &mut FunctionCx<'_, '_, 'tcx>, place: CPlace<'tcx let len = fx.monomorphize(len).eval_target_usize(fx.tcx, ParamEnv::reveal_all()) as i64; fx.bcx.ins().iconst(fx.pointer_type, len) } - ty::Slice(_elem_ty) => { - place.to_ptr_maybe_unsized().1.expect("Length metadata for slice place") - } + ty::Slice(_elem_ty) => place.to_ptr_unsized().1, _ => bug!("Rvalue::Len({:?})", place), } } @@ -919,8 +893,7 @@ pub(crate) fn codegen_place<'tcx>( ty::Slice(elem_ty) => { assert!(from_end, "slice subslices should be `from_end`"); let elem_layout = fx.layout_of(*elem_ty); - let (ptr, len) = cplace.to_ptr_maybe_unsized(); - let len = len.unwrap(); + let (ptr, len) = cplace.to_ptr_unsized(); cplace = CPlace::for_ptr_with_extra( ptr.offset_i64(fx, elem_layout.size.bytes() as i64 * (from as i64)), fx.bcx.ins().iadd_imm(len, -(from as i64 + to as i64)), @@ -993,11 +966,7 @@ fn codegen_panic_inner<'tcx>( args: &[Value], span: Span, ) { - let def_id = fx - .tcx - .lang_items() - .require(lang_item) - .unwrap_or_else(|e| fx.tcx.sess.span_fatal(span, e.to_string())); + let def_id = fx.tcx.require_lang_item(lang_item, Some(span)); let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); let symbol_name = fx.tcx.symbol_name(instance).name; diff --git a/compiler/rustc_codegen_cranelift/src/cast.rs b/compiler/rustc_codegen_cranelift/src/cast.rs index 032d11510..6bf3a866b 100644 --- a/compiler/rustc_codegen_cranelift/src/cast.rs +++ b/compiler/rustc_codegen_cranelift/src/cast.rs @@ -103,7 +103,7 @@ pub(crate) fn clif_int_or_float_cast( vec![AbiParam::new(types::I64X2)], &[from], )[0]; - // FIXME use bitcast instead of store to get from i64x2 to i128 + // FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128 let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, size: 16, diff --git a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs index f674ce776..f751d8c17 100644 --- a/compiler/rustc_codegen_cranelift/src/codegen_i128.rs +++ b/compiler/rustc_codegen_cranelift/src/codegen_i128.rs @@ -7,7 +7,6 @@ use crate::prelude::*; pub(crate) fn maybe_codegen<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, bin_op: BinOp, - checked: bool, lhs: CValue<'tcx>, rhs: CValue<'tcx>, ) -> Option<CValue<'tcx>> { @@ -22,69 +21,23 @@ pub(crate) fn maybe_codegen<'tcx>( let is_signed = type_sign(lhs.layout().ty); match bin_op { - BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => { - assert!(!checked); - None - } - BinOp::Add | BinOp::Sub if !checked => None, - BinOp::Mul if !checked || is_signed => { - if !checked { - let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)]; - let ret_val = fx.lib_call( - "__multi3", - vec![AbiParam::new(types::I128), AbiParam::new(types::I128)], - vec![AbiParam::new(types::I128)], - &args, - )[0]; - Some(CValue::by_val( - ret_val, - fx.layout_of(if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 }), - )) - } else { - let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]); - let oflow = CPlace::new_stack_slot(fx, fx.layout_of(fx.tcx.types.i32)); - let lhs = lhs.load_scalar(fx); - let rhs = rhs.load_scalar(fx); - let oflow_ptr = oflow.to_ptr().get_addr(fx); - let res = fx.lib_call_unadjusted( - "__muloti4", - vec![ - AbiParam::new(types::I128), - AbiParam::new(types::I128), - AbiParam::new(fx.pointer_type), - ], - vec![AbiParam::new(types::I128)], - &[lhs, rhs, oflow_ptr], - )[0]; - let oflow = oflow.to_cvalue(fx).load_scalar(fx); - let oflow = fx.bcx.ins().ireduce(types::I8, oflow); - Some(CValue::by_val_pair(res, oflow, fx.layout_of(out_ty))) - } - } - BinOp::Add | BinOp::Sub | BinOp::Mul => { - assert!(checked); - let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]); - let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty)); - let param_types = vec![ - AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn), - AbiParam::new(types::I128), - AbiParam::new(types::I128), - ]; - let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)]; - let name = match (bin_op, is_signed) { - (BinOp::Add, false) => "__rust_u128_addo", - (BinOp::Add, true) => "__rust_i128_addo", - (BinOp::Sub, false) => "__rust_u128_subo", - (BinOp::Sub, true) => "__rust_i128_subo", - (BinOp::Mul, false) => "__rust_u128_mulo", - _ => unreachable!(), - }; - fx.lib_call(name, param_types, vec![], &args); - Some(out_place.to_cvalue(fx)) + BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => None, + BinOp::Add | BinOp::Sub => None, + BinOp::Mul => { + let args = [lhs.load_scalar(fx), rhs.load_scalar(fx)]; + let ret_val = fx.lib_call( + "__multi3", + vec![AbiParam::new(types::I128), AbiParam::new(types::I128)], + vec![AbiParam::new(types::I128)], + &args, + )[0]; + Some(CValue::by_val( + ret_val, + fx.layout_of(if is_signed { fx.tcx.types.i128 } else { fx.tcx.types.u128 }), + )) } BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), BinOp::Div | BinOp::Rem => { - assert!(!checked); let name = match (bin_op, is_signed) { (BinOp::Div, false) => "__udivti3", (BinOp::Div, true) => "__divti3", @@ -100,7 +53,7 @@ pub(crate) fn maybe_codegen<'tcx>( vec![AbiParam::new(types::I64X2)], &args, )[0]; - // FIXME use bitcast instead of store to get from i64x2 to i128 + // FIXME(bytecodealliance/wasmtime#6104) use bitcast instead of store to get from i64x2 to i128 let ret_place = CPlace::new_stack_slot(fx, lhs.layout()); ret_place.to_ptr().store(fx, ret, MemFlags::trusted()); Some(ret_place.to_cvalue(fx)) @@ -115,10 +68,72 @@ pub(crate) fn maybe_codegen<'tcx>( Some(CValue::by_val(ret_val, lhs.layout())) } } - BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => { - assert!(!checked); - None - } + BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => None, BinOp::Shl | BinOp::Shr => None, } } + +pub(crate) fn maybe_codegen_checked<'tcx>( + fx: &mut FunctionCx<'_, '_, 'tcx>, + bin_op: BinOp, + lhs: CValue<'tcx>, + rhs: CValue<'tcx>, +) -> Option<CValue<'tcx>> { + if lhs.layout().ty != fx.tcx.types.u128 + && lhs.layout().ty != fx.tcx.types.i128 + && rhs.layout().ty != fx.tcx.types.u128 + && rhs.layout().ty != fx.tcx.types.i128 + { + return None; + } + + let is_signed = type_sign(lhs.layout().ty); + + match bin_op { + BinOp::BitAnd | BinOp::BitOr | BinOp::BitXor => unreachable!(), + BinOp::Mul if is_signed => { + let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]); + let oflow = CPlace::new_stack_slot(fx, fx.layout_of(fx.tcx.types.i32)); + let lhs = lhs.load_scalar(fx); + let rhs = rhs.load_scalar(fx); + let oflow_ptr = oflow.to_ptr().get_addr(fx); + let res = fx.lib_call_unadjusted( + "__muloti4", + vec![ + AbiParam::new(types::I128), + AbiParam::new(types::I128), + AbiParam::new(fx.pointer_type), + ], + vec![AbiParam::new(types::I128)], + &[lhs, rhs, oflow_ptr], + )[0]; + let oflow = oflow.to_cvalue(fx).load_scalar(fx); + let oflow = fx.bcx.ins().ireduce(types::I8, oflow); + Some(CValue::by_val_pair(res, oflow, fx.layout_of(out_ty))) + } + BinOp::Add | BinOp::Sub | BinOp::Mul => { + let out_ty = fx.tcx.mk_tup(&[lhs.layout().ty, fx.tcx.types.bool]); + let out_place = CPlace::new_stack_slot(fx, fx.layout_of(out_ty)); + let param_types = vec![ + AbiParam::special(fx.pointer_type, ArgumentPurpose::StructReturn), + AbiParam::new(types::I128), + AbiParam::new(types::I128), + ]; + let args = [out_place.to_ptr().get_addr(fx), lhs.load_scalar(fx), rhs.load_scalar(fx)]; + let name = match (bin_op, is_signed) { + (BinOp::Add, false) => "__rust_u128_addo", + (BinOp::Add, true) => "__rust_i128_addo", + (BinOp::Sub, false) => "__rust_u128_subo", + (BinOp::Sub, true) => "__rust_i128_subo", + (BinOp::Mul, false) => "__rust_u128_mulo", + _ => unreachable!(), + }; + fx.lib_call(name, param_types, vec![], &args); + Some(out_place.to_cvalue(fx)) + } + BinOp::Offset => unreachable!("offset should only be used on pointers, not 128bit ints"), + BinOp::Div | BinOp::Rem => unreachable!(), + BinOp::Lt | BinOp::Le | BinOp::Eq | BinOp::Ge | BinOp::Gt | BinOp::Ne => unreachable!(), + BinOp::Shl | BinOp::Shr => unreachable!(), + } +} diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index d39bf7000..5eaa988dd 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -2,7 +2,7 @@ use cranelift_codegen::isa::TargetFrontendConfig; use gimli::write::FileId; use rustc_data_structures::sync::Lrc; -use rustc_index::vec::IndexVec; +use rustc_index::IndexVec; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, }; @@ -72,19 +72,6 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ pointer_ty(tcx) } } - ty::Adt(adt_def, _) if adt_def.repr().simd() => { - let (element, count) = match &tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap().abi - { - Abi::Vector { element, count } => (*element, *count), - _ => unreachable!(), - }; - - match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) { - // Cranelift currently only implements icmp for 128bit vectors. - Some(vector_ty) if vector_ty.bits() == 128 => vector_ty, - _ => return None, - } - } ty::Param(_) => bug!("ty param {:?}", ty), _ => return None, }) @@ -96,12 +83,7 @@ fn clif_pair_type_from_ty<'tcx>( ) -> Option<(types::Type, types::Type)> { Some(match ty.kind() { ty::Tuple(types) if types.len() == 2 => { - let a = clif_type_from_ty(tcx, types[0])?; - let b = clif_type_from_ty(tcx, types[1])?; - if a.is_vector() || b.is_vector() { - return None; - } - (a, b) + (clif_type_from_ty(tcx, types[0])?, clif_type_from_ty(tcx, types[1])?) } ty::RawPtr(TypeAndMut { ty: pointee_ty, mutbl: _ }) | ty::Ref(_, pointee_ty, _) => { if has_ptr_meta(tcx, *pointee_ty) { @@ -379,7 +361,7 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { self.instance.subst_mir_and_normalize_erasing_regions( self.tcx, ty::ParamEnv::reveal_all(), - value, + ty::EarlyBinder(value), ) } @@ -495,7 +477,7 @@ impl<'tcx> LayoutOfHelpers<'tcx> for RevealAllLayoutCx<'tcx> { #[inline] fn handle_layout_err(&self, err: LayoutError<'tcx>, span: Span, ty: Ty<'tcx>) -> ! { if let layout::LayoutError::SizeOverflow(_) = err { - self.0.sess.span_fatal(span, &err.to_string()) + self.0.sess.span_fatal(span, err.to_string()) } else { span_bug!(span, "failed to get layout for `{}`: {}", ty, err) } @@ -513,7 +495,7 @@ impl<'tcx> FnAbiOfHelpers<'tcx> for RevealAllLayoutCx<'tcx> { fn_abi_request: FnAbiRequest<'tcx>, ) -> ! { if let FnAbiError::Layout(LayoutError::SizeOverflow(_)) = err { - self.0.sess.span_fatal(span, &err.to_string()) + self.0.sess.span_fatal(span, err.to_string()) } else { match fn_abi_request { FnAbiRequest::OfFnPtr { sig, extra_args } => { diff --git a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs index 203219a8a..d2b928db7 100644 --- a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs +++ b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs @@ -25,8 +25,18 @@ impl ConcurrencyLimiter { .clone() .into_helper_thread(move |token| { let mut state = state_helper.lock().unwrap(); - state.add_new_token(token.unwrap()); - available_token_condvar_helper.notify_one(); + match token { + Ok(token) => { + state.add_new_token(token); + available_token_condvar_helper.notify_one(); + } + Err(err) => { + state.poison(format!("failed to acquire jobserver token: {}", err)); + // Notify all threads waiting for a token to give them a chance to + // gracefully exit. + available_token_condvar_helper.notify_all(); + } + } }) .unwrap(); ConcurrencyLimiter { @@ -37,16 +47,31 @@ impl ConcurrencyLimiter { } } - pub(super) fn acquire(&mut self) -> ConcurrencyLimiterToken { + pub(super) fn acquire(&mut self, handler: &rustc_errors::Handler) -> ConcurrencyLimiterToken { let mut state = self.state.lock().unwrap(); loop { state.assert_invariants(); - if state.try_start_job() { - return ConcurrencyLimiterToken { - state: self.state.clone(), - available_token_condvar: self.available_token_condvar.clone(), - }; + match state.try_start_job() { + Ok(true) => { + return ConcurrencyLimiterToken { + state: self.state.clone(), + available_token_condvar: self.available_token_condvar.clone(), + }; + } + Ok(false) => {} + Err(err) => { + // An error happened when acquiring the token. Raise it as fatal error. + // Make sure to drop the mutex guard first to prevent poisoning the mutex. + drop(state); + if let Some(err) = err { + handler.fatal(err).raise(); + } else { + // The error was already emitted, but compilation continued. Raise a silent + // fatal error. + rustc_errors::FatalError.raise(); + } + } } self.helper_thread.as_mut().unwrap().request_token(); @@ -100,13 +125,22 @@ mod state { pending_jobs: usize, active_jobs: usize, + poisoned: bool, + stored_error: Option<String>, + // None is used to represent the implicit token, Some to represent explicit tokens tokens: Vec<Option<Acquired>>, } impl ConcurrencyLimiterState { pub(super) fn new(pending_jobs: usize) -> Self { - ConcurrencyLimiterState { pending_jobs, active_jobs: 0, tokens: vec![None] } + ConcurrencyLimiterState { + pending_jobs, + active_jobs: 0, + poisoned: false, + stored_error: None, + tokens: vec![None], + } } pub(super) fn assert_invariants(&self) { @@ -127,14 +161,18 @@ mod state { self.drop_excess_capacity(); } - pub(super) fn try_start_job(&mut self) -> bool { + pub(super) fn try_start_job(&mut self) -> Result<bool, Option<String>> { + if self.poisoned { + return Err(self.stored_error.take()); + } + if self.active_jobs < self.tokens.len() { // Using existing token self.job_started(); - return true; + return Ok(true); } - false + Ok(false) } pub(super) fn job_started(&mut self) { @@ -161,6 +199,11 @@ mod state { self.assert_invariants(); } + pub(super) fn poison(&mut self, error: String) { + self.poisoned = true; + self.stored_error = Some(error); + } + fn drop_excess_capacity(&mut self) { self.assert_invariants(); diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index e87f4e258..77af561a5 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -91,7 +91,7 @@ pub(crate) fn eval_mir_constant<'tcx>( ), }, ConstantKind::Unevaluated(mir::UnevaluatedConst { def, .. }, _) - if fx.tcx.is_static(def.did) => + if fx.tcx.is_static(def) => { span_bug!(constant.span, "MIR constant refers to static"); } @@ -159,6 +159,8 @@ pub(crate) fn codegen_const_value<'tcx>( _ => unreachable!(), }; + // FIXME avoid this extra copy to the stack and directly write to the final + // destination let place = CPlace::new_stack_slot(fx, layout); place.to_ptr().store(fx, val, MemFlags::trusted()); place.to_cvalue(fx) @@ -306,7 +308,7 @@ fn data_id_for_static( attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL), ) { Ok(data_id) => data_id, - Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(&format!( + Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(format!( "attempt to declare `{symbol_name}` as static, but it was already declared as function" )), Err(err) => Err::<_, _>(err).unwrap(), @@ -354,7 +356,7 @@ fn data_id_for_static( attrs.flags.contains(CodegenFnAttrFlags::THREAD_LOCAL), ) { Ok(data_id) => data_id, - Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(&format!( + Err(ModuleError::IncompatibleDeclaration(_)) => tcx.sess.fatal(format!( "attempt to declare `{symbol_name}` as static, but it was already declared as function" )), Err(err) => Err::<_, _>(err).unwrap(), @@ -402,7 +404,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant if let Some(names) = section_name.split_once(',') { names } else { - tcx.sess.fatal(&format!( + tcx.sess.fatal(format!( "#[link_section = \"{}\"] is not valid for macos target: must be segment and section separated by comma", section_name )); @@ -447,7 +449,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant GlobalAlloc::Static(def_id) => { if tcx.codegen_fn_attrs(def_id).flags.contains(CodegenFnAttrFlags::THREAD_LOCAL) { - tcx.sess.fatal(&format!( + tcx.sess.fatal(format!( "Allocation {:?} contains reference to TLS value {:?}", alloc_id, def_id )); diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 3e2e2af96..aad9a9647 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -69,7 +69,7 @@ impl OngoingCodegen { let module_codegen_result = match module_codegen_result { Ok(module_codegen_result) => module_codegen_result, - Err(err) => sess.fatal(&err), + Err(err) => sess.fatal(err), }; let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } = module_codegen_result; @@ -324,6 +324,10 @@ fn module_codegen( OngoingModuleCodegen::Async(std::thread::spawn(move || { cx.profiler.clone().verbose_generic_activity_with_arg("compile functions", &*cgu_name).run( || { + cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( + cx.profiler.clone(), + ))); + let mut cached_context = Context::new(); for codegened_func in codegened_functions { crate::base::compile_fn( @@ -407,7 +411,7 @@ pub(crate) fn run_aot( backend_config.clone(), global_asm_config.clone(), cgu.name(), - concurrency_limiter.acquire(), + concurrency_limiter.acquire(tcx.sess.diagnostic()), ), module_codegen, Some(rustc_middle::dep_graph::hash_result), @@ -464,7 +468,7 @@ pub(crate) fn run_aot( let obj = create_compressed_metadata_file(tcx.sess, &metadata, &symbol_name); if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.sess.fatal(&format!("error writing metadata object file: {}", err)); + tcx.sess.fatal(format!("error writing metadata object file: {}", err)); } (metadata_cgu_name, tmp_file) diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index f6a48e325..3118105a4 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -224,6 +224,10 @@ pub(crate) fn codegen_and_compile_fn<'tcx>( module: &mut dyn Module, instance: Instance<'tcx>, ) { + cranelift_codegen::timing::set_thread_profiler(Box::new(super::MeasuremeProfiler( + cx.profiler.clone(), + ))); + tcx.prof.generic_activity("codegen and compile fn").run(|| { let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); diff --git a/compiler/rustc_codegen_cranelift/src/driver/mod.rs b/compiler/rustc_codegen_cranelift/src/driver/mod.rs index d09d3a529..5c52c9c18 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/mod.rs @@ -4,6 +4,7 @@ //! [`codegen_fn`]: crate::base::codegen_fn //! [`codegen_static`]: crate::constant::codegen_static +use rustc_data_structures::profiling::SelfProfilerRef; use rustc_middle::mir::mono::{Linkage as RLinkage, MonoItem, Visibility}; use crate::prelude::*; @@ -39,3 +40,31 @@ fn predefine_mono_items<'tcx>( } }); } + +struct MeasuremeProfiler(SelfProfilerRef); + +struct TimingGuard { + profiler: std::mem::ManuallyDrop<SelfProfilerRef>, + inner: Option<rustc_data_structures::profiling::TimingGuard<'static>>, +} + +impl Drop for TimingGuard { + fn drop(&mut self) { + self.inner.take(); + unsafe { + std::mem::ManuallyDrop::drop(&mut self.profiler); + } + } +} + +impl cranelift_codegen::timing::Profiler for MeasuremeProfiler { + fn start_pass(&self, pass: cranelift_codegen::timing::Pass) -> Box<dyn std::any::Any> { + let mut timing_guard = + TimingGuard { profiler: std::mem::ManuallyDrop::new(self.0.clone()), inner: None }; + timing_guard.inner = Some( + unsafe { &*(&*timing_guard.profiler as &SelfProfilerRef as *const SelfProfilerRef) } + .generic_activity(pass.description()), + ); + Box::new(timing_guard) + } +} diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs index a74f8ffa2..63a1f6959 100644 --- a/compiler/rustc_codegen_cranelift/src/global_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs @@ -104,7 +104,6 @@ pub(crate) fn compile_global_asm( return Ok(None); } - // FIXME fix linker error on macOS if cfg!(not(feature = "inline_asm")) { return Err( "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift" diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs index f722e5228..f67fdb592 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs @@ -42,7 +42,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( _ => { fx.tcx .sess - .warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic)); + .warn(format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic)); crate::trap::trap_unimplemented(fx, intrinsic); return; } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs index b431158d2..33b2f4702 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_aarch64.rs @@ -207,7 +207,7 @@ pub(crate) fn codegen_aarch64_llvm_intrinsic_call<'tcx>( } */ _ => { - fx.tcx.sess.warn(&format!( + fx.tcx.sess.warn(format!( "unsupported AArch64 llvm intrinsic {}; replacing with trap", intrinsic )); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs index 0f32d1a25..56d8f13ce 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm_x86.rs @@ -138,10 +138,9 @@ pub(crate) fn codegen_x86_llvm_intrinsic_call<'tcx>( llvm_add_sub(fx, BinOp::Sub, ret, b_in, a, b); } _ => { - fx.tcx.sess.warn(&format!( - "unsupported x86 llvm intrinsic {}; replacing with trap", - intrinsic - )); + fx.tcx + .sess + .warn(format!("unsupported x86 llvm intrinsic {}; replacing with trap", intrinsic)); crate::trap::trap_unimplemented(fx, intrinsic); return; } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index 03f2a65fc..0a513b08b 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -42,7 +42,7 @@ fn report_atomic_type_validation_error<'tcx>( ) { fx.tcx.sess.span_err( span, - &format!( + format!( "`{}` intrinsic: expected basic integer or raw pointer type, found `{:?}`", intrinsic, ty ), @@ -51,17 +51,13 @@ fn report_atomic_type_validation_error<'tcx>( fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); } -pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> { +pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Type { let (element, count) = match layout.abi { Abi::Vector { element, count } => (element, count), _ => unreachable!(), }; - match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) { - // Cranelift currently only implements icmp for 128bit vectors. - Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty), - _ => None, - } + scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()).unwrap() } fn simd_for_each_lane<'tcx>( @@ -534,7 +530,7 @@ fn codegen_regular_intrinsic_call<'tcx>( // The only difference between offset and arith_offset is regarding UB. Because Cranelift // doesn't have UB both are codegen'ed the same way - sym::offset | sym::arith_offset => { + sym::arith_offset => { intrinsic_args!(fx, args => (base, offset); intrinsic); let offset = offset.load_scalar(fx); @@ -1107,8 +1103,8 @@ fn codegen_regular_intrinsic_call<'tcx>( fx.bcx.ins().call_indirect(f_sig, f, &[data]); - let layout = ret.layout(); - let ret_val = CValue::const_val(fx, layout, ty::ScalarInt::null(layout.size)); + let layout = fx.layout_of(fx.tcx.types.i32); + let ret_val = CValue::by_val(fx.bcx.ins().iconst(types::I32, 0), layout); ret.write_cvalue(fx, ret_val); } @@ -1206,7 +1202,7 @@ fn codegen_regular_intrinsic_call<'tcx>( _ => { fx.tcx .sess - .span_fatal(source_info.span, &format!("unsupported intrinsic {}", intrinsic)); + .span_fatal(source_info.span, format!("unsupported intrinsic {}", intrinsic)); } } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 6f54a8d49..5a038bfca 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -13,7 +13,7 @@ fn report_simd_type_validation_error( span: Span, ty: Ty<'_>, ) { - fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty)); + fx.tcx.sess.span_err(span, format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty)); // Prevent verifier error fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); } @@ -150,7 +150,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( _ => { fx.tcx.sess.span_err( span, - &format!( + format!( "simd_shuffle index must be an array of `u32`, got `{}`", idx_ty, ), @@ -248,12 +248,12 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( if idx >= lane_count.into() { fx.tcx.sess.span_fatal( fx.mir.span, - &format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count), + format!("[simd_insert] idx {} >= lane_count {}", idx, lane_count), ); } ret.write_cvalue(fx, base); - let ret_lane = ret.place_field(fx, FieldIdx::new(idx.try_into().unwrap())); + let ret_lane = ret.place_lane(fx, idx.try_into().unwrap()); ret_lane.write_cvalue(fx, val); } @@ -296,7 +296,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( if idx >= lane_count.into() { fx.tcx.sess.span_fatal( fx.mir.span, - &format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count), + format!("[simd_extract] idx {} >= lane_count {}", idx, lane_count), ); } @@ -699,7 +699,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( _ => { fx.tcx.sess.span_fatal( span, - &format!( + format!( "invalid monomorphization of `simd_bitmask` intrinsic: \ vector argument `{}`'s element type `{}`, expected integer element \ type", @@ -739,7 +739,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( _ => { fx.tcx.sess.span_fatal( span, - &format!( + format!( "invalid monomorphization of `simd_bitmask` intrinsic: \ cannot return `{}`, expected `u{}` or `[u8; {}]`", ret.layout().ty, @@ -875,7 +875,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( } _ => { - fx.tcx.sess.span_err(span, &format!("Unknown SIMD intrinsic {}", intrinsic)); + fx.tcx.sess.span_err(span, format!("Unknown SIMD intrinsic {}", intrinsic)); // Prevent verifier error fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); } diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index 8cc7f6c34..9966cc2ef 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -90,7 +90,7 @@ mod prelude { pub(crate) use rustc_data_structures::fx::FxHashMap; - pub(crate) use rustc_index::vec::Idx; + pub(crate) use rustc_index::Idx; pub(crate) use cranelift_codegen::ir::condcodes::{FloatCC, IntCC}; pub(crate) use cranelift_codegen::ir::function::Function; @@ -110,7 +110,7 @@ mod prelude { pub(crate) use crate::common::*; pub(crate) use crate::debuginfo::{DebugContext, UnwindContext}; pub(crate) use crate::pointer::Pointer; - pub(crate) use crate::value_and_place::{CPlace, CPlaceInner, CValue}; + pub(crate) use crate::value_and_place::{CPlace, CValue}; } struct PrintOnPanic<F: Fn() -> String>(F); @@ -185,7 +185,7 @@ impl CodegenBackend for CraneliftCodegenBackend { let mut config = self.config.borrow_mut(); if config.is_none() { let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args) - .unwrap_or_else(|err| sess.fatal(&err)); + .unwrap_or_else(|err| sess.fatal(err)); *config = Some(new_config); } } @@ -245,7 +245,7 @@ impl CodegenBackend for CraneliftCodegenBackend { fn target_triple(sess: &Session) -> target_lexicon::Triple { match sess.target.llvm_target.parse() { Ok(triple) => triple, - Err(err) => sess.fatal(&format!("target not recognized: {}", err)), + Err(err) => sess.fatal(format!("target not recognized: {}", err)), } } @@ -307,7 +307,7 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::Tar Some(value) => { let mut builder = cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| { - sess.fatal(&format!("can't compile for {}: {}", target_triple, err)); + sess.fatal(format!("can't compile for {}: {}", target_triple, err)); }); if let Err(_) = builder.enable(value) { sess.fatal("the specified target cpu isn't currently supported by Cranelift."); @@ -317,7 +317,7 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::Tar None => { let mut builder = cranelift_codegen::isa::lookup(target_triple.clone()).unwrap_or_else(|err| { - sess.fatal(&format!("can't compile for {}: {}", target_triple, err)); + sess.fatal(format!("can't compile for {}: {}", target_triple, err)); }); if target_triple.architecture == target_lexicon::Architecture::X86_64 { // Don't use "haswell" as the default, as it implies `has_lzcnt`. @@ -330,7 +330,7 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Arc<dyn isa::Tar match isa_builder.finish(flags) { Ok(target_isa) => target_isa, - Err(err) => sess.fatal(&format!("failed to build TargetIsa: {}", err)), + Err(err) => sess.fatal(format!("failed to build TargetIsa: {}", err)), } } diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs index 205411e8c..20ba73f38 100644 --- a/compiler/rustc_codegen_cranelift/src/main_shim.rs +++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs @@ -75,7 +75,7 @@ pub(crate) fn maybe_create_entry_wrapper( Ok(func_id) => func_id, Err(err) => { tcx.sess - .fatal(&format!("entry symbol `{entry_name}` declared multiple times: {err}")); + .fatal(format!("entry symbol `{entry_name}` declared multiple times: {err}")); } }; @@ -171,7 +171,7 @@ pub(crate) fn maybe_create_entry_wrapper( } if let Err(err) = m.define_function(cmain_func_id, &mut ctx) { - tcx.sess.fatal(&format!("entry symbol `{entry_name}` defined multiple times: {err}")); + tcx.sess.fatal(format!("entry symbol `{entry_name}` defined multiple times: {err}")); } unwind_context.add_function(cmain_func_id, &ctx, m.isa()); diff --git a/compiler/rustc_codegen_cranelift/src/num.rs b/compiler/rustc_codegen_cranelift/src/num.rs index 1357b7be1..ba53e01c7 100644 --- a/compiler/rustc_codegen_cranelift/src/num.rs +++ b/compiler/rustc_codegen_cranelift/src/num.rs @@ -118,7 +118,7 @@ pub(crate) fn codegen_int_binop<'tcx>( ); } - if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, false, in_lhs, in_rhs) { + if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, in_lhs, in_rhs) { return res; } @@ -173,7 +173,7 @@ pub(crate) fn codegen_checked_int_binop<'tcx>( let lhs = in_lhs.load_scalar(fx); let rhs = in_rhs.load_scalar(fx); - if let Some(res) = crate::codegen_i128::maybe_codegen(fx, bin_op, true, in_lhs, in_rhs) { + if let Some(res) = crate::codegen_i128::maybe_codegen_checked(fx, bin_op, in_lhs, in_rhs) { return res; } diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs index e0a081c9d..1007b33ec 100644 --- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -7,48 +7,51 @@ //! test compile //! target x86_64 //! -//! function u0:0(i64, i64, i64) system_v { -//! ; symbol _ZN119_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$u27$a$u20$$RF$$u27$b$u20$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17he85059d5e6a760a0E -//! ; instance Instance { def: Item(DefId(0/0:29 ~ example[8787]::{{impl}}[0]::call_once[0])), substs: [ReErased, ReErased] } -//! ; sig ([IsNotEmpty, (&&[u16],)]; c_variadic: false)->(u8, u8) +//! function u0:22(i64) -> i8, i8 system_v { +//! ; symbol _ZN97_$LT$example..IsNotEmpty$u20$as$u20$mini_core..FnOnce$LT$$LP$$RF$$RF$$u5b$u16$u5d$$C$$RP$$GT$$GT$9call_once17hd517c453d67c0915E +//! ; instance Instance { def: Item(WithOptConstParam { did: DefId(0:42 ~ example[4e51]::{impl#0}::call_once), const_param_did: None }), substs: [ReErased, ReErased] } +//! ; abi FnAbi { args: [ArgAbi { layout: TyAndLayout { ty: IsNotEmpty, layout: Layout { size: Size(0 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: Aggregate { sized: true }, fields: Arbitrary { offsets: [], memory_index: [] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Ignore }, ArgAbi { layout: TyAndLayout { ty: &&[u16], layout: Layout { size: Size(8 bytes), align: AbiAndPrefAlign { abi: Align(8 bytes), pref: Align(8 bytes) }, abi: Scalar(Initialized { value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), fields: Primitive, largest_niche: Some(Niche { offset: Size(0 bytes), value: Pointer(AddressSpace(0)), valid_range: 1..=18446744073709551615 }), variants: Single { index: 0 } } }, mode: Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) }], ret: ArgAbi { layout: TyAndLayout { ty: (u8, u8), layout: Layout { size: Size(2 bytes), align: AbiAndPrefAlign { abi: Align(1 bytes), pref: Align(8 bytes) }, abi: ScalarPair(Initialized { value: Int(I8, false), valid_range: 0..=255 }, Initialized { value: Int(I8, false), valid_range: 0..=255 }), fields: Arbitrary { offsets: [Size(0 bytes), Size(1 bytes)], memory_index: [0, 1] }, largest_niche: None, variants: Single { index: 0 } } }, mode: Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) }, c_variadic: false, fixed_count: 1, conv: Rust, can_unwind: false } //! -//! ; ssa {_2: NOT_SSA, _4: NOT_SSA, _0: NOT_SSA, _3: (empty), _1: NOT_SSA} -//! ; msg loc.idx param pass mode ssa flags ty -//! ; ret _0 = v0 ByRef NOT_SSA (u8, u8) -//! ; arg _1 = v1 ByRef NOT_SSA IsNotEmpty -//! ; arg _2.0 = v2 ByVal(types::I64) NOT_SSA &&[u16] +//! ; kind loc.idx param pass mode ty +//! ; ssa _0 (u8, u8) 2b 1, 8 var=(0, 1) +//! ; ret _0 - Pair(ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }, ArgAttributes { regular: NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: None }) (u8, u8) +//! ; arg _1 - Ignore IsNotEmpty +//! ; arg _2.0 = v0 Direct(ArgAttributes { regular: NonNull | NoUndef, arg_ext: None, pointee_size: Size(0 bytes), pointee_align: Some(Align(8 bytes)) }) &&[u16] //! -//! ss0 = explicit_slot 0 ; _1: IsNotEmpty size=0 align=1,8 -//! ss1 = explicit_slot 8 ; _2: (&&[u16],) size=8 align=8,8 -//! ss2 = explicit_slot 8 ; _4: (&&[u16],) size=8 align=8,8 -//! sig0 = (i64, i64, i64) system_v -//! sig1 = (i64, i64, i64) system_v -//! fn0 = colocated u0:6 sig1 ; Instance { def: Item(DefId(0/0:31 ~ example[8787]::{{impl}}[1]::call_mut[0])), substs: [ReErased, ReErased] } +//! ; kind local ty size align (abi,pref) +//! ; zst _1 IsNotEmpty 0b 1, 8 align=8,offset= +//! ; stack _2 (&&[u16],) 8b 8, 8 storage=ss0 +//! ; ssa _3 &mut IsNotEmpty 8b 8, 8 var=2 //! -//! block0(v0: i64, v1: i64, v2: i64): -//! v3 = stack_addr.i64 ss0 -//! v4 = stack_addr.i64 ss1 -//! store v2, v4 -//! v5 = stack_addr.i64 ss2 +//! ss0 = explicit_slot 16 +//! sig0 = (i64, i64) -> i8, i8 system_v +//! fn0 = colocated u0:23 sig0 ; Instance { def: Item(WithOptConstParam { did: DefId(0:46 ~ example[4e51]::{impl#1}::call_mut), const_param_did: None }), substs: [ReErased, ReErased] } +//! +//! block0(v0: i64): +//! nop +//! ; write_cvalue: Addr(Pointer { base: Stack(ss0), offset: Offset32(0) }, None): &&[u16] <- ByVal(v0): &&[u16] +//! stack_store v0, ss0 //! jump block1 //! //! block1: //! nop //! ; _3 = &mut _1 -//! ; _4 = _2 -//! v6 = load.i64 v4 -//! store v6, v5 +//! v1 = iconst.i64 8 +//! ; write_cvalue: Var(_3, var2): &mut IsNotEmpty <- ByVal(v1): &mut IsNotEmpty //! ; -//! ; _0 = const mini_core::FnMut::call_mut(move _3, move _4) -//! v7 = load.i64 v5 -//! call fn0(v0, v3, v7) +//! ; _0 = <IsNotEmpty as mini_core::FnMut<(&&[u16],)>>::call_mut(move _3, _2) +//! v2 = stack_load.i64 ss0 +//! v3, v4 = call fn0(v1, v2) ; v1 = 8 +//! v5 -> v3 +//! v6 -> v4 +//! ; write_cvalue: VarPair(_0, var0, var1): (u8, u8) <- ByValPair(v3, v4): (u8, u8) //! jump block2 //! //! block2: //! nop //! ; //! ; return -//! return +//! return v5, v6 //! } //! ``` @@ -224,7 +227,7 @@ pub(crate) fn write_ir_file( // Using early_warn as no Session is available here rustc_session::early_warn( rustc_session::config::ErrorOutputType::default(), - &format!("error writing ir file: {}", err), + format!("error writing ir file: {}", err), ); } } diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index 1b69862ce..b1fda6ff2 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -2,8 +2,8 @@ use crate::prelude::*; +use cranelift_codegen::entity::EntityRef; use cranelift_codegen::ir::immediates::Offset32; -use cranelift_codegen::ir::{InstructionData, Opcode}; fn codegen_field<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, @@ -214,17 +214,7 @@ impl<'tcx> CValue<'tcx> { ) -> CValue<'tcx> { let layout = self.1; match self.0 { - CValueInner::ByVal(val) => match layout.abi { - Abi::Vector { element: _, count } => { - let count = u8::try_from(count).expect("SIMD type with more than 255 lanes???"); - let field = u8::try_from(field.index()).unwrap(); - assert!(field < count); - let lane = fx.bcx.ins().extractlane(val, field); - let field_layout = layout.field(&*fx, usize::from(field)); - CValue::by_val(lane, field_layout) - } - _ => unreachable!("value_field for ByVal with abi {:?}", layout.abi), - }, + CValueInner::ByVal(_) => unreachable!(), CValueInner::ByValPair(val1, val2) => match layout.abi { Abi::ScalarPair(_, _) => { let val = match field.as_u32() { @@ -258,16 +248,7 @@ impl<'tcx> CValue<'tcx> { let lane_layout = fx.layout_of(lane_ty); assert!(lane_idx < lane_count); match self.0 { - CValueInner::ByVal(val) => match layout.abi { - Abi::Vector { element: _, count: _ } => { - assert!(lane_count <= u8::MAX.into(), "SIMD type with more than 255 lanes???"); - let lane_idx = u8::try_from(lane_idx).unwrap(); - let lane = fx.bcx.ins().extractlane(val, lane_idx); - CValue::by_val(lane, lane_layout) - } - _ => unreachable!("value_lane for ByVal with abi {:?}", layout.abi), - }, - CValueInner::ByValPair(_, _) => unreachable!(), + CValueInner::ByVal(_) | CValueInner::ByValPair(_, _) => unreachable!(), CValueInner::ByRef(ptr, None) => { let field_offset = lane_layout.size * lane_idx; let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()); @@ -277,14 +258,6 @@ impl<'tcx> CValue<'tcx> { } } - pub(crate) fn unsize_value(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) { - crate::unsize::coerce_unsized_into(fx, self, dest); - } - - pub(crate) fn coerce_dyn_star(self, fx: &mut FunctionCx<'_, '_, 'tcx>, dest: CPlace<'tcx>) { - crate::unsize::coerce_dyn_star(fx, self, dest); - } - /// If `ty` is signed, `const_val` must already be sign extended. pub(crate) fn const_val( fx: &mut FunctionCx<'_, '_, 'tcx>, @@ -345,10 +318,9 @@ pub(crate) struct CPlace<'tcx> { } #[derive(Debug, Copy, Clone)] -pub(crate) enum CPlaceInner { +enum CPlaceInner { Var(Local, Variable), VarPair(Local, Variable, Variable), - VarLane(Local, Variable, u8), Addr(Pointer, Option<Value>), } @@ -357,10 +329,6 @@ impl<'tcx> CPlace<'tcx> { self.layout } - pub(crate) fn inner(&self) -> &CPlaceInner { - &self.inner - } - pub(crate) fn new_stack_slot( fx: &mut FunctionCx<'_, '_, 'tcx>, layout: TyAndLayout<'tcx>, @@ -376,7 +344,7 @@ impl<'tcx> CPlace<'tcx> { if layout.size.bytes() >= u64::from(u32::MAX - 16) { fx.tcx .sess - .fatal(&format!("values of type {} are too big to store on the stack", layout.ty)); + .fatal(format!("values of type {} are too big to store on the stack", layout.ty)); } let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData { @@ -442,12 +410,6 @@ impl<'tcx> CPlace<'tcx> { //fx.bcx.set_val_label(val2, cranelift_codegen::ir::ValueLabel::new(var2.index())); CValue::by_val_pair(val1, val2, layout) } - CPlaceInner::VarLane(_local, var, lane) => { - let val = fx.bcx.use_var(var); - //fx.bcx.set_val_label(val, cranelift_codegen::ir::ValueLabel::new(var.index())); - let val = fx.bcx.ins().extractlane(val, lane); - CValue::by_val(val, layout) - } CPlaceInner::Addr(ptr, extra) => { if let Some(extra) = extra { CValue::by_ref_unsized(ptr, extra, layout) @@ -458,21 +420,56 @@ impl<'tcx> CPlace<'tcx> { } } + pub(crate) fn debug_comment(self) -> (&'static str, String) { + match self.inner { + CPlaceInner::Var(_local, var) => ("ssa", format!("var={}", var.index())), + CPlaceInner::VarPair(_local, var1, var2) => { + ("ssa", format!("var=({}, {})", var1.index(), var2.index())) + } + CPlaceInner::Addr(ptr, meta) => { + let meta = + if let Some(meta) = meta { format!(",meta={}", meta) } else { String::new() }; + match ptr.debug_base_and_offset() { + (crate::pointer::PointerBase::Addr(addr), offset) => { + ("reuse", format!("storage={}{}{}", addr, offset, meta)) + } + (crate::pointer::PointerBase::Stack(stack_slot), offset) => { + ("stack", format!("storage={}{}{}", stack_slot, offset, meta)) + } + (crate::pointer::PointerBase::Dangling(align), offset) => { + ("zst", format!("align={},offset={}", align.bytes(), offset)) + } + } + } + } + } + #[track_caller] pub(crate) fn to_ptr(self) -> Pointer { - match self.to_ptr_maybe_unsized() { - (ptr, None) => ptr, - (_, Some(_)) => bug!("Expected sized cplace, found {:?}", self), + match self.inner { + CPlaceInner::Addr(ptr, None) => ptr, + CPlaceInner::Addr(_, Some(_)) => bug!("Expected sized cplace, found {:?}", self), + CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => { + bug!("Expected CPlace::Addr, found {:?}", self) + } } } #[track_caller] - pub(crate) fn to_ptr_maybe_unsized(self) -> (Pointer, Option<Value>) { + pub(crate) fn to_ptr_unsized(self) -> (Pointer, Value) { match self.inner { - CPlaceInner::Addr(ptr, extra) => (ptr, extra), - CPlaceInner::Var(_, _) - | CPlaceInner::VarPair(_, _, _) - | CPlaceInner::VarLane(_, _, _) => bug!("Expected CPlace::Addr, found {:?}", self), + CPlaceInner::Addr(ptr, Some(extra)) => (ptr, extra), + CPlaceInner::Addr(_, None) | CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => { + bug!("Expected unsized cplace, found {:?}", self) + } + } + } + + pub(crate) fn try_to_ptr(self) -> Option<Pointer> { + match self.inner { + CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => None, + CPlaceInner::Addr(ptr, None) => Some(ptr), + CPlaceInner::Addr(_, Some(_)) => bug!("Expected sized cplace, found {:?}", self), } } @@ -496,7 +493,7 @@ impl<'tcx> CPlace<'tcx> { from: CValue<'tcx>, method: &'static str, ) { - fn transmute_value<'tcx>( + fn transmute_scalar<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, var: Variable, data: Value, @@ -520,7 +517,7 @@ impl<'tcx> CPlace<'tcx> { | (types::F64, types::I64) => codegen_bitcast(fx, dst_ty, data), _ if src_ty.is_vector() && dst_ty.is_vector() => codegen_bitcast(fx, dst_ty, data), _ if src_ty.is_vector() || dst_ty.is_vector() => { - // FIXME do something more efficient for transmutes between vectors and integers. + // FIXME(bytecodealliance/wasmtime#6104) do something more efficient for transmutes between vectors and integers. let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to @@ -554,7 +551,7 @@ impl<'tcx> CPlace<'tcx> { format!( "{}: {:?}: {:?} <- {:?}: {:?}", method, - self.inner(), + self.inner, self.layout().ty, from.0, from.layout().ty @@ -563,32 +560,11 @@ impl<'tcx> CPlace<'tcx> { } let dst_layout = self.layout(); - let to_ptr = match self.inner { + match self.inner { CPlaceInner::Var(_local, var) => { - if let ty::Array(element, len) = dst_layout.ty.kind() { - // Can only happen for vector types - let len = u32::try_from(len.eval_target_usize(fx.tcx, ParamEnv::reveal_all())) - .unwrap(); - let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap(); - - let data = match from.0 { - CValueInner::ByRef(ptr, None) => { - let mut flags = MemFlags::new(); - flags.set_notrap(); - ptr.load(fx, vector_ty, flags) - } - CValueInner::ByVal(_) - | CValueInner::ByValPair(_, _) - | CValueInner::ByRef(_, Some(_)) => bug!("array should be ByRef"), - }; - - fx.bcx.def_var(var, data); - return; - } let data = CValue(from.0, dst_layout).load_scalar(fx); let dst_ty = fx.clif_type(self.layout().ty).unwrap(); - transmute_value(fx, var, data, dst_ty); - return; + transmute_scalar(fx, var, data, dst_ty); } CPlaceInner::VarPair(_local, var1, var2) => { let (data1, data2) = if from.layout().ty == dst_layout.ty { @@ -599,80 +575,61 @@ impl<'tcx> CPlace<'tcx> { CValue(CValueInner::ByRef(ptr, None), dst_layout).load_scalar_pair(fx) }; let (dst_ty1, dst_ty2) = fx.clif_pair_type(self.layout().ty).unwrap(); - transmute_value(fx, var1, data1, dst_ty1); - transmute_value(fx, var2, data2, dst_ty2); - return; + transmute_scalar(fx, var1, data1, dst_ty1); + transmute_scalar(fx, var2, data2, dst_ty2); } - CPlaceInner::VarLane(_local, var, lane) => { - let data = from.load_scalar(fx); - - // First get the old vector - let vector = fx.bcx.use_var(var); - //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index())); - - // Next insert the written lane into the vector - let vector = fx.bcx.ins().insertlane(vector, data, lane); - - // Finally write the new vector - //fx.bcx.set_val_label(vector, cranelift_codegen::ir::ValueLabel::new(var.index())); - fx.bcx.def_var(var, vector); - - return; - } - CPlaceInner::Addr(ptr, None) => { + CPlaceInner::Addr(_, Some(_)) => bug!("Can't write value to unsized place {:?}", self), + CPlaceInner::Addr(to_ptr, None) => { if dst_layout.size == Size::ZERO || dst_layout.abi == Abi::Uninhabited { return; } - ptr - } - CPlaceInner::Addr(_, Some(_)) => bug!("Can't write value to unsized place {:?}", self), - }; - let mut flags = MemFlags::new(); - flags.set_notrap(); - match from.layout().abi { - // FIXME make Abi::Vector work too - Abi::Scalar(_) => { - let val = from.load_scalar(fx); - to_ptr.store(fx, val, flags); - return; - } - Abi::ScalarPair(a_scalar, b_scalar) => { - let (value, extra) = from.load_scalar_pair(fx); - let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); - to_ptr.store(fx, value, flags); - to_ptr.offset(fx, b_offset).store(fx, extra, flags); - return; - } - _ => {} - } + let mut flags = MemFlags::new(); + flags.set_notrap(); + match from.layout().abi { + Abi::Scalar(_) => { + let val = from.load_scalar(fx); + to_ptr.store(fx, val, flags); + return; + } + Abi::ScalarPair(a_scalar, b_scalar) => { + let (value, extra) = from.load_scalar_pair(fx); + let b_offset = scalar_pair_calculate_b_offset(fx.tcx, a_scalar, b_scalar); + to_ptr.store(fx, value, flags); + to_ptr.offset(fx, b_offset).store(fx, extra, flags); + return; + } + _ => {} + } - match from.0 { - CValueInner::ByVal(val) => { - to_ptr.store(fx, val, flags); - } - CValueInner::ByValPair(_, _) => { - bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi); - } - CValueInner::ByRef(from_ptr, None) => { - let from_addr = from_ptr.get_addr(fx); - let to_addr = to_ptr.get_addr(fx); - let src_layout = from.1; - let size = dst_layout.size.bytes(); - let src_align = src_layout.align.abi.bytes() as u8; - let dst_align = dst_layout.align.abi.bytes() as u8; - fx.bcx.emit_small_memory_copy( - fx.target_config, - to_addr, - from_addr, - size, - dst_align, - src_align, - true, - flags, - ); + match from.0 { + CValueInner::ByVal(val) => { + to_ptr.store(fx, val, flags); + } + CValueInner::ByValPair(_, _) => { + bug!("Non ScalarPair abi {:?} for ByValPair CValue", dst_layout.abi); + } + CValueInner::ByRef(from_ptr, None) => { + let from_addr = from_ptr.get_addr(fx); + let to_addr = to_ptr.get_addr(fx); + let src_layout = from.1; + let size = dst_layout.size.bytes(); + let src_align = src_layout.align.abi.bytes() as u8; + let dst_align = dst_layout.align.abi.bytes() as u8; + fx.bcx.emit_small_memory_copy( + fx.target_config, + to_addr, + from_addr, + size, + dst_align, + src_align, + true, + flags, + ); + } + CValueInner::ByRef(_, Some(_)) => todo!(), + } } - CValueInner::ByRef(_, Some(_)) => todo!(), } } @@ -692,40 +649,6 @@ impl<'tcx> CPlace<'tcx> { let layout = self.layout(); match self.inner { - CPlaceInner::Var(local, var) => match layout.ty.kind() { - ty::Array(_, _) => { - // Can only happen for vector types - return CPlace { - inner: CPlaceInner::VarLane(local, var, field.as_u32().try_into().unwrap()), - layout: layout.field(fx, field.as_u32().try_into().unwrap()), - }; - } - ty::Adt(adt_def, substs) if layout.ty.is_simd() => { - let f0 = &adt_def.non_enum_variant().fields[FieldIdx::from_u32(0)]; - let f0_ty = f0.ty(fx.tcx, substs); - - match f0_ty.kind() { - ty::Array(_, _) => { - assert_eq!(field.as_u32(), 0); - return CPlace { - inner: CPlaceInner::Var(local, var), - layout: layout.field(fx, field.as_u32().try_into().unwrap()), - }; - } - _ => { - return CPlace { - inner: CPlaceInner::VarLane( - local, - var, - field.as_u32().try_into().unwrap(), - ), - layout: layout.field(fx, field.as_u32().try_into().unwrap()), - }; - } - } - } - _ => {} - }, CPlaceInner::VarPair(local, var1, var2) => { let layout = layout.field(&*fx, field.index()); @@ -738,7 +661,12 @@ impl<'tcx> CPlace<'tcx> { _ => {} } - let (base, extra) = self.to_ptr_maybe_unsized(); + let (base, extra) = match self.inner { + CPlaceInner::Addr(ptr, extra) => (ptr, extra), + CPlaceInner::Var(_, _) | CPlaceInner::VarPair(_, _, _) => { + bug!("Expected CPlace::Addr, found {:?}", self) + } + }; let (field_ptr, field_layout) = codegen_field(fx, base, extra, layout, field); if field_layout.is_unsized() { @@ -767,15 +695,8 @@ impl<'tcx> CPlace<'tcx> { assert!(lane_idx < lane_count); match self.inner { - CPlaceInner::Var(local, var) => { - assert!(matches!(layout.abi, Abi::Vector { .. })); - CPlace { - inner: CPlaceInner::VarLane(local, var, lane_idx.try_into().unwrap()), - layout: lane_layout, - } - } + CPlaceInner::Var(_, _) => unreachable!(), CPlaceInner::VarPair(_, _, _) => unreachable!(), - CPlaceInner::VarLane(_, _, _) => unreachable!(), CPlaceInner::Addr(ptr, None) => { let field_offset = lane_layout.size * lane_idx; let field_ptr = ptr.offset_i64(fx, i64::try_from(field_offset.bytes()).unwrap()); @@ -794,34 +715,13 @@ impl<'tcx> CPlace<'tcx> { ty::Array(elem_ty, _) => { let elem_layout = fx.layout_of(*elem_ty); match self.inner { - CPlaceInner::Var(local, var) => { - // This is a hack to handle `vector_val.0[1]`. It doesn't allow dynamic - // indexing. - let lane_idx = match fx.bcx.func.dfg.insts - [fx.bcx.func.dfg.value_def(index).unwrap_inst()] - { - InstructionData::UnaryImm { opcode: Opcode::Iconst, imm } => imm, - _ => bug!( - "Dynamic indexing into a vector type is not supported: {self:?}[{index}]" - ), - }; - return CPlace { - inner: CPlaceInner::VarLane( - local, - var, - lane_idx.bits().try_into().unwrap(), - ), - layout: elem_layout, - }; - } CPlaceInner::Addr(addr, None) => (elem_layout, addr), - CPlaceInner::Addr(_, Some(_)) - | CPlaceInner::VarPair(_, _, _) - | CPlaceInner::VarLane(_, _, _) => bug!("Can't index into {self:?}"), + CPlaceInner::Var(_, _) + | CPlaceInner::Addr(_, Some(_)) + | CPlaceInner::VarPair(_, _, _) => bug!("Can't index into {self:?}"), } - // FIXME use VarLane in case of Var with simd type } - ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_maybe_unsized().0), + ty::Slice(elem_ty) => (fx.layout_of(*elem_ty), self.to_ptr_unsized().0), _ => bug!("place_index({:?})", self.layout().ty), }; @@ -846,12 +746,8 @@ impl<'tcx> CPlace<'tcx> { layout: TyAndLayout<'tcx>, ) -> CValue<'tcx> { if has_ptr_meta(fx.tcx, self.layout().ty) { - let (ptr, extra) = self.to_ptr_maybe_unsized(); - CValue::by_val_pair( - ptr.get_addr(fx), - extra.expect("unsized type without metadata"), - layout, - ) + let (ptr, extra) = self.to_ptr_unsized(); + CValue::by_val_pair(ptr.get_addr(fx), extra, layout) } else { CValue::by_val(self.to_ptr().get_addr(fx), layout) } |