diff options
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src')
30 files changed, 1323 insertions, 1032 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs index 37d2679c1..7f4619b5c 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs @@ -24,7 +24,7 @@ pub(super) fn add_arg_comment<'tcx>( local: Option<mir::Local>, local_field: Option<usize>, params: &[Value], - arg_abi_mode: PassMode, + arg_abi_mode: &PassMode, arg_layout: TyAndLayout<'tcx>, ) { if !fx.clif_comments.enabled() { diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs index 815450f68..0497c2570 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs @@ -342,7 +342,7 @@ pub(crate) fn codegen_terminator_call<'tcx>( let ret_place = codegen_place(fx, destination); - // Handle special calls like instrinsics and empty drop glue. + // Handle special calls like intrinsics and empty drop glue. let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() { let instance = ty::Instance::resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs) .unwrap() diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs index 6c10baa53..96e25d3a8 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs @@ -23,7 +23,7 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam { (RegKind::Integer, 9..=16) => types::I128, (RegKind::Float, 4) => types::F32, (RegKind::Float, 8) => types::F64, - (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(), + (RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(), _ => unreachable!("{:?}", reg), }; AbiParam::new(clif_ty) @@ -38,7 +38,7 @@ fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) - param } -fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> { +fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> { let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 { (0, 0) } else { @@ -100,7 +100,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(cast) => cast_target_to_abi_params(cast), + PassMode::Cast(ref cast, pad_i32) => { + assert!(!pad_i32, "padding support not yet implemented"); + cast_target_to_abi_params(cast) + } PassMode::Indirect { attrs, extra_attrs: None, on_stack } => { if on_stack { // Abi requires aligning struct size to pointer size @@ -145,7 +148,9 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { } _ => unreachable!("{:?}", self.layout.abi), }, - PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()), + PassMode::Cast(ref cast, _) => { + (None, cast_target_to_abi_params(cast).into_iter().collect()) + } PassMode::Indirect { attrs: _, extra_attrs: None, on_stack } => { assert!(!on_stack); (Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![]) @@ -160,7 +165,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> { pub(super) fn to_casted_value<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, arg: CValue<'tcx>, - cast: CastTarget, + cast: &CastTarget, ) -> SmallVec<[Value; 2]> { let (ptr, meta) = arg.force_stack(fx); assert!(meta.is_none()); @@ -179,12 +184,12 @@ pub(super) fn from_casted_value<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, block_params: &[Value], layout: TyAndLayout<'tcx>, - cast: CastTarget, + cast: &CastTarget, ) -> CValue<'tcx> { let abi_params = cast_target_to_abi_params(cast); let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum(); let layout_size = u32::try_from(layout.size.bytes()).unwrap(); - let stack_slot = fx.bcx.create_stack_slot(StackSlotData { + let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to // specify stack slot alignment. @@ -193,7 +198,7 @@ pub(super) fn from_casted_value<'tcx>( // larger alignment than the integer. size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16, }); - let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0)); + let ptr = Pointer::stack_slot(stack_slot); let mut offset = 0; let mut block_params_iter = block_params.iter().copied(); for param in abi_params { @@ -224,7 +229,7 @@ pub(super) fn adjust_arg_for_abi<'tcx>( let (a, b) = arg.load_scalar_pair(fx); smallvec![a, b] } - PassMode::Cast(cast) => to_casted_value(fx, arg, cast), + PassMode::Cast(ref cast, _) => to_casted_value(fx, arg, cast), PassMode::Indirect { .. } => { if is_owned { match arg.force_stack(fx) { @@ -268,7 +273,7 @@ pub(super) fn cvalue_for_param<'tcx>( local, local_field, &block_params, - arg_abi.mode, + &arg_abi.mode, arg_abi.layout, ); @@ -282,7 +287,9 @@ pub(super) fn cvalue_for_param<'tcx>( assert_eq!(block_params.len(), 2, "{:?}", block_params); Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout)) } - PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)), + PassMode::Cast(ref cast, _) => { + Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)) + } PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => { assert_eq!(block_params.len(), 1, "{:?}", block_params); Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout)) diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs index ff3bb2dfd..aaa141876 100644 --- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs +++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs @@ -13,7 +13,7 @@ pub(super) fn codegen_return_param<'tcx>( block_params_iter: &mut impl Iterator<Item = Value>, ) -> CPlace<'tcx> { let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode { - PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => { + PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => { let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa; ( super::make_local_place( @@ -44,7 +44,7 @@ pub(super) fn codegen_return_param<'tcx>( Some(RETURN_PLACE), None, &ret_param, - fx.fn_abi.as_ref().unwrap().ret.mode, + &fx.fn_abi.as_ref().unwrap().ret.mode, fx.fn_abi.as_ref().unwrap().ret.layout, ); @@ -75,7 +75,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => { unreachable!("unsized return value") } - PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => (None, None), + PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => (None, None), }; let call_inst = f(fx, return_ptr); @@ -92,7 +92,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>( ret_place .write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout)); } - PassMode::Cast(cast) => { + PassMode::Cast(ref cast, _) => { let results = fx.bcx.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>(); let result = @@ -131,7 +131,7 @@ pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) { let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx); fx.bcx.ins().return_(&[ret_val_a, ret_val_b]); } - PassMode::Cast(cast) => { + PassMode::Cast(ref cast, _) => { let place = fx.get_local_place(RETURN_PLACE); let ret_val = place.to_cvalue(fx); let ret_vals = super::pass_mode::to_casted_value(fx, ret_val, cast); diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs index 35b89358b..0cbb9f3ec 100644 --- a/compiler/rustc_codegen_cranelift/src/analyze.rs +++ b/compiler/rustc_codegen_cranelift/src/analyze.rs @@ -26,7 +26,7 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> { }) .collect::<IndexVec<Local, SsaKind>>(); - for bb in fx.mir.basic_blocks().iter() { + for bb in fx.mir.basic_blocks.iter() { for stmt in bb.statements.iter() { match &stmt.kind { Assign(place_and_rval) => match &place_and_rval.1 { diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs index b4c790961..fb5313f17 100644 --- a/compiler/rustc_codegen_cranelift/src/archive.rs +++ b/compiler/rustc_codegen_cranelift/src/archive.rs @@ -38,6 +38,7 @@ impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder { _lib_name: &str, _dll_imports: &[rustc_session::cstore::DllImport], _tmpdir: &Path, + _is_direct_dependency: bool, ) -> PathBuf { bug!("creating dll imports is not supported"); } diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs index 122e103ff..399474d79 100644 --- a/compiler/rustc_codegen_cranelift/src/base.rs +++ b/compiler/rustc_codegen_cranelift/src/base.rs @@ -6,21 +6,43 @@ use rustc_middle::ty::adjustment::PointerCast; use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::print::with_no_trimmed_paths; -use indexmap::IndexSet; - use crate::constant::ConstantCx; +use crate::debuginfo::FunctionDebugContext; use crate::prelude::*; use crate::pretty_clif::CommentWriter; -pub(crate) fn codegen_fn<'tcx>( - cx: &mut crate::CodegenCx<'tcx>, +pub(crate) struct CodegenedFunction { + symbol_name: String, + func_id: FuncId, + func: Function, + clif_comments: CommentWriter, + func_debug_cx: Option<FunctionDebugContext>, +} + +#[cfg_attr(not(feature = "jit"), allow(dead_code))] +pub(crate) fn codegen_and_compile_fn<'tcx>( + tcx: TyCtxt<'tcx>, + cx: &mut crate::CodegenCx, + cached_context: &mut Context, module: &mut dyn Module, instance: Instance<'tcx>, ) { - let tcx = cx.tcx; - let _inst_guard = crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name)); + + let cached_func = std::mem::replace(&mut cached_context.func, Function::new()); + let codegened_func = codegen_fn(tcx, cx, cached_func, module, instance); + + compile_fn(cx, cached_context, module, codegened_func); +} + +pub(crate) fn codegen_fn<'tcx>( + tcx: TyCtxt<'tcx>, + cx: &mut crate::CodegenCx, + cached_func: Function, + module: &mut dyn Module, + instance: Instance<'tcx>, +) -> CodegenedFunction { debug_assert!(!instance.substs.needs_infer()); let mir = tcx.instance_mir(instance.def); @@ -34,15 +56,14 @@ pub(crate) fn codegen_fn<'tcx>( }); // Declare function - let symbol_name = tcx.symbol_name(instance); + let symbol_name = tcx.symbol_name(instance).name.to_string(); let sig = get_function_sig(tcx, module.isa().triple(), instance); - let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap(); - - cx.cached_context.clear(); + let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap(); // Make the FunctionBuilder let mut func_ctx = FunctionBuilderContext::new(); - let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new()); + let mut func = cached_func; + func.clear(); func.name = ExternalName::user(0, func_id.as_u32()); func.signature = sig; func.collect_debug_info(); @@ -52,13 +73,19 @@ pub(crate) fn codegen_fn<'tcx>( // Predefine blocks let start_block = bcx.create_block(); let block_map: IndexVec<BasicBlock, Block> = - (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect(); + (0..mir.basic_blocks.len()).map(|_| bcx.create_block()).collect(); // Make FunctionCx let target_config = module.target_config(); let pointer_type = target_config.pointer_type(); let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance); + let func_debug_cx = if let Some(debug_context) = &mut cx.debug_context { + Some(debug_context.define_function(tcx, &symbol_name, mir.span)) + } else { + None + }; + let mut fx = FunctionCx { cx, module, @@ -66,6 +93,7 @@ pub(crate) fn codegen_fn<'tcx>( target_config, pointer_type, constants_cx: ConstantCx::new(), + func_debug_cx, instance, symbol_name, @@ -78,81 +106,48 @@ pub(crate) fn codegen_fn<'tcx>( caller_location: None, // set by `codegen_fn_prelude` clif_comments, - source_info_set: indexmap::IndexSet::new(), + last_source_file: None, next_ssa_var: 0, }; - let arg_uninhabited = fx - .mir - .args_iter() - .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); - - if !crate::constant::check_constants(&mut fx) { - fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); - fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); - crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted"); - } else if arg_uninhabited { - fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); - fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); - fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); - } else { - tcx.sess.time("codegen clif ir", || { - tcx.sess - .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block)); - codegen_fn_content(&mut fx); - }); - } + tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block)); // Recover all necessary data from fx, before accessing func will prevent future access to it. - let instance = fx.instance; + let symbol_name = fx.symbol_name; let clif_comments = fx.clif_comments; - let source_info_set = fx.source_info_set; - let local_map = fx.local_map; + let func_debug_cx = fx.func_debug_cx; fx.constants_cx.finalize(fx.tcx, &mut *fx.module); - crate::pretty_clif::write_clif_file( - tcx, - "unopt", - module.isa(), - instance, - &func, - &clif_comments, - ); + if cx.should_write_ir { + crate::pretty_clif::write_clif_file( + tcx.output_filenames(()), + &symbol_name, + "unopt", + module.isa(), + &func, + &clif_comments, + ); + } // Verify function verify_func(tcx, &clif_comments, &func); - compile_fn( - cx, - module, - instance, - symbol_name.name, - func_id, - func, - clif_comments, - source_info_set, - local_map, - ); + CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx } } -fn compile_fn<'tcx>( - cx: &mut crate::CodegenCx<'tcx>, +pub(crate) fn compile_fn( + cx: &mut crate::CodegenCx, + cached_context: &mut Context, module: &mut dyn Module, - instance: Instance<'tcx>, - symbol_name: &str, - func_id: FuncId, - func: Function, - mut clif_comments: CommentWriter, - source_info_set: IndexSet<SourceInfo>, - local_map: IndexVec<mir::Local, CPlace<'tcx>>, + codegened_func: CodegenedFunction, ) { - let tcx = cx.tcx; + let clif_comments = codegened_func.clif_comments; // Store function in context - let context = &mut cx.cached_context; + let context = cached_context; context.clear(); - context.func = func; + context.func = codegened_func.func; // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128` // instruction, which doesn't have an encoding. @@ -164,17 +159,6 @@ fn compile_fn<'tcx>( // invalidate it when it would change. context.domtree.clear(); - // Perform rust specific optimizations - tcx.sess.time("optimize clif ir", || { - crate::optimize::optimize_function( - tcx, - module.isa(), - instance, - context, - &mut clif_comments, - ); - }); - #[cfg(any())] // This is never true let _clif_guard = { use std::fmt::Write; @@ -203,46 +187,44 @@ fn compile_fn<'tcx>( }; // Define function - tcx.sess.time("define function", || { - context.want_disasm = crate::pretty_clif::should_write_ir(tcx); - module.define_function(func_id, context).unwrap(); + cx.profiler.verbose_generic_activity("define function").run(|| { + context.want_disasm = cx.should_write_ir; + module.define_function(codegened_func.func_id, context).unwrap(); }); - // Write optimized function to file for debugging - crate::pretty_clif::write_clif_file( - tcx, - "opt", - module.isa(), - instance, - &context.func, - &clif_comments, - ); + if cx.should_write_ir { + // Write optimized function to file for debugging + crate::pretty_clif::write_clif_file( + &cx.output_filenames, + &codegened_func.symbol_name, + "opt", + module.isa(), + &context.func, + &clif_comments, + ); - if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm { - crate::pretty_clif::write_ir_file( - tcx, - || format!("{}.vcode", tcx.symbol_name(instance).name), - |file| file.write_all(disasm.as_bytes()), - ) + if let Some(disasm) = &context.compiled_code().unwrap().disasm { + crate::pretty_clif::write_ir_file( + &cx.output_filenames, + &format!("{}.vcode", codegened_func.symbol_name), + |file| file.write_all(disasm.as_bytes()), + ) + } } // Define debuginfo for function let isa = module.isa(); let debug_context = &mut cx.debug_context; let unwind_context = &mut cx.unwind_context; - tcx.sess.time("generate debug info", || { + cx.profiler.verbose_generic_activity("generate debug info").run(|| { if let Some(debug_context) = debug_context { - debug_context.define_function( - instance, - func_id, - symbol_name, - isa, + codegened_func.func_debug_cx.unwrap().finalize( + debug_context, + codegened_func.func_id, context, - &source_info_set, - local_map, ); } - unwind_context.add_function(func_id, &context, isa); + unwind_context.add_function(codegened_func.func_id, &context, isa); }); } @@ -268,8 +250,28 @@ pub(crate) fn verify_func( }); } -fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { - for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() { +fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) { + if !crate::constant::check_constants(fx) { + fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); + fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); + // compilation should have been aborted + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + return; + } + + let arg_uninhabited = fx + .mir + .args_iter() + .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited()); + if arg_uninhabited { + fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]); + fx.bcx.switch_to_block(fx.block_map[START_BLOCK]); + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + return; + } + fx.tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(fx, start_block)); + + for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() { let block = fx.get_block(bb); fx.bcx.switch_to_block(block); @@ -457,17 +459,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) { template, operands, *options, + *destination, ); - - match *destination { - Some(destination) => { - let destination_block = fx.get_block(destination); - fx.bcx.ins().jump(destination_block, &[]); - } - None => { - fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); - } - } } TerminatorKind::Resume | TerminatorKind::Abort => { // FIXME implement unwinding @@ -708,12 +701,14 @@ fn codegen_stmt<'tcx>( let operand = codegen_operand(fx, operand); operand.unsize_value(fx, lval); } + Rvalue::Cast(CastKind::DynStar, _, _) => { + // FIXME(dyn-star) + unimplemented!() + } Rvalue::Discriminant(place) => { let place = codegen_place(fx, place); let value = place.to_cvalue(fx); - let discr = - crate::discriminant::codegen_get_discriminant(fx, value, dest_layout); - lval.write_cvalue(fx, discr); + crate::discriminant::codegen_get_discriminant(fx, lval, value, dest_layout); } Rvalue::Repeat(ref operand, times) => { let operand = codegen_operand(fx, operand); @@ -803,20 +798,31 @@ fn codegen_stmt<'tcx>( | StatementKind::AscribeUserType(..) => {} StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"), - StatementKind::CopyNonOverlapping(inner) => { - let dst = codegen_operand(fx, &inner.dst); - let pointee = dst - .layout() - .pointee_info_at(fx, rustc_target::abi::Size::ZERO) - .expect("Expected pointer"); - let dst = dst.load_scalar(fx); - let src = codegen_operand(fx, &inner.src).load_scalar(fx); - let count = codegen_operand(fx, &inner.count).load_scalar(fx); - let elem_size: u64 = pointee.size.bytes(); - let bytes = - if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count }; - fx.bcx.call_memcpy(fx.target_config, dst, src, bytes); - } + StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic { + // We ignore `assume` intrinsics, they are only useful for optimizations + NonDivergingIntrinsic::Assume(_) => {} + NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping { + src, + dst, + count, + }) => { + let dst = codegen_operand(fx, dst); + let pointee = dst + .layout() + .pointee_info_at(fx, rustc_target::abi::Size::ZERO) + .expect("Expected pointer"); + let dst = dst.load_scalar(fx); + let src = codegen_operand(fx, src).load_scalar(fx); + let count = codegen_operand(fx, count).load_scalar(fx); + let elem_size: u64 = pointee.size.bytes(); + let bytes = if elem_size != 1 { + fx.bcx.ins().imul_imm(count, elem_size as i64) + } else { + count + }; + fx.bcx.call_memcpy(fx.target_config, dst, src, bytes); + } + }, } } @@ -934,8 +940,11 @@ pub(crate) fn codegen_panic_inner<'tcx>( args: &[Value], span: Span, ) { - let def_id = - fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s)); + let def_id = fx + .tcx + .lang_items() + .require(lang_item) + .unwrap_or_else(|e| fx.tcx.sess.span_fatal(span, e.to_string())); let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx); let symbol_name = fx.tcx.symbol_name(instance).name; diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs index f9dc1b516..589594465 100644 --- a/compiler/rustc_codegen_cranelift/src/common.rs +++ b/compiler/rustc_codegen_cranelift/src/common.rs @@ -1,14 +1,18 @@ use cranelift_codegen::isa::TargetFrontendConfig; +use gimli::write::FileId; + +use rustc_data_structures::sync::Lrc; use rustc_index::vec::IndexVec; use rustc_middle::ty::layout::{ FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers, }; -use rustc_middle::ty::SymbolName; +use rustc_span::SourceFile; use rustc_target::abi::call::FnAbi; use rustc_target::abi::{Integer, Primitive}; use rustc_target::spec::{HasTargetSpec, Target}; use crate::constant::ConstantCx; +use crate::debuginfo::FunctionDebugContext; use crate::prelude::*; pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type { @@ -74,7 +78,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ _ => unreachable!(), }; - match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) { + match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) { // Cranelift currently only implements icmp for 128bit vectors. Some(vector_ty) if vector_ty.bits() == 128 => vector_ty, _ => return None, @@ -232,15 +236,16 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool { } pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { - pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>, + pub(crate) cx: &'clif mut crate::CodegenCx, pub(crate) module: &'m mut dyn Module, pub(crate) tcx: TyCtxt<'tcx>, pub(crate) target_config: TargetFrontendConfig, // Cached from module pub(crate) pointer_type: Type, // Cached from module pub(crate) constants_cx: ConstantCx, + pub(crate) func_debug_cx: Option<FunctionDebugContext>, pub(crate) instance: Instance<'tcx>, - pub(crate) symbol_name: SymbolName<'tcx>, + pub(crate) symbol_name: String, pub(crate) mir: &'tcx Body<'tcx>, pub(crate) fn_abi: Option<&'tcx FnAbi<'tcx, Ty<'tcx>>>, @@ -252,7 +257,11 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> { pub(crate) caller_location: Option<CValue<'tcx>>, pub(crate) clif_comments: crate::pretty_clif::CommentWriter, - pub(crate) source_info_set: indexmap::IndexSet<SourceInfo>, + + /// Last accessed source file and it's debuginfo file id. + /// + /// For optimization purposes only + pub(crate) last_source_file: Option<(Lrc<SourceFile>, FileId)>, /// This should only be accessed by `CPlace::new_var`. pub(crate) next_ssa_var: u32, @@ -336,8 +345,31 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> { } pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) { - let (index, _) = self.source_info_set.insert_full(source_info); - self.bcx.set_srcloc(SourceLoc::new(index as u32)); + if let Some(debug_context) = &mut self.cx.debug_context { + let (file, line, column) = + DebugContext::get_span_loc(self.tcx, self.mir.span, source_info.span); + + // add_source_file is very slow. + // Optimize for the common case of the current file not being changed. + let mut cached_file_id = None; + if let Some((ref last_source_file, last_file_id)) = self.last_source_file { + // If the allocations are not equal, the files may still be equal, but that + // doesn't matter, as this is just an optimization. + if rustc_data_structures::sync::Lrc::ptr_eq(last_source_file, &file) { + cached_file_id = Some(last_file_id); + } + } + + let file_id = if let Some(file_id) = cached_file_id { + file_id + } else { + debug_context.add_source_file(&file) + }; + + let source_loc = + self.func_debug_cx.as_mut().unwrap().add_dbg_loc(file_id, line, column); + self.bcx.set_srcloc(source_loc); + } } // Note: must be kept in sync with get_caller_location from cg_ssa diff --git a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs new file mode 100644 index 000000000..dfde97920 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs @@ -0,0 +1,168 @@ +use std::sync::{Arc, Condvar, Mutex}; + +use rustc_session::Session; + +use jobserver::HelperThread; + +// FIXME don't panic when a worker thread panics + +pub(super) struct ConcurrencyLimiter { + helper_thread: Option<HelperThread>, + state: Arc<Mutex<state::ConcurrencyLimiterState>>, + available_token_condvar: Arc<Condvar>, +} + +impl ConcurrencyLimiter { + pub(super) fn new(sess: &Session, pending_jobs: usize) -> Self { + let state = Arc::new(Mutex::new(state::ConcurrencyLimiterState::new(pending_jobs))); + let available_token_condvar = Arc::new(Condvar::new()); + + let state_helper = state.clone(); + let available_token_condvar_helper = available_token_condvar.clone(); + let helper_thread = sess + .jobserver + .clone() + .into_helper_thread(move |token| { + let mut state = state_helper.lock().unwrap(); + state.add_new_token(token.unwrap()); + available_token_condvar_helper.notify_one(); + }) + .unwrap(); + ConcurrencyLimiter { + helper_thread: Some(helper_thread), + state, + available_token_condvar: Arc::new(Condvar::new()), + } + } + + pub(super) fn acquire(&mut self) -> ConcurrencyLimiterToken { + let mut state = self.state.lock().unwrap(); + loop { + state.assert_invariants(); + + if state.try_start_job() { + return ConcurrencyLimiterToken { + state: self.state.clone(), + available_token_condvar: self.available_token_condvar.clone(), + }; + } + + self.helper_thread.as_mut().unwrap().request_token(); + state = self.available_token_condvar.wait(state).unwrap(); + } + } + + pub(super) fn job_already_done(&mut self) { + let mut state = self.state.lock().unwrap(); + state.job_already_done(); + } +} + +impl Drop for ConcurrencyLimiter { + fn drop(&mut self) { + // + self.helper_thread.take(); + + // Assert that all jobs have finished + let state = Mutex::get_mut(Arc::get_mut(&mut self.state).unwrap()).unwrap(); + state.assert_done(); + } +} + +#[derive(Debug)] +pub(super) struct ConcurrencyLimiterToken { + state: Arc<Mutex<state::ConcurrencyLimiterState>>, + available_token_condvar: Arc<Condvar>, +} + +impl Drop for ConcurrencyLimiterToken { + fn drop(&mut self) { + let mut state = self.state.lock().unwrap(); + state.job_finished(); + self.available_token_condvar.notify_one(); + } +} + +mod state { + use jobserver::Acquired; + + #[derive(Debug)] + pub(super) struct ConcurrencyLimiterState { + pending_jobs: usize, + active_jobs: usize, + + // None is used to represent the implicit token, Some to represent explicit tokens + tokens: Vec<Option<Acquired>>, + } + + impl ConcurrencyLimiterState { + pub(super) fn new(pending_jobs: usize) -> Self { + ConcurrencyLimiterState { pending_jobs, active_jobs: 0, tokens: vec![None] } + } + + pub(super) fn assert_invariants(&self) { + // There must be no excess active jobs + assert!(self.active_jobs <= self.pending_jobs); + + // There may not be more active jobs than there are tokens + assert!(self.active_jobs <= self.tokens.len()); + } + + pub(super) fn assert_done(&self) { + assert_eq!(self.pending_jobs, 0); + assert_eq!(self.active_jobs, 0); + } + + pub(super) fn add_new_token(&mut self, token: Acquired) { + self.tokens.push(Some(token)); + self.drop_excess_capacity(); + } + + pub(super) fn try_start_job(&mut self) -> bool { + if self.active_jobs < self.tokens.len() { + // Using existing token + self.job_started(); + return true; + } + + false + } + + pub(super) fn job_started(&mut self) { + self.assert_invariants(); + self.active_jobs += 1; + self.drop_excess_capacity(); + self.assert_invariants(); + } + + pub(super) fn job_finished(&mut self) { + self.assert_invariants(); + self.pending_jobs -= 1; + self.active_jobs -= 1; + self.assert_invariants(); + self.drop_excess_capacity(); + self.assert_invariants(); + } + + pub(super) fn job_already_done(&mut self) { + self.assert_invariants(); + self.pending_jobs -= 1; + self.assert_invariants(); + self.drop_excess_capacity(); + self.assert_invariants(); + } + + fn drop_excess_capacity(&mut self) { + self.assert_invariants(); + + // Drop all tokens that can never be used anymore + self.tokens.truncate(std::cmp::max(self.pending_jobs, 1)); + + // Keep some excess tokens to satisfy requests faster + const MAX_EXTRA_CAPACITY: usize = 2; + self.tokens.truncate(std::cmp::max(self.active_jobs + MAX_EXTRA_CAPACITY, 1)); + + self.assert_invariants(); + } + } +} diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs index 7f7fd0e9c..6b4ed9b9d 100644 --- a/compiler/rustc_codegen_cranelift/src/constant.rs +++ b/compiler/rustc_codegen_cranelift/src/constant.rs @@ -41,36 +41,30 @@ impl ConstantCx { pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool { let mut all_constants_ok = true; for constant in &fx.mir.required_consts { - let const_ = match fx.monomorphize(constant.literal) { - ConstantKind::Ty(ct) => ct, + let unevaluated = match fx.monomorphize(constant.literal) { + ConstantKind::Ty(ct) => match ct.kind() { + ConstKind::Unevaluated(uv) => uv.expand(), + ConstKind::Value(_) => continue, + ConstKind::Param(_) + | ConstKind::Infer(_) + | ConstKind::Bound(_, _) + | ConstKind::Placeholder(_) + | ConstKind::Error(_) => unreachable!("{:?}", ct), + }, + ConstantKind::Unevaluated(uv, _) => uv, ConstantKind::Val(..) => continue, }; - match const_.kind() { - ConstKind::Value(_) => {} - ConstKind::Unevaluated(unevaluated) => { - if let Err(err) = - fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) - { - all_constants_ok = false; - match err { - ErrorHandled::Reported(_) | ErrorHandled::Linted => { - fx.tcx.sess.span_err(constant.span, "erroneous constant encountered"); - } - ErrorHandled::TooGeneric => { - span_bug!( - constant.span, - "codgen encountered polymorphic constant: {:?}", - err - ); - } - } + + if let Err(err) = fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) { + all_constants_ok = false; + match err { + ErrorHandled::Reported(_) | ErrorHandled::Linted => { + fx.tcx.sess.span_err(constant.span, "erroneous constant encountered"); + } + ErrorHandled::TooGeneric => { + span_bug!(constant.span, "codegen encountered polymorphic constant: {:?}", err); } } - ConstKind::Param(_) - | ConstKind::Infer(_) - | ConstKind::Bound(_, _) - | ConstKind::Placeholder(_) - | ConstKind::Error(_) => unreachable!("{:?}", const_), } } all_constants_ok @@ -122,36 +116,28 @@ pub(crate) fn codegen_constant<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, constant: &Constant<'tcx>, ) -> CValue<'tcx> { - let const_ = match fx.monomorphize(constant.literal) { - ConstantKind::Ty(ct) => ct, - ConstantKind::Val(val, ty) => return codegen_const_value(fx, val, ty), - }; - let const_val = match const_.kind() { - ConstKind::Value(valtree) => fx.tcx.valtree_to_const_val((const_.ty(), valtree)), - ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted }) + let (const_val, ty) = match fx.monomorphize(constant.literal) { + ConstantKind::Ty(const_) => unreachable!("{:?}", const_), + ConstantKind::Unevaluated(ty::Unevaluated { def, substs, promoted }, ty) if fx.tcx.is_static(def.did) => { assert!(substs.is_empty()); assert!(promoted.is_none()); - return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty())).to_cvalue(fx); + return codegen_static_ref(fx, def.did, fx.layout_of(ty)).to_cvalue(fx); } - ConstKind::Unevaluated(unevaluated) => { + ConstantKind::Unevaluated(unevaluated, ty) => { match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) { - Ok(const_val) => const_val, + Ok(const_val) => (const_val, ty), Err(_) => { span_bug!(constant.span, "erroneous constant not captured by required_consts"); } } } - ConstKind::Param(_) - | ConstKind::Infer(_) - | ConstKind::Bound(_, _) - | ConstKind::Placeholder(_) - | ConstKind::Error(_) => unreachable!("{:?}", const_), + ConstantKind::Val(val, ty) => (val, ty), }; - codegen_const_value(fx, const_val, const_.ty()) + codegen_const_value(fx, const_val, ty) } pub(crate) fn codegen_const_value<'tcx>( @@ -430,7 +416,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec(); data_ctx.define(bytes.into_boxed_slice()); - for &(offset, alloc_id) in alloc.relocations().iter() { + for &(offset, alloc_id) in alloc.provenance().iter() { let addend = { let endianness = tcx.data_layout.endian; let offset = offset.bytes() as usize; @@ -496,6 +482,9 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( .eval_for_mir(fx.tcx, ParamEnv::reveal_all()) .try_to_value(fx.tcx), ConstantKind::Val(val, _) => Some(val), + ConstantKind::Unevaluated(uv, _) => { + fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), uv, None).ok() + } }, // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored // inside a temporary before being passed to the intrinsic requiring the const argument. @@ -505,7 +494,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( return None; } let mut computed_const_val = None; - for bb_data in fx.mir.basic_blocks() { + for bb_data in fx.mir.basic_blocks.iter() { for stmt in &bb_data.statements { match &stmt.kind { StatementKind::Assign(local_and_rvalue) if &local_and_rvalue.0 == place => { @@ -536,9 +525,11 @@ pub(crate) fn mir_operand_get_const_val<'tcx>( { return None; } - StatementKind::CopyNonOverlapping(_) => { - return None; - } // conservative handling + StatementKind::Intrinsic(ref intrinsic) => match **intrinsic { + NonDivergingIntrinsic::CopyNonOverlapping(..) => return None, + NonDivergingIntrinsic::Assume(..) => {} + }, + // conservative handling StatementKind::Assign(_) | StatementKind::FakeRead(_) | StatementKind::SetDiscriminant { .. } diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs index 589910ede..9583cd2ec 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs @@ -9,7 +9,7 @@ use gimli::{RunTimeEndian, SectionId}; use super::object::WriteDebugInfo; use super::DebugContext; -impl DebugContext<'_> { +impl DebugContext { pub(crate) fn emit(&mut self, product: &mut ObjectProduct) { let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone()); let root = self.dwarf.unit.root(); diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs index bbcb95913..463de6a91 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs @@ -3,8 +3,10 @@ use std::ffi::OsStr; use std::path::{Component, Path}; +use crate::debuginfo::FunctionDebugContext; use crate::prelude::*; +use rustc_data_structures::sync::Lrc; use rustc_span::{ FileName, Pos, SourceFile, SourceFileAndLine, SourceFileHash, SourceFileHashAlgorithm, }; @@ -14,7 +16,6 @@ use cranelift_codegen::MachSrcLoc; use gimli::write::{ Address, AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable, - UnitEntryId, }; // OPTIMIZATION: It is cheaper to do this in one pass than using `.parent()` and `.file_name()`. @@ -47,9 +48,9 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] { } } -pub(crate) const MD5_LEN: usize = 16; +const MD5_LEN: usize = 16; -pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> { +fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> { if hash.kind == SourceFileHashAlgorithm::Md5 { let mut buf = [0u8; MD5_LEN]; buf.copy_from_slice(hash.hash_bytes()); @@ -59,160 +60,132 @@ pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> { } } -fn line_program_add_file( - line_program: &mut LineProgram, - line_strings: &mut LineStringTable, - file: &SourceFile, -) -> FileId { - match &file.name { - FileName::Real(path) => { - let (dir_path, file_name) = split_path_dir_and_file(path.remapped_path_if_available()); - let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str()); - let file_name = osstr_as_utf8_bytes(file_name); - - let dir_id = if !dir_name.is_empty() { - let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings); - line_program.add_directory(dir_name) - } else { - line_program.default_directory() - }; - let file_name = LineString::new(file_name, line_program.encoding(), line_strings); +impl DebugContext { + pub(crate) fn get_span_loc( + tcx: TyCtxt<'_>, + function_span: Span, + span: Span, + ) -> (Lrc<SourceFile>, u64, u64) { + // Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131 + // In order to have a good line stepping behavior in debugger, we overwrite debug + // locations of macro expansions with that of the outermost expansion site (when the macro is + // annotated with `#[collapse_debuginfo]` or when `-Zdebug-macros` is provided). + let span = if tcx.should_collapse_debuginfo(span) { + span + } else { + // Walk up the macro expansion chain until we reach a non-expanded span. + // We also stop at the function body level because no line stepping can occur + // at the level above that. + rustc_span::hygiene::walk_chain(span, function_span.ctxt()) + }; - let info = make_file_info(file.src_hash); + match tcx.sess.source_map().lookup_line(span.lo()) { + Ok(SourceFileAndLine { sf: file, line }) => { + let line_pos = file.line_begin_pos(span.lo()); - line_program.file_has_md5 &= info.is_some(); - line_program.add_file(file_name, dir_id, info) + ( + file, + u64::try_from(line).unwrap() + 1, + u64::from((span.lo() - line_pos).to_u32()) + 1, + ) + } + Err(file) => (file, 0, 0), } - // FIXME give more appropriate file names - filename => { - let dir_id = line_program.default_directory(); - let dummy_file_name = LineString::new( - filename.prefer_remapped().to_string().into_bytes(), - line_program.encoding(), - line_strings, - ); - line_program.add_file(dummy_file_name, dir_id, None) + } + + pub(crate) fn add_source_file(&mut self, source_file: &SourceFile) -> FileId { + let line_program: &mut LineProgram = &mut self.dwarf.unit.line_program; + let line_strings: &mut LineStringTable = &mut self.dwarf.line_strings; + + match &source_file.name { + FileName::Real(path) => { + let (dir_path, file_name) = + split_path_dir_and_file(path.remapped_path_if_available()); + let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str()); + let file_name = osstr_as_utf8_bytes(file_name); + + let dir_id = if !dir_name.is_empty() { + let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings); + line_program.add_directory(dir_name) + } else { + line_program.default_directory() + }; + let file_name = LineString::new(file_name, line_program.encoding(), line_strings); + + let info = make_file_info(source_file.src_hash); + + line_program.file_has_md5 &= info.is_some(); + line_program.add_file(file_name, dir_id, info) + } + // FIXME give more appropriate file names + filename => { + let dir_id = line_program.default_directory(); + let dummy_file_name = LineString::new( + filename.prefer_remapped().to_string().into_bytes(), + line_program.encoding(), + line_strings, + ); + line_program.add_file(dummy_file_name, dir_id, None) + } } } } -impl<'tcx> DebugContext<'tcx> { - pub(super) fn emit_location(&mut self, entry_id: UnitEntryId, span: Span) { - let loc = self.tcx.sess.source_map().lookup_char_pos(span.lo()); - - let file_id = line_program_add_file( - &mut self.dwarf.unit.line_program, - &mut self.dwarf.line_strings, - &loc.file, - ); - - let entry = self.dwarf.unit.get_mut(entry_id); - - entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id))); - entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(loc.line as u64)); - entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(loc.col.to_usize() as u64)); +impl FunctionDebugContext { + pub(crate) fn add_dbg_loc(&mut self, file_id: FileId, line: u64, column: u64) -> SourceLoc { + let (index, _) = self.source_loc_set.insert_full((file_id, line, column)); + SourceLoc::new(u32::try_from(index).unwrap()) } pub(super) fn create_debug_lines( &mut self, + debug_context: &mut DebugContext, symbol: usize, - entry_id: UnitEntryId, context: &Context, - function_span: Span, - source_info_set: &indexmap::IndexSet<SourceInfo>, ) -> CodeOffset { - let tcx = self.tcx; - let line_program = &mut self.dwarf.unit.line_program; - - let line_strings = &mut self.dwarf.line_strings; - let mut last_span = None; - let mut last_file = None; - let mut create_row_for_span = |line_program: &mut LineProgram, span: Span| { - if let Some(last_span) = last_span { - if span == last_span { - line_program.generate_row(); - return; - } - } - last_span = Some(span); - - // Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131 - // In order to have a good line stepping behavior in debugger, we overwrite debug - // locations of macro expansions with that of the outermost expansion site - // (unless the crate is being compiled with `-Z debug-macros`). - let span = if !span.from_expansion() || tcx.sess.opts.unstable_opts.debug_macros { - span - } else { - // Walk up the macro expansion chain until we reach a non-expanded span. - // We also stop at the function body level because no line stepping can occur - // at the level above that. - rustc_span::hygiene::walk_chain(span, function_span.ctxt()) + let create_row_for_span = + |debug_context: &mut DebugContext, source_loc: (FileId, u64, u64)| { + let (file_id, line, col) = source_loc; + + debug_context.dwarf.unit.line_program.row().file = file_id; + debug_context.dwarf.unit.line_program.row().line = line; + debug_context.dwarf.unit.line_program.row().column = col; + debug_context.dwarf.unit.line_program.generate_row(); }; - let (file, line, col) = match tcx.sess.source_map().lookup_line(span.lo()) { - Ok(SourceFileAndLine { sf: file, line }) => { - let line_pos = file.line_begin_pos(span.lo()); - - ( - file, - u64::try_from(line).unwrap() + 1, - u64::from((span.lo() - line_pos).to_u32()) + 1, - ) - } - Err(file) => (file, 0, 0), - }; - - // line_program_add_file is very slow. - // Optimize for the common case of the current file not being changed. - let current_file_changed = if let Some(last_file) = &last_file { - // If the allocations are not equal, then the files may still be equal, but that - // is not a problem, as this is just an optimization. - !rustc_data_structures::sync::Lrc::ptr_eq(last_file, &file) - } else { - true - }; - if current_file_changed { - let file_id = line_program_add_file(line_program, line_strings, &file); - line_program.row().file = file_id; - last_file = Some(file); - } - - line_program.row().line = line; - line_program.row().column = col; - line_program.generate_row(); - }; - - line_program.begin_sequence(Some(Address::Symbol { symbol, addend: 0 })); + debug_context + .dwarf + .unit + .line_program + .begin_sequence(Some(Address::Symbol { symbol, addend: 0 })); let mut func_end = 0; - let mcr = context.mach_compile_result.as_ref().unwrap(); + let mcr = context.compiled_code().unwrap(); for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() { - line_program.row().address_offset = u64::from(start); + debug_context.dwarf.unit.line_program.row().address_offset = u64::from(start); if !loc.is_default() { - let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap(); - create_row_for_span(line_program, source_info.span); + let source_loc = *self.source_loc_set.get_index(loc.bits() as usize).unwrap(); + create_row_for_span(debug_context, source_loc); } else { - create_row_for_span(line_program, function_span); + create_row_for_span(debug_context, self.function_source_loc); } func_end = end; } - line_program.end_sequence(u64::from(func_end)); + debug_context.dwarf.unit.line_program.end_sequence(u64::from(func_end)); let func_end = mcr.buffer.total_size(); assert_ne!(func_end, 0); - let entry = self.dwarf.unit.get_mut(entry_id); + let entry = debug_context.dwarf.unit.get_mut(self.entry_id); entry.set( gimli::DW_AT_low_pc, AttributeValue::Address(Address::Symbol { symbol, addend: 0 }), ); entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(func_end))); - self.emit_location(entry_id, function_span); - func_end } } diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs index 693092ba5..c55db2017 100644 --- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs @@ -7,35 +7,34 @@ mod unwind; use crate::prelude::*; -use rustc_index::vec::IndexVec; - -use cranelift_codegen::entity::EntityRef; -use cranelift_codegen::ir::{Endianness, LabelValueLoc, ValueLabel}; +use cranelift_codegen::ir::Endianness; use cranelift_codegen::isa::TargetIsa; -use cranelift_codegen::ValueLocRange; use gimli::write::{ - Address, AttributeValue, DwarfUnit, Expression, LineProgram, LineString, Location, - LocationList, Range, RangeList, UnitEntryId, + Address, AttributeValue, DwarfUnit, FileId, LineProgram, LineString, Range, RangeList, + UnitEntryId, }; -use gimli::{Encoding, Format, LineEncoding, RunTimeEndian, X86_64}; +use gimli::{Encoding, Format, LineEncoding, RunTimeEndian}; +use indexmap::IndexSet; pub(crate) use emit::{DebugReloc, DebugRelocName}; pub(crate) use unwind::UnwindContext; -pub(crate) struct DebugContext<'tcx> { - tcx: TyCtxt<'tcx>, - +pub(crate) struct DebugContext { endian: RunTimeEndian, dwarf: DwarfUnit, unit_range_list: RangeList, +} - types: FxHashMap<Ty<'tcx>, UnitEntryId>, +pub(crate) struct FunctionDebugContext { + entry_id: UnitEntryId, + function_source_loc: (FileId, u64, u64), + source_loc_set: indexmap::IndexSet<(FileId, u64, u64)>, } -impl<'tcx> DebugContext<'tcx> { - pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self { +impl DebugContext { + pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa) -> Self { let encoding = Encoding { format: Format::Dwarf32, // FIXME this should be configurable @@ -101,127 +100,18 @@ impl<'tcx> DebugContext<'tcx> { root.set(gimli::DW_AT_low_pc, AttributeValue::Address(Address::Constant(0))); } - DebugContext { - tcx, - - endian, - - dwarf, - unit_range_list: RangeList(Vec::new()), - - types: FxHashMap::default(), - } - } - - fn dwarf_ty(&mut self, ty: Ty<'tcx>) -> UnitEntryId { - if let Some(type_id) = self.types.get(&ty) { - return *type_id; - } - - let new_entry = |dwarf: &mut DwarfUnit, tag| dwarf.unit.add(dwarf.unit.root(), tag); - - let primitive = |dwarf: &mut DwarfUnit, ate| { - let type_id = new_entry(dwarf, gimli::DW_TAG_base_type); - let type_entry = dwarf.unit.get_mut(type_id); - type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(ate)); - type_id - }; - - let name = format!("{}", ty); - let layout = self.tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap(); - - let type_id = match ty.kind() { - ty::Bool => primitive(&mut self.dwarf, gimli::DW_ATE_boolean), - ty::Char => primitive(&mut self.dwarf, gimli::DW_ATE_UTF), - ty::Uint(_) => primitive(&mut self.dwarf, gimli::DW_ATE_unsigned), - ty::Int(_) => primitive(&mut self.dwarf, gimli::DW_ATE_signed), - ty::Float(_) => primitive(&mut self.dwarf, gimli::DW_ATE_float), - ty::Ref(_, pointee_ty, _mutbl) - | ty::RawPtr(ty::TypeAndMut { ty: pointee_ty, mutbl: _mutbl }) => { - let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_pointer_type); - - // Ensure that type is inserted before recursing to avoid duplicates - self.types.insert(ty, type_id); - - let pointee = self.dwarf_ty(*pointee_ty); - - let type_entry = self.dwarf.unit.get_mut(type_id); - - //type_entry.set(gimli::DW_AT_mutable, AttributeValue::Flag(mutbl == rustc_hir::Mutability::Mut)); - type_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(pointee)); - - type_id - } - ty::Adt(adt_def, _substs) if adt_def.is_struct() && !layout.is_unsized() => { - let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type); - - // Ensure that type is inserted before recursing to avoid duplicates - self.types.insert(ty, type_id); - - let variant = adt_def.non_enum_variant(); - - for (field_idx, field_def) in variant.fields.iter().enumerate() { - let field_offset = layout.fields.offset(field_idx); - let field_layout = layout.field( - &layout::LayoutCx { tcx: self.tcx, param_env: ParamEnv::reveal_all() }, - field_idx, - ); - - let field_type = self.dwarf_ty(field_layout.ty); - - let field_id = self.dwarf.unit.add(type_id, gimli::DW_TAG_member); - let field_entry = self.dwarf.unit.get_mut(field_id); - - field_entry.set( - gimli::DW_AT_name, - AttributeValue::String(field_def.name.as_str().to_string().into_bytes()), - ); - field_entry.set( - gimli::DW_AT_data_member_location, - AttributeValue::Udata(field_offset.bytes()), - ); - field_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(field_type)); - } - - type_id - } - _ => new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type), - }; - - let type_entry = self.dwarf.unit.get_mut(type_id); - - type_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes())); - type_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes())); - - self.types.insert(ty, type_id); - - type_id - } - - fn define_local(&mut self, scope: UnitEntryId, name: String, ty: Ty<'tcx>) -> UnitEntryId { - let dw_ty = self.dwarf_ty(ty); - - let var_id = self.dwarf.unit.add(scope, gimli::DW_TAG_variable); - let var_entry = self.dwarf.unit.get_mut(var_id); - - var_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes())); - var_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty)); - - var_id + DebugContext { endian, dwarf, unit_range_list: RangeList(Vec::new()) } } pub(crate) fn define_function( &mut self, - instance: Instance<'tcx>, - func_id: FuncId, + tcx: TyCtxt<'_>, name: &str, - isa: &dyn TargetIsa, - context: &Context, - source_info_set: &indexmap::IndexSet<SourceInfo>, - local_map: IndexVec<mir::Local, CPlace<'tcx>>, - ) { - let symbol = func_id.as_u32() as usize; - let mir = self.tcx.instance_mir(instance.def); + function_span: Span, + ) -> FunctionDebugContext { + let (file, line, column) = DebugContext::get_span_loc(tcx, function_span, function_span); + + let file_id = self.add_source_file(&file); // FIXME: add to appropriate scope instead of root let scope = self.dwarf.unit.root(); @@ -233,14 +123,35 @@ impl<'tcx> DebugContext<'tcx> { entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id)); entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(name_id)); - let end = self.create_debug_lines(symbol, entry_id, context, mir.span, source_info_set); + entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id))); + entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(line)); + entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(column)); - self.unit_range_list.0.push(Range::StartLength { + FunctionDebugContext { + entry_id, + function_source_loc: (file_id, line, column), + source_loc_set: IndexSet::new(), + } + } +} + +impl FunctionDebugContext { + pub(crate) fn finalize( + mut self, + debug_context: &mut DebugContext, + func_id: FuncId, + context: &Context, + ) { + let symbol = func_id.as_u32() as usize; + + let end = self.create_debug_lines(debug_context, symbol, context); + + debug_context.unit_range_list.0.push(Range::StartLength { begin: Address::Symbol { symbol, addend: 0 }, length: u64::from(end), }); - let func_entry = self.dwarf.unit.get_mut(entry_id); + let func_entry = debug_context.dwarf.unit.get_mut(self.entry_id); // Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped. func_entry.set( gimli::DW_AT_low_pc, @@ -248,110 +159,5 @@ impl<'tcx> DebugContext<'tcx> { ); // Using Udata for DW_AT_high_pc requires at least DWARF4 func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end))); - - // FIXME make it more reliable and implement scopes before re-enabling this. - if false { - let value_labels_ranges = std::collections::HashMap::new(); // FIXME - - for (local, _local_decl) in mir.local_decls.iter_enumerated() { - let ty = self.tcx.subst_and_normalize_erasing_regions( - instance.substs, - ty::ParamEnv::reveal_all(), - mir.local_decls[local].ty, - ); - let var_id = self.define_local(entry_id, format!("{:?}", local), ty); - - let location = place_location( - self, - isa, - symbol, - &local_map, - &value_labels_ranges, - Place { local, projection: ty::List::empty() }, - ); - - let var_entry = self.dwarf.unit.get_mut(var_id); - var_entry.set(gimli::DW_AT_location, location); - } - } - - // FIXME create locals for all entries in mir.var_debug_info - } -} - -fn place_location<'tcx>( - debug_context: &mut DebugContext<'tcx>, - isa: &dyn TargetIsa, - symbol: usize, - local_map: &IndexVec<mir::Local, CPlace<'tcx>>, - #[allow(rustc::default_hash_types)] value_labels_ranges: &std::collections::HashMap< - ValueLabel, - Vec<ValueLocRange>, - >, - place: Place<'tcx>, -) -> AttributeValue { - assert!(place.projection.is_empty()); // FIXME implement them - - match local_map[place.local].inner() { - CPlaceInner::Var(_local, var) => { - let value_label = cranelift_codegen::ir::ValueLabel::new(var.index()); - if let Some(value_loc_ranges) = value_labels_ranges.get(&value_label) { - let loc_list = LocationList( - value_loc_ranges - .iter() - .map(|value_loc_range| Location::StartEnd { - begin: Address::Symbol { - symbol, - addend: i64::from(value_loc_range.start), - }, - end: Address::Symbol { symbol, addend: i64::from(value_loc_range.end) }, - data: translate_loc(isa, value_loc_range.loc).unwrap(), - }) - .collect(), - ); - let loc_list_id = debug_context.dwarf.unit.locations.add(loc_list); - - AttributeValue::LocationListRef(loc_list_id) - } else { - // FIXME set value labels for unused locals - - AttributeValue::Exprloc(Expression::new()) - } - } - CPlaceInner::VarPair(_, _, _) => { - // FIXME implement this - - AttributeValue::Exprloc(Expression::new()) - } - CPlaceInner::VarLane(_, _, _) => { - // FIXME implement this - - AttributeValue::Exprloc(Expression::new()) - } - CPlaceInner::Addr(_, _) => { - // FIXME implement this (used by arguments and returns) - - AttributeValue::Exprloc(Expression::new()) - - // For PointerBase::Stack: - //AttributeValue::Exprloc(translate_loc(ValueLoc::Stack(*stack_slot)).unwrap()) - } - } -} - -// Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137 -fn translate_loc(isa: &dyn TargetIsa, loc: LabelValueLoc) -> Option<Expression> { - match loc { - LabelValueLoc::Reg(reg) => { - let machine_reg = isa.map_regalloc_reg_to_dwarf(reg).unwrap(); - let mut expr = Expression::new(); - expr.op_reg(gimli::Register(machine_reg)); - Some(expr) - } - LabelValueLoc::SPOffset(offset) => { - let mut expr = Expression::new(); - expr.op_breg(X86_64::RSP, offset); - Some(expr) - } } } diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs index f619bb5ed..97b395bcd 100644 --- a/compiler/rustc_codegen_cranelift/src/discriminant.rs +++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs @@ -42,10 +42,10 @@ pub(crate) fn codegen_set_discriminant<'tcx>( Variants::Multiple { tag: _, tag_field, - tag_encoding: TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start }, + tag_encoding: TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start }, variants: _, } => { - if variant_index != dataful_variant { + if variant_index != untagged_variant { let niche = place.place_field(fx, mir::Field::new(tag_field)); let niche_value = variant_index.as_u32() - niche_variants.start().as_u32(); let niche_value = ty::ScalarInt::try_from_uint( @@ -62,16 +62,14 @@ pub(crate) fn codegen_set_discriminant<'tcx>( pub(crate) fn codegen_get_discriminant<'tcx>( fx: &mut FunctionCx<'_, '_, 'tcx>, + dest: CPlace<'tcx>, value: CValue<'tcx>, dest_layout: TyAndLayout<'tcx>, -) -> CValue<'tcx> { +) { let layout = value.layout(); - if layout.abi == Abi::Uninhabited { - let true_ = fx.bcx.ins().iconst(types::I32, 1); - fx.bcx.ins().trapnz(true_, TrapCode::UnreachableCodeReached); - // Return a dummy value - return CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout); + if layout.abi.is_uninhabited() { + return; } let (tag_scalar, tag_field, tag_encoding) = match &layout.variants { @@ -89,7 +87,9 @@ pub(crate) fn codegen_get_discriminant<'tcx>( } else { ty::ScalarInt::try_from_uint(discr_val, dest_layout.size).unwrap() }; - return CValue::const_val(fx, dest_layout, discr_val); + let res = CValue::const_val(fx, dest_layout, discr_val); + dest.write_cvalue(fx, res); + return; } Variants::Multiple { tag, tag_field, tag_encoding, variants: _ } => { (tag, *tag_field, tag_encoding) @@ -110,9 +110,10 @@ pub(crate) fn codegen_get_discriminant<'tcx>( _ => false, }; let val = clif_intcast(fx, tag, cast_to, signed); - CValue::by_val(val, dest_layout) + let res = CValue::by_val(val, dest_layout); + dest.write_cvalue(fx, res); } - TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => { + TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => { // Rebase from niche values to discriminants, and check // whether the result is in range for the niche variants. @@ -168,9 +169,11 @@ pub(crate) fn codegen_get_discriminant<'tcx>( fx.bcx.ins().iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32())) }; - let dataful_variant = fx.bcx.ins().iconst(cast_to, i64::from(dataful_variant.as_u32())); - let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant); - CValue::by_val(discr, dest_layout) + let untagged_variant = + fx.bcx.ins().iconst(cast_to, i64::from(untagged_variant.as_u32())); + let discr = fx.bcx.ins().select(is_niche, niche_discr, untagged_variant); + let res = CValue::by_val(discr, dest_layout); + dest.write_cvalue(fx, res); } } } diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs index 3cd1ef563..8eabe1cbc 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs @@ -1,33 +1,129 @@ //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a //! standalone executable. +use std::fs::File; use std::path::PathBuf; +use std::sync::Arc; +use std::thread::JoinHandle; -use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file; use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind}; +use rustc_data_structures::profiling::SelfProfilerRef; use rustc_data_structures::stable_hasher::{HashStable, StableHasher}; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; use rustc_middle::mir::mono::{CodegenUnit, MonoItem}; use rustc_session::cgu_reuse_tracker::CguReuse; -use rustc_session::config::{DebugInfo, OutputType}; +use rustc_session::config::{DebugInfo, OutputFilenames, OutputType}; use rustc_session::Session; -use cranelift_codegen::isa::TargetIsa; use cranelift_object::{ObjectBuilder, ObjectModule}; +use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken}; +use crate::global_asm::GlobalAsmConfig; use crate::{prelude::*, BackendConfig}; -struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>); +struct ModuleCodegenResult { + module_regular: CompiledModule, + module_global_asm: Option<CompiledModule>, + existing_work_product: Option<(WorkProductId, WorkProduct)>, +} + +enum OngoingModuleCodegen { + Sync(Result<ModuleCodegenResult, String>), + Async(JoinHandle<Result<ModuleCodegenResult, String>>), +} -impl<HCX> HashStable<HCX> for ModuleCodegenResult { +impl<HCX> HashStable<HCX> for OngoingModuleCodegen { fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) { // do nothing } } -fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectModule { +pub(crate) struct OngoingCodegen { + modules: Vec<OngoingModuleCodegen>, + allocator_module: Option<CompiledModule>, + metadata_module: Option<CompiledModule>, + metadata: EncodedMetadata, + crate_info: CrateInfo, + concurrency_limiter: ConcurrencyLimiter, +} + +impl OngoingCodegen { + pub(crate) fn join( + self, + sess: &Session, + backend_config: &BackendConfig, + ) -> (CodegenResults, FxHashMap<WorkProductId, WorkProduct>) { + let mut work_products = FxHashMap::default(); + let mut modules = vec![]; + + for module_codegen in self.modules { + let module_codegen_result = match module_codegen { + OngoingModuleCodegen::Sync(module_codegen_result) => module_codegen_result, + OngoingModuleCodegen::Async(join_handle) => match join_handle.join() { + Ok(module_codegen_result) => module_codegen_result, + Err(panic) => std::panic::resume_unwind(panic), + }, + }; + + let module_codegen_result = match module_codegen_result { + Ok(module_codegen_result) => module_codegen_result, + Err(err) => sess.fatal(&err), + }; + let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } = + module_codegen_result; + + if let Some((work_product_id, work_product)) = existing_work_product { + work_products.insert(work_product_id, work_product); + } else { + let work_product = if backend_config.disable_incr_cache { + None + } else if let Some(module_global_asm) = &module_global_asm { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + sess, + &module_regular.name, + &[ + ("o", &module_regular.object.as_ref().unwrap()), + ("asm.o", &module_global_asm.object.as_ref().unwrap()), + ], + ) + } else { + rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( + sess, + &module_regular.name, + &[("o", &module_regular.object.as_ref().unwrap())], + ) + }; + if let Some((work_product_id, work_product)) = work_product { + work_products.insert(work_product_id, work_product); + } + } + + modules.push(module_regular); + if let Some(module_global_asm) = module_global_asm { + modules.push(module_global_asm); + } + } + + drop(self.concurrency_limiter); + + ( + CodegenResults { + modules, + allocator_module: self.allocator_module, + metadata_module: self.metadata_module, + metadata: self.metadata, + crate_info: self.crate_info, + }, + work_products, + ) + } +} + +fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> ObjectModule { + let isa = crate::build_isa(sess, backend_config); + let mut builder = ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap(); // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size @@ -37,15 +133,15 @@ fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectM ObjectModule::new(builder) } -fn emit_module( - tcx: TyCtxt<'_>, - backend_config: &BackendConfig, +fn emit_cgu( + output_filenames: &OutputFilenames, + prof: &SelfProfilerRef, name: String, - kind: ModuleKind, module: ObjectModule, - debug: Option<DebugContext<'_>>, + debug: Option<DebugContext>, unwind_context: UnwindContext, -) -> ModuleCodegenResult { + global_asm_object_file: Option<PathBuf>, +) -> Result<ModuleCodegenResult, String> { let mut product = module.finish(); if let Some(mut debug) = debug { @@ -54,134 +150,191 @@ fn emit_module( unwind_context.emit(&mut product); - let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name)); - let obj = product.object.write().unwrap(); + let module_regular = + emit_module(output_filenames, prof, product.object, ModuleKind::Regular, name.clone())?; + + Ok(ModuleCodegenResult { + module_regular, + module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule { + name: format!("{name}.asm"), + kind: ModuleKind::Regular, + object: Some(global_asm_object_file), + dwarf_object: None, + bytecode: None, + }), + existing_work_product: None, + }) +} - tcx.sess.prof.artifact_size("object_file", name.clone(), obj.len().try_into().unwrap()); +fn emit_module( + output_filenames: &OutputFilenames, + prof: &SelfProfilerRef, + object: cranelift_object::object::write::Object<'_>, + kind: ModuleKind, + name: String, +) -> Result<CompiledModule, String> { + let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name)); + let mut file = match File::create(&tmp_file) { + Ok(file) => file, + Err(err) => return Err(format!("error creating object file: {}", err)), + }; - if let Err(err) = std::fs::write(&tmp_file, obj) { - tcx.sess.fatal(&format!("error writing object file: {}", err)); + if let Err(err) = object.write_stream(&mut file) { + return Err(format!("error writing object file: {}", err)); } - let work_product = if backend_config.disable_incr_cache { - None - } else { - rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir( - tcx.sess, - &name, - &[("o", &tmp_file)], - ) - }; + prof.artifact_size("object_file", &*name, file.metadata().unwrap().len()); - ModuleCodegenResult( - CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None }, - work_product, - ) + Ok(CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None }) } fn reuse_workproduct_for_cgu( tcx: TyCtxt<'_>, cgu: &CodegenUnit<'_>, - work_products: &mut FxHashMap<WorkProductId, WorkProduct>, -) -> CompiledModule { +) -> Result<ModuleCodegenResult, String> { let work_product = cgu.previous_work_product(tcx); - let obj_out = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); - let source_file = rustc_incremental::in_incr_comp_dir_sess( + let obj_out_regular = + tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str())); + let source_file_regular = rustc_incremental::in_incr_comp_dir_sess( &tcx.sess, &work_product.saved_files.get("o").expect("no saved object file in work product"), ); - if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) { - tcx.sess.err(&format!( + + if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) { + return Err(format!( "unable to copy {} to {}: {}", - source_file.display(), - obj_out.display(), + source_file_regular.display(), + obj_out_regular.display(), err )); } + let obj_out_global_asm = + crate::global_asm::add_file_stem_postfix(obj_out_regular.clone(), ".asm"); + let has_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") { + let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, asm_o); + if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm) + { + return Err(format!( + "unable to copy {} to {}: {}", + source_file_regular.display(), + obj_out_regular.display(), + err + )); + } + true + } else { + false + }; - work_products.insert(cgu.work_product_id(), work_product); - - CompiledModule { - name: cgu.name().to_string(), - kind: ModuleKind::Regular, - object: Some(obj_out), - dwarf_object: None, - bytecode: None, - } + Ok(ModuleCodegenResult { + module_regular: CompiledModule { + name: cgu.name().to_string(), + kind: ModuleKind::Regular, + object: Some(obj_out_regular), + dwarf_object: None, + bytecode: None, + }, + module_global_asm: if has_global_asm { + Some(CompiledModule { + name: cgu.name().to_string(), + kind: ModuleKind::Regular, + object: Some(obj_out_global_asm), + dwarf_object: None, + bytecode: None, + }) + } else { + None + }, + existing_work_product: Some((cgu.work_product_id(), work_product)), + }) } fn module_codegen( tcx: TyCtxt<'_>, - (backend_config, cgu_name): (BackendConfig, rustc_span::Symbol), -) -> ModuleCodegenResult { - let cgu = tcx.codegen_unit(cgu_name); - let mono_items = cgu.items_in_deterministic_order(tcx); - - let isa = crate::build_isa(tcx.sess, &backend_config); - let mut module = make_module(tcx.sess, isa, cgu_name.as_str().to_string()); - - let mut cx = crate::CodegenCx::new( - tcx, - backend_config.clone(), - module.isa(), - tcx.sess.opts.debuginfo != DebugInfo::None, - cgu_name, - ); - super::predefine_mono_items(tcx, &mut module, &mono_items); - for (mono_item, _) in mono_items { - match mono_item { - MonoItem::Fn(inst) => { - cx.tcx - .sess - .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst)); - } - MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id), - MonoItem::GlobalAsm(item_id) => { - let item = cx.tcx.hir().item(item_id); - if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind { - if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { - cx.global_asm.push_str("\n.intel_syntax noprefix\n"); - } else { - cx.global_asm.push_str("\n.att_syntax\n"); - } - for piece in asm.template { - match *piece { - InlineAsmTemplatePiece::String(ref s) => cx.global_asm.push_str(s), - InlineAsmTemplatePiece::Placeholder { .. } => todo!(), - } - } - cx.global_asm.push_str("\n.att_syntax\n\n"); - } else { - bug!("Expected GlobalAsm found {:?}", item); + (backend_config, global_asm_config, cgu_name, token): ( + BackendConfig, + Arc<GlobalAsmConfig>, + rustc_span::Symbol, + ConcurrencyLimiterToken, + ), +) -> OngoingModuleCodegen { + let (cgu_name, mut cx, mut module, codegened_functions) = tcx.sess.time("codegen cgu", || { + let cgu = tcx.codegen_unit(cgu_name); + let mono_items = cgu.items_in_deterministic_order(tcx); + + let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string()); + + let mut cx = crate::CodegenCx::new( + tcx, + backend_config.clone(), + module.isa(), + tcx.sess.opts.debuginfo != DebugInfo::None, + cgu_name, + ); + super::predefine_mono_items(tcx, &mut module, &mono_items); + let mut codegened_functions = vec![]; + for (mono_item, _) in mono_items { + match mono_item { + MonoItem::Fn(inst) => { + tcx.sess.time("codegen fn", || { + let codegened_function = crate::base::codegen_fn( + tcx, + &mut cx, + Function::new(), + &mut module, + inst, + ); + codegened_functions.push(codegened_function); + }); + } + MonoItem::Static(def_id) => { + crate::constant::codegen_static(tcx, &mut module, def_id) + } + MonoItem::GlobalAsm(item_id) => { + crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id); } } } - } - crate::main_shim::maybe_create_entry_wrapper( - tcx, - &mut module, - &mut cx.unwind_context, - false, - cgu.is_primary(), - ); - - let debug_context = cx.debug_context; - let unwind_context = cx.unwind_context; - let codegen_result = tcx.sess.time("write object file", || { - emit_module( + crate::main_shim::maybe_create_entry_wrapper( tcx, - &backend_config, - cgu.name().as_str().to_string(), - ModuleKind::Regular, - module, - debug_context, - unwind_context, - ) + &mut module, + &mut cx.unwind_context, + false, + cgu.is_primary(), + ); + + let cgu_name = cgu.name().as_str().to_owned(); + + (cgu_name, cx, module, codegened_functions) }); - codegen_global_asm(tcx, cgu.name().as_str(), &cx.global_asm); + OngoingModuleCodegen::Async(std::thread::spawn(move || { + cx.profiler.clone().verbose_generic_activity("compile functions").run(|| { + let mut cached_context = Context::new(); + for codegened_func in codegened_functions { + crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func); + } + }); - codegen_result + let global_asm_object_file = + cx.profiler.verbose_generic_activity("compile assembly").run(|| { + crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm) + })?; + + let codegen_result = cx.profiler.verbose_generic_activity("write object file").run(|| { + emit_cgu( + &global_asm_config.output_filenames, + &cx.profiler, + cgu_name, + module, + cx.debug_context, + cx.unwind_context, + global_asm_object_file, + ) + }); + std::mem::drop(token); + codegen_result + })) } pub(crate) fn run_aot( @@ -189,9 +342,7 @@ pub(crate) fn run_aot( backend_config: BackendConfig, metadata: EncodedMetadata, need_metadata_module: bool, -) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> { - let mut work_products = FxHashMap::default(); - +) -> Box<OngoingCodegen> { let cgus = if tcx.sess.opts.output_types.should_codegen() { tcx.collect_and_partition_mono_items(()).1 } else { @@ -206,62 +357,69 @@ pub(crate) fn run_aot( } } + let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx)); + + let mut concurrency_limiter = ConcurrencyLimiter::new(tcx.sess, cgus.len()); + let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || { cgus.iter() .map(|cgu| { - let cgu_reuse = determine_cgu_reuse(tcx, cgu); + let cgu_reuse = if backend_config.disable_incr_cache { + CguReuse::No + } else { + determine_cgu_reuse(tcx, cgu) + }; tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse); match cgu_reuse { - _ if backend_config.disable_incr_cache => {} - CguReuse::No => {} - CguReuse::PreLto => { - return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products); + CguReuse::No => { + let dep_node = cgu.codegen_dep_node(tcx); + tcx.dep_graph + .with_task( + dep_node, + tcx, + ( + backend_config.clone(), + global_asm_config.clone(), + cgu.name(), + concurrency_limiter.acquire(), + ), + module_codegen, + Some(rustc_middle::dep_graph::hash_result), + ) + .0 + } + CguReuse::PreLto => unreachable!(), + CguReuse::PostLto => { + concurrency_limiter.job_already_done(); + OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, &*cgu)) } - CguReuse::PostLto => unreachable!(), - } - - let dep_node = cgu.codegen_dep_node(tcx); - let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task( - dep_node, - tcx, - (backend_config.clone(), cgu.name()), - module_codegen, - Some(rustc_middle::dep_graph::hash_result), - ); - - if let Some((id, product)) = work_product { - work_products.insert(id, product); } - - module }) .collect::<Vec<_>>() }); tcx.sess.abort_if_errors(); - let isa = crate::build_isa(tcx.sess, &backend_config); - let mut allocator_module = make_module(tcx.sess, isa, "allocator_shim".to_string()); - assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type()); + let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string()); let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true); let created_alloc_shim = crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context); let allocator_module = if created_alloc_shim { - let ModuleCodegenResult(module, work_product) = emit_module( - tcx, - &backend_config, - "allocator_shim".to_string(), + let mut product = allocator_module.finish(); + allocator_unwind_context.emit(&mut product); + + match emit_module( + tcx.output_filenames(()), + &tcx.sess.prof, + product.object, ModuleKind::Allocator, - allocator_module, - None, - allocator_unwind_context, - ); - if let Some((id, product)) = work_product { - work_products.insert(id, product); + "allocator_shim".to_owned(), + ) { + Ok(allocator_module) => Some(allocator_module), + Err(err) => tcx.sess.fatal(err), } - Some(module) } else { None }; @@ -308,102 +466,14 @@ pub(crate) fn run_aot( } .to_owned(); - Box::new(( - CodegenResults { - modules, - allocator_module, - metadata_module, - metadata, - crate_info: CrateInfo::new(tcx, target_cpu), - }, - work_products, - )) -} - -fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) { - use std::io::Write; - use std::process::{Command, Stdio}; - - if global_asm.is_empty() { - return; - } - - if cfg!(not(feature = "inline_asm")) - || tcx.sess.target.is_like_osx - || tcx.sess.target.is_like_windows - { - if global_asm.contains("__rust_probestack") { - return; - } - - // FIXME fix linker error on macOS - if cfg!(not(feature = "inline_asm")) { - tcx.sess.fatal( - "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift", - ); - } else { - tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows"); - } - } - - let assembler = crate::toolchain::get_toolchain_binary(tcx.sess, "as"); - let linker = crate::toolchain::get_toolchain_binary(tcx.sess, "ld"); - - // Remove all LLVM style comments - let global_asm = global_asm - .lines() - .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line }) - .collect::<Vec<_>>() - .join("\n"); - - let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name)); - - // Assemble `global_asm` - let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); - let mut child = Command::new(assembler) - .arg("-o") - .arg(&global_asm_object_file) - .stdin(Stdio::piped()) - .spawn() - .expect("Failed to spawn `as`."); - child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); - let status = child.wait().expect("Failed to wait for `as`."); - if !status.success() { - tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm)); - } - - // Link the global asm and main object file together - let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main"); - std::fs::rename(&output_object_file, &main_object_file).unwrap(); - let status = Command::new(linker) - .arg("-r") // Create a new object file - .arg("-o") - .arg(output_object_file) - .arg(&main_object_file) - .arg(&global_asm_object_file) - .status() - .unwrap(); - if !status.success() { - tcx.sess.fatal(&format!( - "Failed to link `{}` and `{}` together", - main_object_file.display(), - global_asm_object_file.display(), - )); - } - - std::fs::remove_file(global_asm_object_file).unwrap(); - std::fs::remove_file(main_object_file).unwrap(); -} - -fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { - let mut new_filename = path.file_stem().unwrap().to_owned(); - new_filename.push(postfix); - if let Some(extension) = path.extension() { - new_filename.push("."); - new_filename.push(extension); - } - path.set_file_name(new_filename); - path + Box::new(OngoingCodegen { + modules, + allocator_module, + metadata_module, + metadata, + crate_info: CrateInfo::new(tcx, target_cpu), + concurrency_limiter, + }) } // Adapted from https://github.com/rust-lang/rust/blob/303d8aff6092709edd4dbd35b1c88e9aa40bf6d8/src/librustc_codegen_ssa/base.rs#L922-L953 @@ -432,5 +502,5 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR cgu.name() ); - if tcx.try_mark_green(&dep_node) { CguReuse::PreLto } else { CguReuse::No } + if tcx.try_mark_green(&dep_node) { CguReuse::PostLto } else { CguReuse::No } } diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs index a56a91000..0e77e4004 100644 --- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs +++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs @@ -61,11 +61,11 @@ impl UnsafeMessage { } } -fn create_jit_module<'tcx>( - tcx: TyCtxt<'tcx>, +fn create_jit_module( + tcx: TyCtxt<'_>, backend_config: &BackendConfig, hotswap: bool, -) -> (JITModule, CodegenCx<'tcx>) { +) -> (JITModule, CodegenCx) { let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string()); let imported_symbols = load_imported_symbols_for_jit(tcx.sess, crate_info); @@ -111,6 +111,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { &backend_config, matches!(backend_config.codegen_mode, CodegenMode::JitLazy), ); + let mut cached_context = Context::new(); let (_, cgus) = tcx.collect_and_partition_mono_items(()); let mono_items = cgus @@ -128,11 +129,19 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! { MonoItem::Fn(inst) => match backend_config.codegen_mode { CodegenMode::Aot => unreachable!(), CodegenMode::Jit => { - cx.tcx.sess.time("codegen fn", || { - crate::base::codegen_fn(&mut cx, &mut jit_module, inst) + tcx.sess.time("codegen fn", || { + crate::base::codegen_and_compile_fn( + tcx, + &mut cx, + &mut cached_context, + &mut jit_module, + inst, + ) }); } - CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst), + CodegenMode::JitLazy => { + codegen_shim(tcx, &mut cx, &mut cached_context, &mut jit_module, inst) + } }, MonoItem::Static(def_id) => { crate::constant::codegen_static(tcx, &mut jit_module, def_id); @@ -259,7 +268,15 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) -> false, Symbol::intern("dummy_cgu_name"), ); - tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance)); + tcx.sess.time("codegen fn", || { + crate::base::codegen_and_compile_fn( + tcx, + &mut cx, + &mut Context::new(), + jit_module, + instance, + ) + }); assert!(cx.global_asm.is_empty()); jit_module.finalize_definitions(); @@ -334,9 +351,13 @@ fn load_imported_symbols_for_jit( imported_symbols } -fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) { - let tcx = cx.tcx; - +fn codegen_shim<'tcx>( + tcx: TyCtxt<'tcx>, + cx: &mut CodegenCx, + cached_context: &mut Context, + module: &mut JITModule, + inst: Instance<'tcx>, +) { let pointer_type = module.target_config().pointer_type(); let name = tcx.symbol_name(inst).name; @@ -357,8 +378,9 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In ) .unwrap(); - cx.cached_context.clear(); - let trampoline = &mut cx.cached_context.func; + let context = cached_context; + context.clear(); + let trampoline = &mut context.func; trampoline.signature = sig.clone(); let mut builder_ctx = FunctionBuilderContext::new(); @@ -381,5 +403,6 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec(); trampoline_builder.ins().return_(&ret_vals); - module.define_function(func_id, &mut cx.cached_context).unwrap(); + module.define_function(func_id, context).unwrap(); + cx.unwind_context.add_function(func_id, context, module.isa()); } diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs new file mode 100644 index 000000000..dcbcaba30 --- /dev/null +++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs @@ -0,0 +1,114 @@ +//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a +//! standalone executable. + +use std::io::Write; +use std::path::PathBuf; +use std::process::{Command, Stdio}; +use std::sync::Arc; + +use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece}; +use rustc_hir::ItemId; +use rustc_session::config::{OutputFilenames, OutputType}; + +use crate::prelude::*; + +pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) { + let item = tcx.hir().item(item_id); + if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind { + if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) { + global_asm.push_str("\n.intel_syntax noprefix\n"); + } else { + global_asm.push_str("\n.att_syntax\n"); + } + for piece in asm.template { + match *piece { + InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s), + InlineAsmTemplatePiece::Placeholder { .. } => todo!(), + } + } + global_asm.push_str("\n.att_syntax\n\n"); + } else { + bug!("Expected GlobalAsm found {:?}", item); + } +} + +#[derive(Debug)] +pub(crate) struct GlobalAsmConfig { + asm_enabled: bool, + assembler: PathBuf, + pub(crate) output_filenames: Arc<OutputFilenames>, +} + +impl GlobalAsmConfig { + pub(crate) fn new(tcx: TyCtxt<'_>) -> Self { + let asm_enabled = cfg!(feature = "inline_asm") && !tcx.sess.target.is_like_windows; + + GlobalAsmConfig { + asm_enabled, + assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"), + output_filenames: tcx.output_filenames(()).clone(), + } + } +} + +pub(crate) fn compile_global_asm( + config: &GlobalAsmConfig, + cgu_name: &str, + global_asm: &str, +) -> Result<Option<PathBuf>, String> { + if global_asm.is_empty() { + return Ok(None); + } + + if !config.asm_enabled { + if global_asm.contains("__rust_probestack") { + return Ok(None); + } + + // FIXME fix linker error on macOS + if cfg!(not(feature = "inline_asm")) { + return Err( + "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift" + .to_owned(), + ); + } else { + return Err("asm! and global_asm! are not yet supported on Windows".to_owned()); + } + } + + // Remove all LLVM style comments + let global_asm = global_asm + .lines() + .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line }) + .collect::<Vec<_>>() + .join("\n"); + + let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name)); + + // Assemble `global_asm` + let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm"); + let mut child = Command::new(&config.assembler) + .arg("-o") + .arg(&global_asm_object_file) + .stdin(Stdio::piped()) + .spawn() + .expect("Failed to spawn `as`."); + child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap(); + let status = child.wait().expect("Failed to wait for `as`."); + if !status.success() { + return Err(format!("Failed to assemble `{}`", global_asm)); + } + + Ok(Some(global_asm_object_file)) +} + +pub(crate) fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf { + let mut new_filename = path.file_stem().unwrap().to_owned(); + new_filename.push(postfix); + if let Some(extension) = path.extension() { + new_filename.push("."); + new_filename.push(extension); + } + path.set_file_name(new_filename); + path +} diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs index 241de5e36..8b3d475cb 100644 --- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs +++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs @@ -15,15 +15,19 @@ pub(crate) fn codegen_inline_asm<'tcx>( template: &[InlineAsmTemplatePiece], operands: &[InlineAsmOperand<'tcx>], options: InlineAsmOptions, + destination: Option<mir::BasicBlock>, ) { // FIXME add .eh_frame unwind info directives if !template.is_empty() { + // Used by panic_abort if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) { - let true_ = fx.bcx.ins().iconst(types::I32, 1); - fx.bcx.ins().trapnz(true_, TrapCode::User(1)); + fx.bcx.ins().trap(TrapCode::User(1)); return; - } else if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string()) + } + + // Used by stdarch + if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string()) && matches!( template[1], InlineAsmTemplatePiece::Placeholder { @@ -47,51 +51,46 @@ pub(crate) fn codegen_inline_asm<'tcx>( { assert_eq!(operands.len(), 4); let (leaf, eax_place) = match operands[1] { - InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { - assert_eq!( - reg, - InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)) - ); - ( - crate::base::codegen_operand(fx, in_value).load_scalar(fx), - crate::base::codegen_place(fx, out_place.unwrap()), - ) - } + InlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), + late: true, + ref in_value, + out_place: Some(out_place), + } => ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place), + ), _ => unreachable!(), }; let ebx_place = match operands[0] { - InlineAsmOperand::Out { reg, late: true, place } => { - assert_eq!( - reg, + InlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86( - X86InlineAsmRegClass::reg - )) - ); - crate::base::codegen_place(fx, place.unwrap()) - } + X86InlineAsmRegClass::reg, + )), + late: true, + place: Some(place), + } => crate::base::codegen_place(fx, place), _ => unreachable!(), }; let (sub_leaf, ecx_place) = match operands[2] { - InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => { - assert_eq!( - reg, - InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)) - ); - ( - crate::base::codegen_operand(fx, in_value).load_scalar(fx), - crate::base::codegen_place(fx, out_place.unwrap()), - ) - } + InlineAsmOperand::InOut { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)), + late: true, + ref in_value, + out_place: Some(out_place), + } => ( + crate::base::codegen_operand(fx, in_value).load_scalar(fx), + crate::base::codegen_place(fx, out_place), + ), _ => unreachable!(), }; let edx_place = match operands[3] { - InlineAsmOperand::Out { reg, late: true, place } => { - assert_eq!( - reg, - InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)) - ); - crate::base::codegen_place(fx, place.unwrap()) - } + InlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), + late: true, + place: Some(place), + } => crate::base::codegen_place(fx, place), _ => unreachable!(), }; @@ -101,12 +100,99 @@ pub(crate) fn codegen_inline_asm<'tcx>( ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32))); ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32))); edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32))); + let destination_block = fx.get_block(destination.unwrap()); + fx.bcx.ins().jump(destination_block, &[]); return; - } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") { + } + + // Used by compiler-builtins + if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") { // ___chkstk, ___chkstk_ms and __alloca are only used on Windows crate::trap::trap_unimplemented(fx, "Stack probes are not supported"); + return; } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" { crate::trap::trap_unimplemented(fx, "Alloca is not supported"); + return; + } + + // Used by measureme + if template[0] == InlineAsmTemplatePiece::String("xor %eax, %eax".to_string()) + && template[1] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[2] == InlineAsmTemplatePiece::String("mov %rbx, ".to_string()) + && matches!( + template[3], + InlineAsmTemplatePiece::Placeholder { + operand_idx: 0, + modifier: Some('r'), + span: _ + } + ) + && template[4] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[5] == InlineAsmTemplatePiece::String("cpuid".to_string()) + && template[6] == InlineAsmTemplatePiece::String("\n".to_string()) + && template[7] == InlineAsmTemplatePiece::String("mov ".to_string()) + && matches!( + template[8], + InlineAsmTemplatePiece::Placeholder { + operand_idx: 0, + modifier: Some('r'), + span: _ + } + ) + && template[9] == InlineAsmTemplatePiece::String(", %rbx".to_string()) + { + let destination_block = fx.get_block(destination.unwrap()); + fx.bcx.ins().jump(destination_block, &[]); + return; + } else if template[0] == InlineAsmTemplatePiece::String("rdpmc".to_string()) { + // Return zero dummy values for all performance counters + match operands[0] { + InlineAsmOperand::In { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)), + value: _, + } => {} + _ => unreachable!(), + }; + let lo = match operands[1] { + InlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)), + late: true, + place: Some(place), + } => crate::base::codegen_place(fx, place), + _ => unreachable!(), + }; + let hi = match operands[2] { + InlineAsmOperand::Out { + reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)), + late: true, + place: Some(place), + } => crate::base::codegen_place(fx, place), + _ => unreachable!(), + }; + + let u32_layout = fx.layout_of(fx.tcx.types.u32); + let zero = fx.bcx.ins().iconst(types::I32, 0); + lo.write_cvalue(fx, CValue::by_val(zero, u32_layout)); + hi.write_cvalue(fx, CValue::by_val(zero, u32_layout)); + + let destination_block = fx.get_block(destination.unwrap()); + fx.bcx.ins().jump(destination_block, &[]); + return; + } else if template[0] == InlineAsmTemplatePiece::String("lock xadd ".to_string()) + && matches!( + template[1], + InlineAsmTemplatePiece::Placeholder { operand_idx: 1, modifier: None, span: _ } + ) + && template[2] == InlineAsmTemplatePiece::String(", (".to_string()) + && matches!( + template[3], + InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: None, span: _ } + ) + && template[4] == InlineAsmTemplatePiece::String(")".to_string()) + { + let destination_block = fx.get_block(destination.unwrap()); + fx.bcx.ins().jump(destination_block, &[]); + return; } } @@ -175,6 +261,16 @@ pub(crate) fn codegen_inline_asm<'tcx>( } call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs); + + match destination { + Some(destination) => { + let destination_block = fx.get_block(destination); + fx.bcx.ins().jump(destination_block, &[]); + } + None => { + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); + } + } } struct InlineAssemblyGenerator<'a, 'tcx> { @@ -637,7 +733,7 @@ fn call_inline_asm<'tcx>( inputs: Vec<(Size, Value)>, outputs: Vec<(Size, CPlace<'tcx>)>, ) { - let stack_slot = fx.bcx.func.create_stack_slot(StackSlotData { + let stack_slot = fx.bcx.func.create_sized_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, size: u32::try_from(slot_size.bytes()).unwrap(), }); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs index d02dfd93c..5120b89c4 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs @@ -62,7 +62,7 @@ pub(crate) fn codegen_cpuid_call<'tcx>( fx.bcx.ins().jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]); fx.bcx.switch_to_block(unsupported_leaf); - crate::trap::trap_unreachable( + crate::trap::trap_unimplemented( fx, "__cpuid_count arch intrinsic doesn't yet support specified leaf", ); diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs index 869670c8c..a799dca93 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs @@ -139,6 +139,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>( .sess .warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic)); crate::trap::trap_unimplemented(fx, intrinsic); + return; } } diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs index b2a83e1d4..2e4ca594f 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs @@ -44,7 +44,7 @@ fn report_atomic_type_validation_error<'tcx>( ), ); // Prevent verifier error - crate::trap::trap_unreachable(fx, "compilation should not have succeeded"); + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); } pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> { @@ -53,7 +53,7 @@ pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx _ => unreachable!(), }; - match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) { + match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) { // Cranelift currently only implements icmp for 128bit vectors. Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty), _ => None, @@ -203,7 +203,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>( sym::transmute => { crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info); } - _ => unimplemented!("unsupported instrinsic {}", intrinsic), + _ => unimplemented!("unsupported intrinsic {}", intrinsic), } return; }; @@ -301,7 +301,44 @@ fn codegen_float_intrinsic_call<'tcx>( _ => unreachable!(), }; - let res = fx.easy_call(name, &args, ty); + let layout = fx.layout_of(ty); + let res = match intrinsic { + sym::fmaf32 | sym::fmaf64 => { + let a = args[0].load_scalar(fx); + let b = args[1].load_scalar(fx); + let c = args[2].load_scalar(fx); + CValue::by_val(fx.bcx.ins().fma(a, b, c), layout) + } + sym::copysignf32 | sym::copysignf64 => { + let a = args[0].load_scalar(fx); + let b = args[1].load_scalar(fx); + CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout) + } + sym::fabsf32 + | sym::fabsf64 + | sym::floorf32 + | sym::floorf64 + | sym::ceilf32 + | sym::ceilf64 + | sym::truncf32 + | sym::truncf64 => { + let a = args[0].load_scalar(fx); + + let val = match intrinsic { + sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a), + sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a), + sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a), + sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a), + _ => unreachable!(), + }; + + CValue::by_val(val, layout) + } + // These intrinsics aren't supported natively by Cranelift. + // Lower them to a libcall. + _ => fx.easy_call(name, &args, ty), + }; + ret.write_cvalue(fx, res); true @@ -320,9 +357,6 @@ fn codegen_regular_intrinsic_call<'tcx>( let usize_layout = fx.layout_of(fx.tcx.types.usize); match intrinsic { - sym::assume => { - intrinsic_args!(fx, args => (_a); intrinsic); - } sym::likely | sym::unlikely => { intrinsic_args!(fx, args => (a); intrinsic); @@ -540,6 +574,13 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, CValue::by_val(res, base.layout())); } + sym::ptr_mask => { + intrinsic_args!(fx, args => (ptr, mask); intrinsic); + let ptr = ptr.load_scalar(fx); + let mask = mask.load_scalar(fx); + fx.bcx.ins().band(ptr, mask); + } + sym::transmute => { intrinsic_args!(fx, args => (from); intrinsic); @@ -775,20 +816,13 @@ fn codegen_regular_intrinsic_call<'tcx>( ret.write_cvalue(fx, val); } - sym::ptr_guaranteed_eq => { + sym::ptr_guaranteed_cmp => { intrinsic_args!(fx, args => (a, b); intrinsic); let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b); ret.write_cvalue(fx, val); } - sym::ptr_guaranteed_ne => { - intrinsic_args!(fx, args => (a, b); intrinsic); - - let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b); - ret.write_cvalue(fx, val); - } - sym::caller_location => { intrinsic_args!(fx, args => (); intrinsic); @@ -818,8 +852,6 @@ fn codegen_regular_intrinsic_call<'tcx>( if fx.tcx.is_compiler_builtins(LOCAL_CRATE) { // special case for compiler-builtins to avoid having to patch it crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported"); - let ret_block = fx.get_block(destination.unwrap()); - fx.bcx.ins().jump(ret_block, &[]); return; } else { fx.tcx @@ -851,8 +883,6 @@ fn codegen_regular_intrinsic_call<'tcx>( if fx.tcx.is_compiler_builtins(LOCAL_CRATE) { // special case for compiler-builtins to avoid having to patch it crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported"); - let ret_block = fx.get_block(destination.unwrap()); - fx.bcx.ins().jump(ret_block, &[]); return; } else { fx.tcx diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs index 30e3d1125..1f358b1bb 100644 --- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs +++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs @@ -14,7 +14,7 @@ fn report_simd_type_validation_error( ) { fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty)); // Prevent verifier error - crate::trap::trap_unreachable(fx, "compilation should not have succeeded"); + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); } pub(super) fn codegen_simd_intrinsic_call<'tcx>( @@ -157,7 +157,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( ), ); // Prevent verifier error - crate::trap::trap_unreachable(fx, "compilation should not have succeeded"); + fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); return; } } @@ -186,7 +186,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let size = Size::from_bytes( 4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */ ); - alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap() + alloc + .inner() + .get_bytes_strip_provenance(fx, alloc_range(offset, size)) + .unwrap() } _ => unreachable!("{:?}", idx_const), }; @@ -274,12 +277,17 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( idx_const } else { fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant"); - let res = crate::trap::trap_unimplemented_ret_value( + let trap_block = fx.bcx.create_block(); + let dummy_block = fx.bcx.create_block(); + let true_ = fx.bcx.ins().iconst(types::I8, 1); + fx.bcx.ins().brnz(true_, trap_block, &[]); + fx.bcx.ins().jump(dummy_block, &[]); + fx.bcx.switch_to_block(trap_block); + crate::trap::trap_unimplemented( fx, - ret.layout(), "Index argument for `simd_extract` is not a constant", ); - ret.write_cvalue(fx, res); + fx.bcx.switch_to_block(dummy_block); return; }; @@ -392,21 +400,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>( let layout = a.layout(); let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx); + let res_lane_layout = fx.layout_of(lane_ty); for lane in 0..lane_count { - let a_lane = a.value_lane(fx, lane); - let b_lane = b.value_lane(fx, lane); - let c_lane = c.value_lane(fx, lane); + let a_lane = a.value_lane(fx, lane).load_scalar(fx); + let b_lane = b.value_lane(fx, lane).load_scalar(fx); + let c_lane = c.value_lane(fx, lane).load_scalar(fx); - let res_lane = match lane_ty.kind() { - ty::Float(FloatTy::F32) => { - fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty) - } - ty::Float(FloatTy::F64) => { - fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty) - } - _ => unreachable!(), - }; + let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane); + let res_lane = CValue::by_val(res_lane, res_lane_layout); ret.place_lane(fx, lane).write_cvalue(fx, res_lane); } diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs index bb0793b1d..913414e76 100644 --- a/compiler/rustc_codegen_cranelift/src/lib.rs +++ b/compiler/rustc_codegen_cranelift/src/lib.rs @@ -4,6 +4,7 @@ #![warn(unused_lifetimes)] #![warn(unreachable_pub)] +extern crate jobserver; #[macro_use] extern crate rustc_middle; extern crate rustc_ast; @@ -25,10 +26,12 @@ extern crate rustc_target; extern crate rustc_driver; use std::any::Any; -use std::cell::Cell; +use std::cell::{Cell, RefCell}; +use std::sync::Arc; use rustc_codegen_ssa::traits::CodegenBackend; use rustc_codegen_ssa::CodegenResults; +use rustc_data_structures::profiling::SelfProfilerRef; use rustc_errors::ErrorGuaranteed; use rustc_metadata::EncodedMetadata; use rustc_middle::dep_graph::{WorkProduct, WorkProductId}; @@ -51,11 +54,13 @@ mod cast; mod codegen_i128; mod common; mod compiler_builtins; +mod concurrency_limiter; mod config; mod constant; mod debuginfo; mod discriminant; mod driver; +mod global_asm; mod inline_asm; mod intrinsics; mod linkage; @@ -119,19 +124,20 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> { /// The codegen context holds any information shared between the codegen of individual functions /// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module). -struct CodegenCx<'tcx> { - tcx: TyCtxt<'tcx>, +struct CodegenCx { + profiler: SelfProfilerRef, + output_filenames: Arc<OutputFilenames>, + should_write_ir: bool, global_asm: String, inline_asm_index: Cell<usize>, - cached_context: Context, - debug_context: Option<DebugContext<'tcx>>, + debug_context: Option<DebugContext>, unwind_context: UnwindContext, cgu_name: Symbol, } -impl<'tcx> CodegenCx<'tcx> { +impl CodegenCx { fn new( - tcx: TyCtxt<'tcx>, + tcx: TyCtxt<'_>, backend_config: BackendConfig, isa: &dyn TargetIsa, debug_info: bool, @@ -147,10 +153,11 @@ impl<'tcx> CodegenCx<'tcx> { None }; CodegenCx { - tcx, + profiler: tcx.prof.clone(), + output_filenames: tcx.output_filenames(()).clone(), + should_write_ir: crate::pretty_clif::should_write_ir(tcx), global_asm: String::new(), inline_asm_index: Cell::new(0), - cached_context: Context::new(), debug_context, unwind_context, cgu_name, @@ -159,7 +166,7 @@ impl<'tcx> CodegenCx<'tcx> { } pub struct CraneliftCodegenBackend { - pub config: Option<BackendConfig>, + pub config: RefCell<Option<BackendConfig>>, } impl CodegenBackend for CraneliftCodegenBackend { @@ -169,6 +176,13 @@ impl CodegenBackend for CraneliftCodegenBackend { Lto::No | Lto::ThinLocal => {} Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."), } + + let mut config = self.config.borrow_mut(); + if config.is_none() { + let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args) + .unwrap_or_else(|err| sess.fatal(&err)); + *config = Some(new_config); + } } fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec<rustc_span::Symbol> { @@ -186,15 +200,7 @@ impl CodegenBackend for CraneliftCodegenBackend { need_metadata_module: bool, ) -> Box<dyn Any> { tcx.sess.abort_if_errors(); - let config = if let Some(config) = self.config.clone() { - config - } else { - if !tcx.sess.unstable_options() && !tcx.sess.opts.cg.llvm_args.is_empty() { - tcx.sess.fatal("`-Z unstable-options` must be passed to allow configuring cg_clif"); - } - BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args) - .unwrap_or_else(|err| tcx.sess.fatal(&err)) - }; + let config = self.config.borrow().clone().unwrap(); match config.codegen_mode { CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module), CodegenMode::Jit | CodegenMode::JitLazy => { @@ -210,12 +216,13 @@ impl CodegenBackend for CraneliftCodegenBackend { fn join_codegen( &self, ongoing_codegen: Box<dyn Any>, - _sess: &Session, + sess: &Session, _outputs: &OutputFilenames, ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> { - Ok(*ongoing_codegen - .downcast::<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)>() - .unwrap()) + Ok(ongoing_codegen + .downcast::<driver::aot::OngoingCodegen>() + .unwrap() + .join(sess, self.config.borrow().as_ref().unwrap())) } fn link( @@ -312,5 +319,5 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar /// This is the entrypoint for a hot plugged rustc_codegen_cranelift #[no_mangle] pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> { - Box::new(CraneliftCodegenBackend { config: None }) + Box::new(CraneliftCodegenBackend { config: RefCell::new(None) }) } diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs index c67b6e98b..3c024a84d 100644 --- a/compiler/rustc_codegen_cranelift/src/main_shim.rs +++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs @@ -1,7 +1,7 @@ use rustc_hir::LangItem; use rustc_middle::ty::subst::GenericArg; use rustc_middle::ty::AssocKind; -use rustc_session::config::EntryFnType; +use rustc_session::config::{sigpipe, EntryFnType}; use rustc_span::symbol::Ident; use crate::prelude::*; @@ -15,12 +15,12 @@ pub(crate) fn maybe_create_entry_wrapper( is_jit: bool, is_primary_cgu: bool, ) { - let (main_def_id, is_main_fn) = match tcx.entry_fn(()) { + let (main_def_id, (is_main_fn, sigpipe)) = match tcx.entry_fn(()) { Some((def_id, entry_ty)) => ( def_id, match entry_ty { - EntryFnType::Main => true, - EntryFnType::Start => false, + EntryFnType::Main { sigpipe } => (true, sigpipe), + EntryFnType::Start => (false, sigpipe::DEFAULT), }, ), None => return, @@ -35,7 +35,7 @@ pub(crate) fn maybe_create_entry_wrapper( return; } - create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn); + create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn, sigpipe); fn create_entry_fn( tcx: TyCtxt<'_>, @@ -44,6 +44,7 @@ pub(crate) fn maybe_create_entry_wrapper( rust_main_def_id: DefId, ignore_lang_start_wrapper: bool, is_main_fn: bool, + sigpipe: u8, ) { let main_ret_ty = tcx.fn_sig(rust_main_def_id).output(); // Given that `main()` has no arguments, @@ -83,6 +84,7 @@ pub(crate) fn maybe_create_entry_wrapper( bcx.switch_to_block(block); let arg_argc = bcx.append_block_param(block, m.target_config().pointer_type()); let arg_argv = bcx.append_block_param(block, m.target_config().pointer_type()); + let arg_sigpipe = bcx.ins().iconst(types::I8, sigpipe as i64); let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func); @@ -143,7 +145,8 @@ pub(crate) fn maybe_create_entry_wrapper( let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref); let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func); - let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]); + let call_inst = + bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv, arg_sigpipe]); bcx.inst_results(call_inst)[0] } else { // using user-defined start fn diff --git a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs index d1f89adb3..0df7e8229 100644 --- a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs +++ b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs @@ -1,20 +1,3 @@ //! Various optimizations specific to cg_clif -use cranelift_codegen::isa::TargetIsa; - -use crate::prelude::*; - pub(crate) mod peephole; - -pub(crate) fn optimize_function<'tcx>( - tcx: TyCtxt<'tcx>, - isa: &dyn TargetIsa, - instance: Instance<'tcx>, - ctx: &mut Context, - clif_comments: &mut crate::pretty_clif::CommentWriter, -) { - // FIXME classify optimizations over opt levels once we have more - - crate::pretty_clif::write_clif_file(tcx, "preopt", isa, instance, &ctx.func, &*clif_comments); - crate::base::verify_func(tcx, &*clif_comments, &ctx.func); -} diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs index 1d1ec2168..a7af16268 100644 --- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs +++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs @@ -62,7 +62,7 @@ use cranelift_codegen::{ }; use rustc_middle::ty::layout::FnAbiOf; -use rustc_session::config::OutputType; +use rustc_session::config::{OutputFilenames, OutputType}; use crate::prelude::*; @@ -205,15 +205,11 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool { } pub(crate) fn write_ir_file( - tcx: TyCtxt<'_>, - name: impl FnOnce() -> String, + output_filenames: &OutputFilenames, + name: &str, write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>, ) { - if !should_write_ir(tcx) { - return; - } - - let clif_output_dir = tcx.output_filenames(()).with_extension("clif"); + let clif_output_dir = output_filenames.with_extension("clif"); match std::fs::create_dir(&clif_output_dir) { Ok(()) => {} @@ -221,44 +217,43 @@ pub(crate) fn write_ir_file( res @ Err(_) => res.unwrap(), } - let clif_file_name = clif_output_dir.join(name()); + let clif_file_name = clif_output_dir.join(name); let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file)); if let Err(err) = res { - tcx.sess.warn(&format!("error writing ir file: {}", err)); + // Using early_warn as no Session is available here + rustc_session::early_warn( + rustc_session::config::ErrorOutputType::default(), + &format!("error writing ir file: {}", err), + ); } } -pub(crate) fn write_clif_file<'tcx>( - tcx: TyCtxt<'tcx>, +pub(crate) fn write_clif_file( + output_filenames: &OutputFilenames, + symbol_name: &str, postfix: &str, isa: &dyn cranelift_codegen::isa::TargetIsa, - instance: Instance<'tcx>, func: &cranelift_codegen::ir::Function, mut clif_comments: &CommentWriter, ) { // FIXME work around filename too long errors - write_ir_file( - tcx, - || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix), - |file| { - let mut clif = String::new(); - cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func) - .unwrap(); + write_ir_file(output_filenames, &format!("{}.{}.clif", symbol_name, postfix), |file| { + let mut clif = String::new(); + cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func).unwrap(); - for flag in isa.flags().iter() { - writeln!(file, "set {}", flag)?; - } - write!(file, "target {}", isa.triple().architecture.to_string())?; - for isa_flag in isa.isa_flags().iter() { - write!(file, " {}", isa_flag)?; - } - writeln!(file, "\n")?; - writeln!(file)?; - file.write_all(clif.as_bytes())?; - Ok(()) - }, - ); + for flag in isa.flags().iter() { + writeln!(file, "set {}", flag)?; + } + write!(file, "target {}", isa.triple().architecture.to_string())?; + for isa_flag in isa.isa_flags().iter() { + write!(file, " {}", isa_flag)?; + } + writeln!(file, "\n")?; + writeln!(file)?; + file.write_all(clif.as_bytes())?; + Ok(()) + }); } impl fmt::Debug for FunctionCx<'_, '_, '_> { diff --git a/compiler/rustc_codegen_cranelift/src/toolchain.rs b/compiler/rustc_codegen_cranelift/src/toolchain.rs index f86236ef3..b6b465e1f 100644 --- a/compiler/rustc_codegen_cranelift/src/toolchain.rs +++ b/compiler/rustc_codegen_cranelift/src/toolchain.rs @@ -8,10 +8,8 @@ use rustc_session::Session; /// Tries to infer the path of a binary for the target toolchain from the linker name. pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf { let (mut linker, _linker_flavor) = linker_and_flavor(sess); - let linker_file_name = linker - .file_name() - .and_then(|name| name.to_str()) - .unwrap_or_else(|| sess.fatal("couldn't extract file name from specified linker")); + let linker_file_name = + linker.file_name().unwrap().to_str().expect("linker filename should be valid UTF-8"); if linker_file_name == "ld.lld" { if tool != "ld" { diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs index 923269c4d..82a2ec579 100644 --- a/compiler/rustc_codegen_cranelift/src/trap.rs +++ b/compiler/rustc_codegen_cranelift/src/trap.rs @@ -25,33 +25,10 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) { fx.bcx.ins().call(puts, &[msg_ptr]); } -/// Use this for example when a function call should never return. This will fill the current block, -/// so you can **not** add instructions to it afterwards. -/// -/// Trap code: user65535 -pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) { - codegen_print(fx, msg.as_ref()); - fx.bcx.ins().trap(TrapCode::UnreachableCodeReached); -} /// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen. -/// Unlike `trap_unreachable` this will not fill the current block, so you **must** add instructions -/// to it afterwards. /// /// Trap code: user65535 pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) { codegen_print(fx, msg.as_ref()); - let true_ = fx.bcx.ins().iconst(types::I32, 1); - fx.bcx.ins().trapnz(true_, TrapCode::User(!0)); -} - -/// Like `trap_unimplemented` but returns a fake value of the specified type. -/// -/// Trap code: user65535 -pub(crate) fn trap_unimplemented_ret_value<'tcx>( - fx: &mut FunctionCx<'_, '_, 'tcx>, - dest_layout: TyAndLayout<'tcx>, - msg: impl AsRef<str>, -) -> CValue<'tcx> { - trap_unimplemented(fx, msg); - CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout) + fx.bcx.ins().trap(TrapCode::User(!0)); } diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs index 052ca0a08..dd9d891dd 100644 --- a/compiler/rustc_codegen_cranelift/src/unsize.rs +++ b/compiler/rustc_codegen_cranelift/src/unsize.rs @@ -29,6 +29,7 @@ pub(crate) fn unsized_info<'tcx>( let old_info = old_info.expect("unsized_info: missing old info for trait upcasting coercion"); if data_a.principal_def_id() == data_b.principal_def_id() { + // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables. return old_info; } diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs index 45ae2bd8f..cfaadca94 100644 --- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs +++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs @@ -122,7 +122,7 @@ impl<'tcx> CValue<'tcx> { let clif_ty = match layout.abi { Abi::Scalar(scalar) => scalar_to_clif_type(fx.tcx, scalar), Abi::Vector { element, count } => scalar_to_clif_type(fx.tcx, element) - .by(u16::try_from(count).unwrap()) + .by(u32::try_from(count).unwrap()) .unwrap(), _ => unreachable!("{:?}", layout.ty), }; @@ -330,7 +330,7 @@ impl<'tcx> CPlace<'tcx> { .fatal(&format!("values of type {} are too big to store on the stack", layout.ty)); } - let stack_slot = fx.bcx.create_stack_slot(StackSlotData { + let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to // specify stack slot alignment. @@ -472,7 +472,7 @@ impl<'tcx> CPlace<'tcx> { } _ if src_ty.is_vector() || dst_ty.is_vector() => { // FIXME do something more efficient for transmutes between vectors and integers. - let stack_slot = fx.bcx.create_stack_slot(StackSlotData { + let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData { kind: StackSlotKind::ExplicitSlot, // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to // specify stack slot alignment. @@ -519,7 +519,7 @@ impl<'tcx> CPlace<'tcx> { if let ty::Array(element, len) = dst_layout.ty.kind() { // Can only happen for vector types let len = - u16::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap(); + u32::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap(); let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap(); let data = match from.0 { @@ -614,7 +614,7 @@ impl<'tcx> CPlace<'tcx> { dst_align, src_align, true, - MemFlags::trusted(), + flags, ); } CValueInner::ByRef(_, Some(_)) => todo!(), @@ -815,7 +815,8 @@ pub(crate) fn assert_assignable<'tcx>( ); // fn(&T) -> for<'l> fn(&'l T) is allowed } - (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => { + (&ty::Dynamic(from_traits, _, _from_kind), &ty::Dynamic(to_traits, _, _to_kind)) => { + // FIXME(dyn-star): Do the right thing with DynKinds for (from, to) in from_traits.iter().zip(to_traits) { let from = fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from); |