summaryrefslogtreecommitdiffstats
path: root/compiler/rustc_codegen_cranelift/src
diff options
context:
space:
mode:
Diffstat (limited to 'compiler/rustc_codegen_cranelift/src')
-rw-r--r--compiler/rustc_codegen_cranelift/src/abi/comments.rs2
-rw-r--r--compiler/rustc_codegen_cranelift/src/abi/mod.rs2
-rw-r--r--compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs29
-rw-r--r--compiler/rustc_codegen_cranelift/src/abi/returning.rs10
-rw-r--r--compiler/rustc_codegen_cranelift/src/analyze.rs2
-rw-r--r--compiler/rustc_codegen_cranelift/src/archive.rs1
-rw-r--r--compiler/rustc_codegen_cranelift/src/base.rs281
-rw-r--r--compiler/rustc_codegen_cranelift/src/common.rs46
-rw-r--r--compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs168
-rw-r--r--compiler/rustc_codegen_cranelift/src/constant.rs85
-rw-r--r--compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs2
-rw-r--r--compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs213
-rw-r--r--compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs282
-rw-r--r--compiler/rustc_codegen_cranelift/src/discriminant.rs31
-rw-r--r--compiler/rustc_codegen_cranelift/src/driver/aot.rs560
-rw-r--r--compiler/rustc_codegen_cranelift/src/driver/jit.rs49
-rw-r--r--compiler/rustc_codegen_cranelift/src/global_asm.rs114
-rw-r--r--compiler/rustc_codegen_cranelift/src/inline_asm.rs176
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs2
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs1
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs68
-rw-r--r--compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs38
-rw-r--r--compiler/rustc_codegen_cranelift/src/lib.rs55
-rw-r--r--compiler/rustc_codegen_cranelift/src/main_shim.rs15
-rw-r--r--compiler/rustc_codegen_cranelift/src/optimize/mod.rs17
-rw-r--r--compiler/rustc_codegen_cranelift/src/pretty_clif.rs61
-rw-r--r--compiler/rustc_codegen_cranelift/src/toolchain.rs6
-rw-r--r--compiler/rustc_codegen_cranelift/src/trap.rs25
-rw-r--r--compiler/rustc_codegen_cranelift/src/unsize.rs1
-rw-r--r--compiler/rustc_codegen_cranelift/src/value_and_place.rs13
30 files changed, 1323 insertions, 1032 deletions
diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
index 37d2679c1..7f4619b5c 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
@@ -24,7 +24,7 @@ pub(super) fn add_arg_comment<'tcx>(
local: Option<mir::Local>,
local_field: Option<usize>,
params: &[Value],
- arg_abi_mode: PassMode,
+ arg_abi_mode: &PassMode,
arg_layout: TyAndLayout<'tcx>,
) {
if !fx.clif_comments.enabled() {
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index 815450f68..0497c2570 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -342,7 +342,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
let ret_place = codegen_place(fx, destination);
- // Handle special calls like instrinsics and empty drop glue.
+ // Handle special calls like intrinsics and empty drop glue.
let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
let instance = ty::Instance::resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs)
.unwrap()
diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
index 6c10baa53..96e25d3a8 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
@@ -23,7 +23,7 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam {
(RegKind::Integer, 9..=16) => types::I128,
(RegKind::Float, 4) => types::F32,
(RegKind::Float, 8) => types::F64,
- (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(),
+ (RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(),
_ => unreachable!("{:?}", reg),
};
AbiParam::new(clif_ty)
@@ -38,7 +38,7 @@ fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -
param
}
-fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> {
+fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 {
(0, 0)
} else {
@@ -100,7 +100,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
}
_ => unreachable!("{:?}", self.layout.abi),
},
- PassMode::Cast(cast) => cast_target_to_abi_params(cast),
+ PassMode::Cast(ref cast, pad_i32) => {
+ assert!(!pad_i32, "padding support not yet implemented");
+ cast_target_to_abi_params(cast)
+ }
PassMode::Indirect { attrs, extra_attrs: None, on_stack } => {
if on_stack {
// Abi requires aligning struct size to pointer size
@@ -145,7 +148,9 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
}
_ => unreachable!("{:?}", self.layout.abi),
},
- PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()),
+ PassMode::Cast(ref cast, _) => {
+ (None, cast_target_to_abi_params(cast).into_iter().collect())
+ }
PassMode::Indirect { attrs: _, extra_attrs: None, on_stack } => {
assert!(!on_stack);
(Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![])
@@ -160,7 +165,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
pub(super) fn to_casted_value<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
arg: CValue<'tcx>,
- cast: CastTarget,
+ cast: &CastTarget,
) -> SmallVec<[Value; 2]> {
let (ptr, meta) = arg.force_stack(fx);
assert!(meta.is_none());
@@ -179,12 +184,12 @@ pub(super) fn from_casted_value<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
block_params: &[Value],
layout: TyAndLayout<'tcx>,
- cast: CastTarget,
+ cast: &CastTarget,
) -> CValue<'tcx> {
let abi_params = cast_target_to_abi_params(cast);
let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
let layout_size = u32::try_from(layout.size.bytes()).unwrap();
- let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
+ let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
@@ -193,7 +198,7 @@ pub(super) fn from_casted_value<'tcx>(
// larger alignment than the integer.
size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16,
});
- let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
+ let ptr = Pointer::stack_slot(stack_slot);
let mut offset = 0;
let mut block_params_iter = block_params.iter().copied();
for param in abi_params {
@@ -224,7 +229,7 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
let (a, b) = arg.load_scalar_pair(fx);
smallvec![a, b]
}
- PassMode::Cast(cast) => to_casted_value(fx, arg, cast),
+ PassMode::Cast(ref cast, _) => to_casted_value(fx, arg, cast),
PassMode::Indirect { .. } => {
if is_owned {
match arg.force_stack(fx) {
@@ -268,7 +273,7 @@ pub(super) fn cvalue_for_param<'tcx>(
local,
local_field,
&block_params,
- arg_abi.mode,
+ &arg_abi.mode,
arg_abi.layout,
);
@@ -282,7 +287,9 @@ pub(super) fn cvalue_for_param<'tcx>(
assert_eq!(block_params.len(), 2, "{:?}", block_params);
Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout))
}
- PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)),
+ PassMode::Cast(ref cast, _) => {
+ Some(from_casted_value(fx, &block_params, arg_abi.layout, cast))
+ }
PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
assert_eq!(block_params.len(), 1, "{:?}", block_params);
Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout))
diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
index ff3bb2dfd..aaa141876 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
@@ -13,7 +13,7 @@ pub(super) fn codegen_return_param<'tcx>(
block_params_iter: &mut impl Iterator<Item = Value>,
) -> CPlace<'tcx> {
let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode {
- PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => {
+ PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => {
let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa;
(
super::make_local_place(
@@ -44,7 +44,7 @@ pub(super) fn codegen_return_param<'tcx>(
Some(RETURN_PLACE),
None,
&ret_param,
- fx.fn_abi.as_ref().unwrap().ret.mode,
+ &fx.fn_abi.as_ref().unwrap().ret.mode,
fx.fn_abi.as_ref().unwrap().ret.layout,
);
@@ -75,7 +75,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
unreachable!("unsized return value")
}
- PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => (None, None),
+ PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => (None, None),
};
let call_inst = f(fx, return_ptr);
@@ -92,7 +92,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
ret_place
.write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout));
}
- PassMode::Cast(cast) => {
+ PassMode::Cast(ref cast, _) => {
let results =
fx.bcx.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>();
let result =
@@ -131,7 +131,7 @@ pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) {
let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx);
fx.bcx.ins().return_(&[ret_val_a, ret_val_b]);
}
- PassMode::Cast(cast) => {
+ PassMode::Cast(ref cast, _) => {
let place = fx.get_local_place(RETURN_PLACE);
let ret_val = place.to_cvalue(fx);
let ret_vals = super::pass_mode::to_casted_value(fx, ret_val, cast);
diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs
index 35b89358b..0cbb9f3ec 100644
--- a/compiler/rustc_codegen_cranelift/src/analyze.rs
+++ b/compiler/rustc_codegen_cranelift/src/analyze.rs
@@ -26,7 +26,7 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> {
})
.collect::<IndexVec<Local, SsaKind>>();
- for bb in fx.mir.basic_blocks().iter() {
+ for bb in fx.mir.basic_blocks.iter() {
for stmt in bb.statements.iter() {
match &stmt.kind {
Assign(place_and_rval) => match &place_and_rval.1 {
diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs
index b4c790961..fb5313f17 100644
--- a/compiler/rustc_codegen_cranelift/src/archive.rs
+++ b/compiler/rustc_codegen_cranelift/src/archive.rs
@@ -38,6 +38,7 @@ impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
_lib_name: &str,
_dll_imports: &[rustc_session::cstore::DllImport],
_tmpdir: &Path,
+ _is_direct_dependency: bool,
) -> PathBuf {
bug!("creating dll imports is not supported");
}
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 122e103ff..399474d79 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -6,21 +6,43 @@ use rustc_middle::ty::adjustment::PointerCast;
use rustc_middle::ty::layout::FnAbiOf;
use rustc_middle::ty::print::with_no_trimmed_paths;
-use indexmap::IndexSet;
-
use crate::constant::ConstantCx;
+use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;
use crate::pretty_clif::CommentWriter;
-pub(crate) fn codegen_fn<'tcx>(
- cx: &mut crate::CodegenCx<'tcx>,
+pub(crate) struct CodegenedFunction {
+ symbol_name: String,
+ func_id: FuncId,
+ func: Function,
+ clif_comments: CommentWriter,
+ func_debug_cx: Option<FunctionDebugContext>,
+}
+
+#[cfg_attr(not(feature = "jit"), allow(dead_code))]
+pub(crate) fn codegen_and_compile_fn<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ cx: &mut crate::CodegenCx,
+ cached_context: &mut Context,
module: &mut dyn Module,
instance: Instance<'tcx>,
) {
- let tcx = cx.tcx;
-
let _inst_guard =
crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
+
+ let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
+ let codegened_func = codegen_fn(tcx, cx, cached_func, module, instance);
+
+ compile_fn(cx, cached_context, module, codegened_func);
+}
+
+pub(crate) fn codegen_fn<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ cx: &mut crate::CodegenCx,
+ cached_func: Function,
+ module: &mut dyn Module,
+ instance: Instance<'tcx>,
+) -> CodegenedFunction {
debug_assert!(!instance.substs.needs_infer());
let mir = tcx.instance_mir(instance.def);
@@ -34,15 +56,14 @@ pub(crate) fn codegen_fn<'tcx>(
});
// Declare function
- let symbol_name = tcx.symbol_name(instance);
+ let symbol_name = tcx.symbol_name(instance).name.to_string();
let sig = get_function_sig(tcx, module.isa().triple(), instance);
- let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap();
-
- cx.cached_context.clear();
+ let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap();
// Make the FunctionBuilder
let mut func_ctx = FunctionBuilderContext::new();
- let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new());
+ let mut func = cached_func;
+ func.clear();
func.name = ExternalName::user(0, func_id.as_u32());
func.signature = sig;
func.collect_debug_info();
@@ -52,13 +73,19 @@ pub(crate) fn codegen_fn<'tcx>(
// Predefine blocks
let start_block = bcx.create_block();
let block_map: IndexVec<BasicBlock, Block> =
- (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
+ (0..mir.basic_blocks.len()).map(|_| bcx.create_block()).collect();
// Make FunctionCx
let target_config = module.target_config();
let pointer_type = target_config.pointer_type();
let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
+ let func_debug_cx = if let Some(debug_context) = &mut cx.debug_context {
+ Some(debug_context.define_function(tcx, &symbol_name, mir.span))
+ } else {
+ None
+ };
+
let mut fx = FunctionCx {
cx,
module,
@@ -66,6 +93,7 @@ pub(crate) fn codegen_fn<'tcx>(
target_config,
pointer_type,
constants_cx: ConstantCx::new(),
+ func_debug_cx,
instance,
symbol_name,
@@ -78,81 +106,48 @@ pub(crate) fn codegen_fn<'tcx>(
caller_location: None, // set by `codegen_fn_prelude`
clif_comments,
- source_info_set: indexmap::IndexSet::new(),
+ last_source_file: None,
next_ssa_var: 0,
};
- let arg_uninhabited = fx
- .mir
- .args_iter()
- .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
-
- if !crate::constant::check_constants(&mut fx) {
- fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
- fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
- crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted");
- } else if arg_uninhabited {
- fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
- fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
- fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
- } else {
- tcx.sess.time("codegen clif ir", || {
- tcx.sess
- .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block));
- codegen_fn_content(&mut fx);
- });
- }
+ tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block));
// Recover all necessary data from fx, before accessing func will prevent future access to it.
- let instance = fx.instance;
+ let symbol_name = fx.symbol_name;
let clif_comments = fx.clif_comments;
- let source_info_set = fx.source_info_set;
- let local_map = fx.local_map;
+ let func_debug_cx = fx.func_debug_cx;
fx.constants_cx.finalize(fx.tcx, &mut *fx.module);
- crate::pretty_clif::write_clif_file(
- tcx,
- "unopt",
- module.isa(),
- instance,
- &func,
- &clif_comments,
- );
+ if cx.should_write_ir {
+ crate::pretty_clif::write_clif_file(
+ tcx.output_filenames(()),
+ &symbol_name,
+ "unopt",
+ module.isa(),
+ &func,
+ &clif_comments,
+ );
+ }
// Verify function
verify_func(tcx, &clif_comments, &func);
- compile_fn(
- cx,
- module,
- instance,
- symbol_name.name,
- func_id,
- func,
- clif_comments,
- source_info_set,
- local_map,
- );
+ CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }
}
-fn compile_fn<'tcx>(
- cx: &mut crate::CodegenCx<'tcx>,
+pub(crate) fn compile_fn(
+ cx: &mut crate::CodegenCx,
+ cached_context: &mut Context,
module: &mut dyn Module,
- instance: Instance<'tcx>,
- symbol_name: &str,
- func_id: FuncId,
- func: Function,
- mut clif_comments: CommentWriter,
- source_info_set: IndexSet<SourceInfo>,
- local_map: IndexVec<mir::Local, CPlace<'tcx>>,
+ codegened_func: CodegenedFunction,
) {
- let tcx = cx.tcx;
+ let clif_comments = codegened_func.clif_comments;
// Store function in context
- let context = &mut cx.cached_context;
+ let context = cached_context;
context.clear();
- context.func = func;
+ context.func = codegened_func.func;
// If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
// instruction, which doesn't have an encoding.
@@ -164,17 +159,6 @@ fn compile_fn<'tcx>(
// invalidate it when it would change.
context.domtree.clear();
- // Perform rust specific optimizations
- tcx.sess.time("optimize clif ir", || {
- crate::optimize::optimize_function(
- tcx,
- module.isa(),
- instance,
- context,
- &mut clif_comments,
- );
- });
-
#[cfg(any())] // This is never true
let _clif_guard = {
use std::fmt::Write;
@@ -203,46 +187,44 @@ fn compile_fn<'tcx>(
};
// Define function
- tcx.sess.time("define function", || {
- context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
- module.define_function(func_id, context).unwrap();
+ cx.profiler.verbose_generic_activity("define function").run(|| {
+ context.want_disasm = cx.should_write_ir;
+ module.define_function(codegened_func.func_id, context).unwrap();
});
- // Write optimized function to file for debugging
- crate::pretty_clif::write_clif_file(
- tcx,
- "opt",
- module.isa(),
- instance,
- &context.func,
- &clif_comments,
- );
+ if cx.should_write_ir {
+ // Write optimized function to file for debugging
+ crate::pretty_clif::write_clif_file(
+ &cx.output_filenames,
+ &codegened_func.symbol_name,
+ "opt",
+ module.isa(),
+ &context.func,
+ &clif_comments,
+ );
- if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm {
- crate::pretty_clif::write_ir_file(
- tcx,
- || format!("{}.vcode", tcx.symbol_name(instance).name),
- |file| file.write_all(disasm.as_bytes()),
- )
+ if let Some(disasm) = &context.compiled_code().unwrap().disasm {
+ crate::pretty_clif::write_ir_file(
+ &cx.output_filenames,
+ &format!("{}.vcode", codegened_func.symbol_name),
+ |file| file.write_all(disasm.as_bytes()),
+ )
+ }
}
// Define debuginfo for function
let isa = module.isa();
let debug_context = &mut cx.debug_context;
let unwind_context = &mut cx.unwind_context;
- tcx.sess.time("generate debug info", || {
+ cx.profiler.verbose_generic_activity("generate debug info").run(|| {
if let Some(debug_context) = debug_context {
- debug_context.define_function(
- instance,
- func_id,
- symbol_name,
- isa,
+ codegened_func.func_debug_cx.unwrap().finalize(
+ debug_context,
+ codegened_func.func_id,
context,
- &source_info_set,
- local_map,
);
}
- unwind_context.add_function(func_id, &context, isa);
+ unwind_context.add_function(codegened_func.func_id, &context, isa);
});
}
@@ -268,8 +250,28 @@ pub(crate) fn verify_func(
});
}
-fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
- for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() {
+fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
+ if !crate::constant::check_constants(fx) {
+ fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+ fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
+ // compilation should have been aborted
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ return;
+ }
+
+ let arg_uninhabited = fx
+ .mir
+ .args_iter()
+ .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
+ if arg_uninhabited {
+ fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+ fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ return;
+ }
+ fx.tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(fx, start_block));
+
+ for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() {
let block = fx.get_block(bb);
fx.bcx.switch_to_block(block);
@@ -457,17 +459,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
template,
operands,
*options,
+ *destination,
);
-
- match *destination {
- Some(destination) => {
- let destination_block = fx.get_block(destination);
- fx.bcx.ins().jump(destination_block, &[]);
- }
- None => {
- fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
- }
- }
}
TerminatorKind::Resume | TerminatorKind::Abort => {
// FIXME implement unwinding
@@ -708,12 +701,14 @@ fn codegen_stmt<'tcx>(
let operand = codegen_operand(fx, operand);
operand.unsize_value(fx, lval);
}
+ Rvalue::Cast(CastKind::DynStar, _, _) => {
+ // FIXME(dyn-star)
+ unimplemented!()
+ }
Rvalue::Discriminant(place) => {
let place = codegen_place(fx, place);
let value = place.to_cvalue(fx);
- let discr =
- crate::discriminant::codegen_get_discriminant(fx, value, dest_layout);
- lval.write_cvalue(fx, discr);
+ crate::discriminant::codegen_get_discriminant(fx, lval, value, dest_layout);
}
Rvalue::Repeat(ref operand, times) => {
let operand = codegen_operand(fx, operand);
@@ -803,20 +798,31 @@ fn codegen_stmt<'tcx>(
| StatementKind::AscribeUserType(..) => {}
StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
- StatementKind::CopyNonOverlapping(inner) => {
- let dst = codegen_operand(fx, &inner.dst);
- let pointee = dst
- .layout()
- .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
- .expect("Expected pointer");
- let dst = dst.load_scalar(fx);
- let src = codegen_operand(fx, &inner.src).load_scalar(fx);
- let count = codegen_operand(fx, &inner.count).load_scalar(fx);
- let elem_size: u64 = pointee.size.bytes();
- let bytes =
- if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
- fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
- }
+ StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic {
+ // We ignore `assume` intrinsics, they are only useful for optimizations
+ NonDivergingIntrinsic::Assume(_) => {}
+ NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
+ src,
+ dst,
+ count,
+ }) => {
+ let dst = codegen_operand(fx, dst);
+ let pointee = dst
+ .layout()
+ .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+ .expect("Expected pointer");
+ let dst = dst.load_scalar(fx);
+ let src = codegen_operand(fx, src).load_scalar(fx);
+ let count = codegen_operand(fx, count).load_scalar(fx);
+ let elem_size: u64 = pointee.size.bytes();
+ let bytes = if elem_size != 1 {
+ fx.bcx.ins().imul_imm(count, elem_size as i64)
+ } else {
+ count
+ };
+ fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
+ }
+ },
}
}
@@ -934,8 +940,11 @@ pub(crate) fn codegen_panic_inner<'tcx>(
args: &[Value],
span: Span,
) {
- let def_id =
- fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s));
+ let def_id = fx
+ .tcx
+ .lang_items()
+ .require(lang_item)
+ .unwrap_or_else(|e| fx.tcx.sess.span_fatal(span, e.to_string()));
let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
let symbol_name = fx.tcx.symbol_name(instance).name;
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index f9dc1b516..589594465 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -1,14 +1,18 @@
use cranelift_codegen::isa::TargetFrontendConfig;
+use gimli::write::FileId;
+
+use rustc_data_structures::sync::Lrc;
use rustc_index::vec::IndexVec;
use rustc_middle::ty::layout::{
FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
};
-use rustc_middle::ty::SymbolName;
+use rustc_span::SourceFile;
use rustc_target::abi::call::FnAbi;
use rustc_target::abi::{Integer, Primitive};
use rustc_target::spec::{HasTargetSpec, Target};
use crate::constant::ConstantCx;
+use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;
pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
@@ -74,7 +78,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
_ => unreachable!(),
};
- match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
+ match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
// Cranelift currently only implements icmp for 128bit vectors.
Some(vector_ty) if vector_ty.bits() == 128 => vector_ty,
_ => return None,
@@ -232,15 +236,16 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
}
pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
- pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>,
+ pub(crate) cx: &'clif mut crate::CodegenCx,
pub(crate) module: &'m mut dyn Module,
pub(crate) tcx: TyCtxt<'tcx>,
pub(crate) target_config: TargetFrontendConfig, // Cached from module
pub(crate) pointer_type: Type, // Cached from module
pub(crate) constants_cx: ConstantCx,
+ pub(crate) func_debug_cx: Option<FunctionDebugContext>,
pub(crate) instance: Instance<'tcx>,
- pub(crate) symbol_name: SymbolName<'tcx>,
+ pub(crate) symbol_name: String,
pub(crate) mir: &'tcx Body<'tcx>,
pub(crate) fn_abi: Option<&'tcx FnAbi<'tcx, Ty<'tcx>>>,
@@ -252,7 +257,11 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
pub(crate) caller_location: Option<CValue<'tcx>>,
pub(crate) clif_comments: crate::pretty_clif::CommentWriter,
- pub(crate) source_info_set: indexmap::IndexSet<SourceInfo>,
+
+ /// Last accessed source file and it's debuginfo file id.
+ ///
+ /// For optimization purposes only
+ pub(crate) last_source_file: Option<(Lrc<SourceFile>, FileId)>,
/// This should only be accessed by `CPlace::new_var`.
pub(crate) next_ssa_var: u32,
@@ -336,8 +345,31 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
}
pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) {
- let (index, _) = self.source_info_set.insert_full(source_info);
- self.bcx.set_srcloc(SourceLoc::new(index as u32));
+ if let Some(debug_context) = &mut self.cx.debug_context {
+ let (file, line, column) =
+ DebugContext::get_span_loc(self.tcx, self.mir.span, source_info.span);
+
+ // add_source_file is very slow.
+ // Optimize for the common case of the current file not being changed.
+ let mut cached_file_id = None;
+ if let Some((ref last_source_file, last_file_id)) = self.last_source_file {
+ // If the allocations are not equal, the files may still be equal, but that
+ // doesn't matter, as this is just an optimization.
+ if rustc_data_structures::sync::Lrc::ptr_eq(last_source_file, &file) {
+ cached_file_id = Some(last_file_id);
+ }
+ }
+
+ let file_id = if let Some(file_id) = cached_file_id {
+ file_id
+ } else {
+ debug_context.add_source_file(&file)
+ };
+
+ let source_loc =
+ self.func_debug_cx.as_mut().unwrap().add_dbg_loc(file_id, line, column);
+ self.bcx.set_srcloc(source_loc);
+ }
}
// Note: must be kept in sync with get_caller_location from cg_ssa
diff --git a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
new file mode 100644
index 000000000..dfde97920
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
@@ -0,0 +1,168 @@
+use std::sync::{Arc, Condvar, Mutex};
+
+use rustc_session::Session;
+
+use jobserver::HelperThread;
+
+// FIXME don't panic when a worker thread panics
+
+pub(super) struct ConcurrencyLimiter {
+ helper_thread: Option<HelperThread>,
+ state: Arc<Mutex<state::ConcurrencyLimiterState>>,
+ available_token_condvar: Arc<Condvar>,
+}
+
+impl ConcurrencyLimiter {
+ pub(super) fn new(sess: &Session, pending_jobs: usize) -> Self {
+ let state = Arc::new(Mutex::new(state::ConcurrencyLimiterState::new(pending_jobs)));
+ let available_token_condvar = Arc::new(Condvar::new());
+
+ let state_helper = state.clone();
+ let available_token_condvar_helper = available_token_condvar.clone();
+ let helper_thread = sess
+ .jobserver
+ .clone()
+ .into_helper_thread(move |token| {
+ let mut state = state_helper.lock().unwrap();
+ state.add_new_token(token.unwrap());
+ available_token_condvar_helper.notify_one();
+ })
+ .unwrap();
+ ConcurrencyLimiter {
+ helper_thread: Some(helper_thread),
+ state,
+ available_token_condvar: Arc::new(Condvar::new()),
+ }
+ }
+
+ pub(super) fn acquire(&mut self) -> ConcurrencyLimiterToken {
+ let mut state = self.state.lock().unwrap();
+ loop {
+ state.assert_invariants();
+
+ if state.try_start_job() {
+ return ConcurrencyLimiterToken {
+ state: self.state.clone(),
+ available_token_condvar: self.available_token_condvar.clone(),
+ };
+ }
+
+ self.helper_thread.as_mut().unwrap().request_token();
+ state = self.available_token_condvar.wait(state).unwrap();
+ }
+ }
+
+ pub(super) fn job_already_done(&mut self) {
+ let mut state = self.state.lock().unwrap();
+ state.job_already_done();
+ }
+}
+
+impl Drop for ConcurrencyLimiter {
+ fn drop(&mut self) {
+ //
+ self.helper_thread.take();
+
+ // Assert that all jobs have finished
+ let state = Mutex::get_mut(Arc::get_mut(&mut self.state).unwrap()).unwrap();
+ state.assert_done();
+ }
+}
+
+#[derive(Debug)]
+pub(super) struct ConcurrencyLimiterToken {
+ state: Arc<Mutex<state::ConcurrencyLimiterState>>,
+ available_token_condvar: Arc<Condvar>,
+}
+
+impl Drop for ConcurrencyLimiterToken {
+ fn drop(&mut self) {
+ let mut state = self.state.lock().unwrap();
+ state.job_finished();
+ self.available_token_condvar.notify_one();
+ }
+}
+
+mod state {
+ use jobserver::Acquired;
+
+ #[derive(Debug)]
+ pub(super) struct ConcurrencyLimiterState {
+ pending_jobs: usize,
+ active_jobs: usize,
+
+ // None is used to represent the implicit token, Some to represent explicit tokens
+ tokens: Vec<Option<Acquired>>,
+ }
+
+ impl ConcurrencyLimiterState {
+ pub(super) fn new(pending_jobs: usize) -> Self {
+ ConcurrencyLimiterState { pending_jobs, active_jobs: 0, tokens: vec![None] }
+ }
+
+ pub(super) fn assert_invariants(&self) {
+ // There must be no excess active jobs
+ assert!(self.active_jobs <= self.pending_jobs);
+
+ // There may not be more active jobs than there are tokens
+ assert!(self.active_jobs <= self.tokens.len());
+ }
+
+ pub(super) fn assert_done(&self) {
+ assert_eq!(self.pending_jobs, 0);
+ assert_eq!(self.active_jobs, 0);
+ }
+
+ pub(super) fn add_new_token(&mut self, token: Acquired) {
+ self.tokens.push(Some(token));
+ self.drop_excess_capacity();
+ }
+
+ pub(super) fn try_start_job(&mut self) -> bool {
+ if self.active_jobs < self.tokens.len() {
+ // Using existing token
+ self.job_started();
+ return true;
+ }
+
+ false
+ }
+
+ pub(super) fn job_started(&mut self) {
+ self.assert_invariants();
+ self.active_jobs += 1;
+ self.drop_excess_capacity();
+ self.assert_invariants();
+ }
+
+ pub(super) fn job_finished(&mut self) {
+ self.assert_invariants();
+ self.pending_jobs -= 1;
+ self.active_jobs -= 1;
+ self.assert_invariants();
+ self.drop_excess_capacity();
+ self.assert_invariants();
+ }
+
+ pub(super) fn job_already_done(&mut self) {
+ self.assert_invariants();
+ self.pending_jobs -= 1;
+ self.assert_invariants();
+ self.drop_excess_capacity();
+ self.assert_invariants();
+ }
+
+ fn drop_excess_capacity(&mut self) {
+ self.assert_invariants();
+
+ // Drop all tokens that can never be used anymore
+ self.tokens.truncate(std::cmp::max(self.pending_jobs, 1));
+
+ // Keep some excess tokens to satisfy requests faster
+ const MAX_EXTRA_CAPACITY: usize = 2;
+ self.tokens.truncate(std::cmp::max(self.active_jobs + MAX_EXTRA_CAPACITY, 1));
+
+ self.assert_invariants();
+ }
+ }
+}
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index 7f7fd0e9c..6b4ed9b9d 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -41,36 +41,30 @@ impl ConstantCx {
pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
let mut all_constants_ok = true;
for constant in &fx.mir.required_consts {
- let const_ = match fx.monomorphize(constant.literal) {
- ConstantKind::Ty(ct) => ct,
+ let unevaluated = match fx.monomorphize(constant.literal) {
+ ConstantKind::Ty(ct) => match ct.kind() {
+ ConstKind::Unevaluated(uv) => uv.expand(),
+ ConstKind::Value(_) => continue,
+ ConstKind::Param(_)
+ | ConstKind::Infer(_)
+ | ConstKind::Bound(_, _)
+ | ConstKind::Placeholder(_)
+ | ConstKind::Error(_) => unreachable!("{:?}", ct),
+ },
+ ConstantKind::Unevaluated(uv, _) => uv,
ConstantKind::Val(..) => continue,
};
- match const_.kind() {
- ConstKind::Value(_) => {}
- ConstKind::Unevaluated(unevaluated) => {
- if let Err(err) =
- fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None)
- {
- all_constants_ok = false;
- match err {
- ErrorHandled::Reported(_) | ErrorHandled::Linted => {
- fx.tcx.sess.span_err(constant.span, "erroneous constant encountered");
- }
- ErrorHandled::TooGeneric => {
- span_bug!(
- constant.span,
- "codgen encountered polymorphic constant: {:?}",
- err
- );
- }
- }
+
+ if let Err(err) = fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {
+ all_constants_ok = false;
+ match err {
+ ErrorHandled::Reported(_) | ErrorHandled::Linted => {
+ fx.tcx.sess.span_err(constant.span, "erroneous constant encountered");
+ }
+ ErrorHandled::TooGeneric => {
+ span_bug!(constant.span, "codegen encountered polymorphic constant: {:?}", err);
}
}
- ConstKind::Param(_)
- | ConstKind::Infer(_)
- | ConstKind::Bound(_, _)
- | ConstKind::Placeholder(_)
- | ConstKind::Error(_) => unreachable!("{:?}", const_),
}
}
all_constants_ok
@@ -122,36 +116,28 @@ pub(crate) fn codegen_constant<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
constant: &Constant<'tcx>,
) -> CValue<'tcx> {
- let const_ = match fx.monomorphize(constant.literal) {
- ConstantKind::Ty(ct) => ct,
- ConstantKind::Val(val, ty) => return codegen_const_value(fx, val, ty),
- };
- let const_val = match const_.kind() {
- ConstKind::Value(valtree) => fx.tcx.valtree_to_const_val((const_.ty(), valtree)),
- ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+ let (const_val, ty) = match fx.monomorphize(constant.literal) {
+ ConstantKind::Ty(const_) => unreachable!("{:?}", const_),
+ ConstantKind::Unevaluated(ty::Unevaluated { def, substs, promoted }, ty)
if fx.tcx.is_static(def.did) =>
{
assert!(substs.is_empty());
assert!(promoted.is_none());
- return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty())).to_cvalue(fx);
+ return codegen_static_ref(fx, def.did, fx.layout_of(ty)).to_cvalue(fx);
}
- ConstKind::Unevaluated(unevaluated) => {
+ ConstantKind::Unevaluated(unevaluated, ty) => {
match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {
- Ok(const_val) => const_val,
+ Ok(const_val) => (const_val, ty),
Err(_) => {
span_bug!(constant.span, "erroneous constant not captured by required_consts");
}
}
}
- ConstKind::Param(_)
- | ConstKind::Infer(_)
- | ConstKind::Bound(_, _)
- | ConstKind::Placeholder(_)
- | ConstKind::Error(_) => unreachable!("{:?}", const_),
+ ConstantKind::Val(val, ty) => (val, ty),
};
- codegen_const_value(fx, const_val, const_.ty())
+ codegen_const_value(fx, const_val, ty)
}
pub(crate) fn codegen_const_value<'tcx>(
@@ -430,7 +416,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();
data_ctx.define(bytes.into_boxed_slice());
- for &(offset, alloc_id) in alloc.relocations().iter() {
+ for &(offset, alloc_id) in alloc.provenance().iter() {
let addend = {
let endianness = tcx.data_layout.endian;
let offset = offset.bytes() as usize;
@@ -496,6 +482,9 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
.eval_for_mir(fx.tcx, ParamEnv::reveal_all())
.try_to_value(fx.tcx),
ConstantKind::Val(val, _) => Some(val),
+ ConstantKind::Unevaluated(uv, _) => {
+ fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), uv, None).ok()
+ }
},
// FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored
// inside a temporary before being passed to the intrinsic requiring the const argument.
@@ -505,7 +494,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
return None;
}
let mut computed_const_val = None;
- for bb_data in fx.mir.basic_blocks() {
+ for bb_data in fx.mir.basic_blocks.iter() {
for stmt in &bb_data.statements {
match &stmt.kind {
StatementKind::Assign(local_and_rvalue) if &local_and_rvalue.0 == place => {
@@ -536,9 +525,11 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
{
return None;
}
- StatementKind::CopyNonOverlapping(_) => {
- return None;
- } // conservative handling
+ StatementKind::Intrinsic(ref intrinsic) => match **intrinsic {
+ NonDivergingIntrinsic::CopyNonOverlapping(..) => return None,
+ NonDivergingIntrinsic::Assume(..) => {}
+ },
+ // conservative handling
StatementKind::Assign(_)
| StatementKind::FakeRead(_)
| StatementKind::SetDiscriminant { .. }
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
index 589910ede..9583cd2ec 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
@@ -9,7 +9,7 @@ use gimli::{RunTimeEndian, SectionId};
use super::object::WriteDebugInfo;
use super::DebugContext;
-impl DebugContext<'_> {
+impl DebugContext {
pub(crate) fn emit(&mut self, product: &mut ObjectProduct) {
let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone());
let root = self.dwarf.unit.root();
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
index bbcb95913..463de6a91 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
@@ -3,8 +3,10 @@
use std::ffi::OsStr;
use std::path::{Component, Path};
+use crate::debuginfo::FunctionDebugContext;
use crate::prelude::*;
+use rustc_data_structures::sync::Lrc;
use rustc_span::{
FileName, Pos, SourceFile, SourceFileAndLine, SourceFileHash, SourceFileHashAlgorithm,
};
@@ -14,7 +16,6 @@ use cranelift_codegen::MachSrcLoc;
use gimli::write::{
Address, AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable,
- UnitEntryId,
};
// OPTIMIZATION: It is cheaper to do this in one pass than using `.parent()` and `.file_name()`.
@@ -47,9 +48,9 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
}
}
-pub(crate) const MD5_LEN: usize = 16;
+const MD5_LEN: usize = 16;
-pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
+fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
if hash.kind == SourceFileHashAlgorithm::Md5 {
let mut buf = [0u8; MD5_LEN];
buf.copy_from_slice(hash.hash_bytes());
@@ -59,160 +60,132 @@ pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
}
}
-fn line_program_add_file(
- line_program: &mut LineProgram,
- line_strings: &mut LineStringTable,
- file: &SourceFile,
-) -> FileId {
- match &file.name {
- FileName::Real(path) => {
- let (dir_path, file_name) = split_path_dir_and_file(path.remapped_path_if_available());
- let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str());
- let file_name = osstr_as_utf8_bytes(file_name);
-
- let dir_id = if !dir_name.is_empty() {
- let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings);
- line_program.add_directory(dir_name)
- } else {
- line_program.default_directory()
- };
- let file_name = LineString::new(file_name, line_program.encoding(), line_strings);
+impl DebugContext {
+ pub(crate) fn get_span_loc(
+ tcx: TyCtxt<'_>,
+ function_span: Span,
+ span: Span,
+ ) -> (Lrc<SourceFile>, u64, u64) {
+ // Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131
+ // In order to have a good line stepping behavior in debugger, we overwrite debug
+ // locations of macro expansions with that of the outermost expansion site (when the macro is
+ // annotated with `#[collapse_debuginfo]` or when `-Zdebug-macros` is provided).
+ let span = if tcx.should_collapse_debuginfo(span) {
+ span
+ } else {
+ // Walk up the macro expansion chain until we reach a non-expanded span.
+ // We also stop at the function body level because no line stepping can occur
+ // at the level above that.
+ rustc_span::hygiene::walk_chain(span, function_span.ctxt())
+ };
- let info = make_file_info(file.src_hash);
+ match tcx.sess.source_map().lookup_line(span.lo()) {
+ Ok(SourceFileAndLine { sf: file, line }) => {
+ let line_pos = file.line_begin_pos(span.lo());
- line_program.file_has_md5 &= info.is_some();
- line_program.add_file(file_name, dir_id, info)
+ (
+ file,
+ u64::try_from(line).unwrap() + 1,
+ u64::from((span.lo() - line_pos).to_u32()) + 1,
+ )
+ }
+ Err(file) => (file, 0, 0),
}
- // FIXME give more appropriate file names
- filename => {
- let dir_id = line_program.default_directory();
- let dummy_file_name = LineString::new(
- filename.prefer_remapped().to_string().into_bytes(),
- line_program.encoding(),
- line_strings,
- );
- line_program.add_file(dummy_file_name, dir_id, None)
+ }
+
+ pub(crate) fn add_source_file(&mut self, source_file: &SourceFile) -> FileId {
+ let line_program: &mut LineProgram = &mut self.dwarf.unit.line_program;
+ let line_strings: &mut LineStringTable = &mut self.dwarf.line_strings;
+
+ match &source_file.name {
+ FileName::Real(path) => {
+ let (dir_path, file_name) =
+ split_path_dir_and_file(path.remapped_path_if_available());
+ let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str());
+ let file_name = osstr_as_utf8_bytes(file_name);
+
+ let dir_id = if !dir_name.is_empty() {
+ let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings);
+ line_program.add_directory(dir_name)
+ } else {
+ line_program.default_directory()
+ };
+ let file_name = LineString::new(file_name, line_program.encoding(), line_strings);
+
+ let info = make_file_info(source_file.src_hash);
+
+ line_program.file_has_md5 &= info.is_some();
+ line_program.add_file(file_name, dir_id, info)
+ }
+ // FIXME give more appropriate file names
+ filename => {
+ let dir_id = line_program.default_directory();
+ let dummy_file_name = LineString::new(
+ filename.prefer_remapped().to_string().into_bytes(),
+ line_program.encoding(),
+ line_strings,
+ );
+ line_program.add_file(dummy_file_name, dir_id, None)
+ }
}
}
}
-impl<'tcx> DebugContext<'tcx> {
- pub(super) fn emit_location(&mut self, entry_id: UnitEntryId, span: Span) {
- let loc = self.tcx.sess.source_map().lookup_char_pos(span.lo());
-
- let file_id = line_program_add_file(
- &mut self.dwarf.unit.line_program,
- &mut self.dwarf.line_strings,
- &loc.file,
- );
-
- let entry = self.dwarf.unit.get_mut(entry_id);
-
- entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
- entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(loc.line as u64));
- entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(loc.col.to_usize() as u64));
+impl FunctionDebugContext {
+ pub(crate) fn add_dbg_loc(&mut self, file_id: FileId, line: u64, column: u64) -> SourceLoc {
+ let (index, _) = self.source_loc_set.insert_full((file_id, line, column));
+ SourceLoc::new(u32::try_from(index).unwrap())
}
pub(super) fn create_debug_lines(
&mut self,
+ debug_context: &mut DebugContext,
symbol: usize,
- entry_id: UnitEntryId,
context: &Context,
- function_span: Span,
- source_info_set: &indexmap::IndexSet<SourceInfo>,
) -> CodeOffset {
- let tcx = self.tcx;
- let line_program = &mut self.dwarf.unit.line_program;
-
- let line_strings = &mut self.dwarf.line_strings;
- let mut last_span = None;
- let mut last_file = None;
- let mut create_row_for_span = |line_program: &mut LineProgram, span: Span| {
- if let Some(last_span) = last_span {
- if span == last_span {
- line_program.generate_row();
- return;
- }
- }
- last_span = Some(span);
-
- // Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131
- // In order to have a good line stepping behavior in debugger, we overwrite debug
- // locations of macro expansions with that of the outermost expansion site
- // (unless the crate is being compiled with `-Z debug-macros`).
- let span = if !span.from_expansion() || tcx.sess.opts.unstable_opts.debug_macros {
- span
- } else {
- // Walk up the macro expansion chain until we reach a non-expanded span.
- // We also stop at the function body level because no line stepping can occur
- // at the level above that.
- rustc_span::hygiene::walk_chain(span, function_span.ctxt())
+ let create_row_for_span =
+ |debug_context: &mut DebugContext, source_loc: (FileId, u64, u64)| {
+ let (file_id, line, col) = source_loc;
+
+ debug_context.dwarf.unit.line_program.row().file = file_id;
+ debug_context.dwarf.unit.line_program.row().line = line;
+ debug_context.dwarf.unit.line_program.row().column = col;
+ debug_context.dwarf.unit.line_program.generate_row();
};
- let (file, line, col) = match tcx.sess.source_map().lookup_line(span.lo()) {
- Ok(SourceFileAndLine { sf: file, line }) => {
- let line_pos = file.line_begin_pos(span.lo());
-
- (
- file,
- u64::try_from(line).unwrap() + 1,
- u64::from((span.lo() - line_pos).to_u32()) + 1,
- )
- }
- Err(file) => (file, 0, 0),
- };
-
- // line_program_add_file is very slow.
- // Optimize for the common case of the current file not being changed.
- let current_file_changed = if let Some(last_file) = &last_file {
- // If the allocations are not equal, then the files may still be equal, but that
- // is not a problem, as this is just an optimization.
- !rustc_data_structures::sync::Lrc::ptr_eq(last_file, &file)
- } else {
- true
- };
- if current_file_changed {
- let file_id = line_program_add_file(line_program, line_strings, &file);
- line_program.row().file = file_id;
- last_file = Some(file);
- }
-
- line_program.row().line = line;
- line_program.row().column = col;
- line_program.generate_row();
- };
-
- line_program.begin_sequence(Some(Address::Symbol { symbol, addend: 0 }));
+ debug_context
+ .dwarf
+ .unit
+ .line_program
+ .begin_sequence(Some(Address::Symbol { symbol, addend: 0 }));
let mut func_end = 0;
- let mcr = context.mach_compile_result.as_ref().unwrap();
+ let mcr = context.compiled_code().unwrap();
for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
- line_program.row().address_offset = u64::from(start);
+ debug_context.dwarf.unit.line_program.row().address_offset = u64::from(start);
if !loc.is_default() {
- let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap();
- create_row_for_span(line_program, source_info.span);
+ let source_loc = *self.source_loc_set.get_index(loc.bits() as usize).unwrap();
+ create_row_for_span(debug_context, source_loc);
} else {
- create_row_for_span(line_program, function_span);
+ create_row_for_span(debug_context, self.function_source_loc);
}
func_end = end;
}
- line_program.end_sequence(u64::from(func_end));
+ debug_context.dwarf.unit.line_program.end_sequence(u64::from(func_end));
let func_end = mcr.buffer.total_size();
assert_ne!(func_end, 0);
- let entry = self.dwarf.unit.get_mut(entry_id);
+ let entry = debug_context.dwarf.unit.get_mut(self.entry_id);
entry.set(
gimli::DW_AT_low_pc,
AttributeValue::Address(Address::Symbol { symbol, addend: 0 }),
);
entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(func_end)));
- self.emit_location(entry_id, function_span);
-
func_end
}
}
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
index 693092ba5..c55db2017 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
@@ -7,35 +7,34 @@ mod unwind;
use crate::prelude::*;
-use rustc_index::vec::IndexVec;
-
-use cranelift_codegen::entity::EntityRef;
-use cranelift_codegen::ir::{Endianness, LabelValueLoc, ValueLabel};
+use cranelift_codegen::ir::Endianness;
use cranelift_codegen::isa::TargetIsa;
-use cranelift_codegen::ValueLocRange;
use gimli::write::{
- Address, AttributeValue, DwarfUnit, Expression, LineProgram, LineString, Location,
- LocationList, Range, RangeList, UnitEntryId,
+ Address, AttributeValue, DwarfUnit, FileId, LineProgram, LineString, Range, RangeList,
+ UnitEntryId,
};
-use gimli::{Encoding, Format, LineEncoding, RunTimeEndian, X86_64};
+use gimli::{Encoding, Format, LineEncoding, RunTimeEndian};
+use indexmap::IndexSet;
pub(crate) use emit::{DebugReloc, DebugRelocName};
pub(crate) use unwind::UnwindContext;
-pub(crate) struct DebugContext<'tcx> {
- tcx: TyCtxt<'tcx>,
-
+pub(crate) struct DebugContext {
endian: RunTimeEndian,
dwarf: DwarfUnit,
unit_range_list: RangeList,
+}
- types: FxHashMap<Ty<'tcx>, UnitEntryId>,
+pub(crate) struct FunctionDebugContext {
+ entry_id: UnitEntryId,
+ function_source_loc: (FileId, u64, u64),
+ source_loc_set: indexmap::IndexSet<(FileId, u64, u64)>,
}
-impl<'tcx> DebugContext<'tcx> {
- pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self {
+impl DebugContext {
+ pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa) -> Self {
let encoding = Encoding {
format: Format::Dwarf32,
// FIXME this should be configurable
@@ -101,127 +100,18 @@ impl<'tcx> DebugContext<'tcx> {
root.set(gimli::DW_AT_low_pc, AttributeValue::Address(Address::Constant(0)));
}
- DebugContext {
- tcx,
-
- endian,
-
- dwarf,
- unit_range_list: RangeList(Vec::new()),
-
- types: FxHashMap::default(),
- }
- }
-
- fn dwarf_ty(&mut self, ty: Ty<'tcx>) -> UnitEntryId {
- if let Some(type_id) = self.types.get(&ty) {
- return *type_id;
- }
-
- let new_entry = |dwarf: &mut DwarfUnit, tag| dwarf.unit.add(dwarf.unit.root(), tag);
-
- let primitive = |dwarf: &mut DwarfUnit, ate| {
- let type_id = new_entry(dwarf, gimli::DW_TAG_base_type);
- let type_entry = dwarf.unit.get_mut(type_id);
- type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(ate));
- type_id
- };
-
- let name = format!("{}", ty);
- let layout = self.tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap();
-
- let type_id = match ty.kind() {
- ty::Bool => primitive(&mut self.dwarf, gimli::DW_ATE_boolean),
- ty::Char => primitive(&mut self.dwarf, gimli::DW_ATE_UTF),
- ty::Uint(_) => primitive(&mut self.dwarf, gimli::DW_ATE_unsigned),
- ty::Int(_) => primitive(&mut self.dwarf, gimli::DW_ATE_signed),
- ty::Float(_) => primitive(&mut self.dwarf, gimli::DW_ATE_float),
- ty::Ref(_, pointee_ty, _mutbl)
- | ty::RawPtr(ty::TypeAndMut { ty: pointee_ty, mutbl: _mutbl }) => {
- let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_pointer_type);
-
- // Ensure that type is inserted before recursing to avoid duplicates
- self.types.insert(ty, type_id);
-
- let pointee = self.dwarf_ty(*pointee_ty);
-
- let type_entry = self.dwarf.unit.get_mut(type_id);
-
- //type_entry.set(gimli::DW_AT_mutable, AttributeValue::Flag(mutbl == rustc_hir::Mutability::Mut));
- type_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(pointee));
-
- type_id
- }
- ty::Adt(adt_def, _substs) if adt_def.is_struct() && !layout.is_unsized() => {
- let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type);
-
- // Ensure that type is inserted before recursing to avoid duplicates
- self.types.insert(ty, type_id);
-
- let variant = adt_def.non_enum_variant();
-
- for (field_idx, field_def) in variant.fields.iter().enumerate() {
- let field_offset = layout.fields.offset(field_idx);
- let field_layout = layout.field(
- &layout::LayoutCx { tcx: self.tcx, param_env: ParamEnv::reveal_all() },
- field_idx,
- );
-
- let field_type = self.dwarf_ty(field_layout.ty);
-
- let field_id = self.dwarf.unit.add(type_id, gimli::DW_TAG_member);
- let field_entry = self.dwarf.unit.get_mut(field_id);
-
- field_entry.set(
- gimli::DW_AT_name,
- AttributeValue::String(field_def.name.as_str().to_string().into_bytes()),
- );
- field_entry.set(
- gimli::DW_AT_data_member_location,
- AttributeValue::Udata(field_offset.bytes()),
- );
- field_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(field_type));
- }
-
- type_id
- }
- _ => new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type),
- };
-
- let type_entry = self.dwarf.unit.get_mut(type_id);
-
- type_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
- type_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes()));
-
- self.types.insert(ty, type_id);
-
- type_id
- }
-
- fn define_local(&mut self, scope: UnitEntryId, name: String, ty: Ty<'tcx>) -> UnitEntryId {
- let dw_ty = self.dwarf_ty(ty);
-
- let var_id = self.dwarf.unit.add(scope, gimli::DW_TAG_variable);
- let var_entry = self.dwarf.unit.get_mut(var_id);
-
- var_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
- var_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty));
-
- var_id
+ DebugContext { endian, dwarf, unit_range_list: RangeList(Vec::new()) }
}
pub(crate) fn define_function(
&mut self,
- instance: Instance<'tcx>,
- func_id: FuncId,
+ tcx: TyCtxt<'_>,
name: &str,
- isa: &dyn TargetIsa,
- context: &Context,
- source_info_set: &indexmap::IndexSet<SourceInfo>,
- local_map: IndexVec<mir::Local, CPlace<'tcx>>,
- ) {
- let symbol = func_id.as_u32() as usize;
- let mir = self.tcx.instance_mir(instance.def);
+ function_span: Span,
+ ) -> FunctionDebugContext {
+ let (file, line, column) = DebugContext::get_span_loc(tcx, function_span, function_span);
+
+ let file_id = self.add_source_file(&file);
// FIXME: add to appropriate scope instead of root
let scope = self.dwarf.unit.root();
@@ -233,14 +123,35 @@ impl<'tcx> DebugContext<'tcx> {
entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id));
entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(name_id));
- let end = self.create_debug_lines(symbol, entry_id, context, mir.span, source_info_set);
+ entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
+ entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(line));
+ entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(column));
- self.unit_range_list.0.push(Range::StartLength {
+ FunctionDebugContext {
+ entry_id,
+ function_source_loc: (file_id, line, column),
+ source_loc_set: IndexSet::new(),
+ }
+ }
+}
+
+impl FunctionDebugContext {
+ pub(crate) fn finalize(
+ mut self,
+ debug_context: &mut DebugContext,
+ func_id: FuncId,
+ context: &Context,
+ ) {
+ let symbol = func_id.as_u32() as usize;
+
+ let end = self.create_debug_lines(debug_context, symbol, context);
+
+ debug_context.unit_range_list.0.push(Range::StartLength {
begin: Address::Symbol { symbol, addend: 0 },
length: u64::from(end),
});
- let func_entry = self.dwarf.unit.get_mut(entry_id);
+ let func_entry = debug_context.dwarf.unit.get_mut(self.entry_id);
// Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped.
func_entry.set(
gimli::DW_AT_low_pc,
@@ -248,110 +159,5 @@ impl<'tcx> DebugContext<'tcx> {
);
// Using Udata for DW_AT_high_pc requires at least DWARF4
func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end)));
-
- // FIXME make it more reliable and implement scopes before re-enabling this.
- if false {
- let value_labels_ranges = std::collections::HashMap::new(); // FIXME
-
- for (local, _local_decl) in mir.local_decls.iter_enumerated() {
- let ty = self.tcx.subst_and_normalize_erasing_regions(
- instance.substs,
- ty::ParamEnv::reveal_all(),
- mir.local_decls[local].ty,
- );
- let var_id = self.define_local(entry_id, format!("{:?}", local), ty);
-
- let location = place_location(
- self,
- isa,
- symbol,
- &local_map,
- &value_labels_ranges,
- Place { local, projection: ty::List::empty() },
- );
-
- let var_entry = self.dwarf.unit.get_mut(var_id);
- var_entry.set(gimli::DW_AT_location, location);
- }
- }
-
- // FIXME create locals for all entries in mir.var_debug_info
- }
-}
-
-fn place_location<'tcx>(
- debug_context: &mut DebugContext<'tcx>,
- isa: &dyn TargetIsa,
- symbol: usize,
- local_map: &IndexVec<mir::Local, CPlace<'tcx>>,
- #[allow(rustc::default_hash_types)] value_labels_ranges: &std::collections::HashMap<
- ValueLabel,
- Vec<ValueLocRange>,
- >,
- place: Place<'tcx>,
-) -> AttributeValue {
- assert!(place.projection.is_empty()); // FIXME implement them
-
- match local_map[place.local].inner() {
- CPlaceInner::Var(_local, var) => {
- let value_label = cranelift_codegen::ir::ValueLabel::new(var.index());
- if let Some(value_loc_ranges) = value_labels_ranges.get(&value_label) {
- let loc_list = LocationList(
- value_loc_ranges
- .iter()
- .map(|value_loc_range| Location::StartEnd {
- begin: Address::Symbol {
- symbol,
- addend: i64::from(value_loc_range.start),
- },
- end: Address::Symbol { symbol, addend: i64::from(value_loc_range.end) },
- data: translate_loc(isa, value_loc_range.loc).unwrap(),
- })
- .collect(),
- );
- let loc_list_id = debug_context.dwarf.unit.locations.add(loc_list);
-
- AttributeValue::LocationListRef(loc_list_id)
- } else {
- // FIXME set value labels for unused locals
-
- AttributeValue::Exprloc(Expression::new())
- }
- }
- CPlaceInner::VarPair(_, _, _) => {
- // FIXME implement this
-
- AttributeValue::Exprloc(Expression::new())
- }
- CPlaceInner::VarLane(_, _, _) => {
- // FIXME implement this
-
- AttributeValue::Exprloc(Expression::new())
- }
- CPlaceInner::Addr(_, _) => {
- // FIXME implement this (used by arguments and returns)
-
- AttributeValue::Exprloc(Expression::new())
-
- // For PointerBase::Stack:
- //AttributeValue::Exprloc(translate_loc(ValueLoc::Stack(*stack_slot)).unwrap())
- }
- }
-}
-
-// Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137
-fn translate_loc(isa: &dyn TargetIsa, loc: LabelValueLoc) -> Option<Expression> {
- match loc {
- LabelValueLoc::Reg(reg) => {
- let machine_reg = isa.map_regalloc_reg_to_dwarf(reg).unwrap();
- let mut expr = Expression::new();
- expr.op_reg(gimli::Register(machine_reg));
- Some(expr)
- }
- LabelValueLoc::SPOffset(offset) => {
- let mut expr = Expression::new();
- expr.op_breg(X86_64::RSP, offset);
- Some(expr)
- }
}
}
diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs
index f619bb5ed..97b395bcd 100644
--- a/compiler/rustc_codegen_cranelift/src/discriminant.rs
+++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs
@@ -42,10 +42,10 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
Variants::Multiple {
tag: _,
tag_field,
- tag_encoding: TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
+ tag_encoding: TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
variants: _,
} => {
- if variant_index != dataful_variant {
+ if variant_index != untagged_variant {
let niche = place.place_field(fx, mir::Field::new(tag_field));
let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
let niche_value = ty::ScalarInt::try_from_uint(
@@ -62,16 +62,14 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
pub(crate) fn codegen_get_discriminant<'tcx>(
fx: &mut FunctionCx<'_, '_, 'tcx>,
+ dest: CPlace<'tcx>,
value: CValue<'tcx>,
dest_layout: TyAndLayout<'tcx>,
-) -> CValue<'tcx> {
+) {
let layout = value.layout();
- if layout.abi == Abi::Uninhabited {
- let true_ = fx.bcx.ins().iconst(types::I32, 1);
- fx.bcx.ins().trapnz(true_, TrapCode::UnreachableCodeReached);
- // Return a dummy value
- return CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout);
+ if layout.abi.is_uninhabited() {
+ return;
}
let (tag_scalar, tag_field, tag_encoding) = match &layout.variants {
@@ -89,7 +87,9 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
} else {
ty::ScalarInt::try_from_uint(discr_val, dest_layout.size).unwrap()
};
- return CValue::const_val(fx, dest_layout, discr_val);
+ let res = CValue::const_val(fx, dest_layout, discr_val);
+ dest.write_cvalue(fx, res);
+ return;
}
Variants::Multiple { tag, tag_field, tag_encoding, variants: _ } => {
(tag, *tag_field, tag_encoding)
@@ -110,9 +110,10 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
_ => false,
};
let val = clif_intcast(fx, tag, cast_to, signed);
- CValue::by_val(val, dest_layout)
+ let res = CValue::by_val(val, dest_layout);
+ dest.write_cvalue(fx, res);
}
- TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
+ TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
// Rebase from niche values to discriminants, and check
// whether the result is in range for the niche variants.
@@ -168,9 +169,11 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
fx.bcx.ins().iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
};
- let dataful_variant = fx.bcx.ins().iconst(cast_to, i64::from(dataful_variant.as_u32()));
- let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant);
- CValue::by_val(discr, dest_layout)
+ let untagged_variant =
+ fx.bcx.ins().iconst(cast_to, i64::from(untagged_variant.as_u32()));
+ let discr = fx.bcx.ins().select(is_niche, niche_discr, untagged_variant);
+ let res = CValue::by_val(discr, dest_layout);
+ dest.write_cvalue(fx, res);
}
}
}
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index 3cd1ef563..8eabe1cbc 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -1,33 +1,129 @@
//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a
//! standalone executable.
+use std::fs::File;
use std::path::PathBuf;
+use std::sync::Arc;
+use std::thread::JoinHandle;
-use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file;
use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
+use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
use rustc_metadata::EncodedMetadata;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
use rustc_session::cgu_reuse_tracker::CguReuse;
-use rustc_session::config::{DebugInfo, OutputType};
+use rustc_session::config::{DebugInfo, OutputFilenames, OutputType};
use rustc_session::Session;
-use cranelift_codegen::isa::TargetIsa;
use cranelift_object::{ObjectBuilder, ObjectModule};
+use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken};
+use crate::global_asm::GlobalAsmConfig;
use crate::{prelude::*, BackendConfig};
-struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>);
+struct ModuleCodegenResult {
+ module_regular: CompiledModule,
+ module_global_asm: Option<CompiledModule>,
+ existing_work_product: Option<(WorkProductId, WorkProduct)>,
+}
+
+enum OngoingModuleCodegen {
+ Sync(Result<ModuleCodegenResult, String>),
+ Async(JoinHandle<Result<ModuleCodegenResult, String>>),
+}
-impl<HCX> HashStable<HCX> for ModuleCodegenResult {
+impl<HCX> HashStable<HCX> for OngoingModuleCodegen {
fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) {
// do nothing
}
}
-fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectModule {
+pub(crate) struct OngoingCodegen {
+ modules: Vec<OngoingModuleCodegen>,
+ allocator_module: Option<CompiledModule>,
+ metadata_module: Option<CompiledModule>,
+ metadata: EncodedMetadata,
+ crate_info: CrateInfo,
+ concurrency_limiter: ConcurrencyLimiter,
+}
+
+impl OngoingCodegen {
+ pub(crate) fn join(
+ self,
+ sess: &Session,
+ backend_config: &BackendConfig,
+ ) -> (CodegenResults, FxHashMap<WorkProductId, WorkProduct>) {
+ let mut work_products = FxHashMap::default();
+ let mut modules = vec![];
+
+ for module_codegen in self.modules {
+ let module_codegen_result = match module_codegen {
+ OngoingModuleCodegen::Sync(module_codegen_result) => module_codegen_result,
+ OngoingModuleCodegen::Async(join_handle) => match join_handle.join() {
+ Ok(module_codegen_result) => module_codegen_result,
+ Err(panic) => std::panic::resume_unwind(panic),
+ },
+ };
+
+ let module_codegen_result = match module_codegen_result {
+ Ok(module_codegen_result) => module_codegen_result,
+ Err(err) => sess.fatal(&err),
+ };
+ let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } =
+ module_codegen_result;
+
+ if let Some((work_product_id, work_product)) = existing_work_product {
+ work_products.insert(work_product_id, work_product);
+ } else {
+ let work_product = if backend_config.disable_incr_cache {
+ None
+ } else if let Some(module_global_asm) = &module_global_asm {
+ rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
+ sess,
+ &module_regular.name,
+ &[
+ ("o", &module_regular.object.as_ref().unwrap()),
+ ("asm.o", &module_global_asm.object.as_ref().unwrap()),
+ ],
+ )
+ } else {
+ rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
+ sess,
+ &module_regular.name,
+ &[("o", &module_regular.object.as_ref().unwrap())],
+ )
+ };
+ if let Some((work_product_id, work_product)) = work_product {
+ work_products.insert(work_product_id, work_product);
+ }
+ }
+
+ modules.push(module_regular);
+ if let Some(module_global_asm) = module_global_asm {
+ modules.push(module_global_asm);
+ }
+ }
+
+ drop(self.concurrency_limiter);
+
+ (
+ CodegenResults {
+ modules,
+ allocator_module: self.allocator_module,
+ metadata_module: self.metadata_module,
+ metadata: self.metadata,
+ crate_info: self.crate_info,
+ },
+ work_products,
+ )
+ }
+}
+
+fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> ObjectModule {
+ let isa = crate::build_isa(sess, backend_config);
+
let mut builder =
ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();
// Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size
@@ -37,15 +133,15 @@ fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectM
ObjectModule::new(builder)
}
-fn emit_module(
- tcx: TyCtxt<'_>,
- backend_config: &BackendConfig,
+fn emit_cgu(
+ output_filenames: &OutputFilenames,
+ prof: &SelfProfilerRef,
name: String,
- kind: ModuleKind,
module: ObjectModule,
- debug: Option<DebugContext<'_>>,
+ debug: Option<DebugContext>,
unwind_context: UnwindContext,
-) -> ModuleCodegenResult {
+ global_asm_object_file: Option<PathBuf>,
+) -> Result<ModuleCodegenResult, String> {
let mut product = module.finish();
if let Some(mut debug) = debug {
@@ -54,134 +150,191 @@ fn emit_module(
unwind_context.emit(&mut product);
- let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name));
- let obj = product.object.write().unwrap();
+ let module_regular =
+ emit_module(output_filenames, prof, product.object, ModuleKind::Regular, name.clone())?;
+
+ Ok(ModuleCodegenResult {
+ module_regular,
+ module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule {
+ name: format!("{name}.asm"),
+ kind: ModuleKind::Regular,
+ object: Some(global_asm_object_file),
+ dwarf_object: None,
+ bytecode: None,
+ }),
+ existing_work_product: None,
+ })
+}
- tcx.sess.prof.artifact_size("object_file", name.clone(), obj.len().try_into().unwrap());
+fn emit_module(
+ output_filenames: &OutputFilenames,
+ prof: &SelfProfilerRef,
+ object: cranelift_object::object::write::Object<'_>,
+ kind: ModuleKind,
+ name: String,
+) -> Result<CompiledModule, String> {
+ let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name));
+ let mut file = match File::create(&tmp_file) {
+ Ok(file) => file,
+ Err(err) => return Err(format!("error creating object file: {}", err)),
+ };
- if let Err(err) = std::fs::write(&tmp_file, obj) {
- tcx.sess.fatal(&format!("error writing object file: {}", err));
+ if let Err(err) = object.write_stream(&mut file) {
+ return Err(format!("error writing object file: {}", err));
}
- let work_product = if backend_config.disable_incr_cache {
- None
- } else {
- rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
- tcx.sess,
- &name,
- &[("o", &tmp_file)],
- )
- };
+ prof.artifact_size("object_file", &*name, file.metadata().unwrap().len());
- ModuleCodegenResult(
- CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None },
- work_product,
- )
+ Ok(CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None })
}
fn reuse_workproduct_for_cgu(
tcx: TyCtxt<'_>,
cgu: &CodegenUnit<'_>,
- work_products: &mut FxHashMap<WorkProductId, WorkProduct>,
-) -> CompiledModule {
+) -> Result<ModuleCodegenResult, String> {
let work_product = cgu.previous_work_product(tcx);
- let obj_out = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str()));
- let source_file = rustc_incremental::in_incr_comp_dir_sess(
+ let obj_out_regular =
+ tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str()));
+ let source_file_regular = rustc_incremental::in_incr_comp_dir_sess(
&tcx.sess,
&work_product.saved_files.get("o").expect("no saved object file in work product"),
);
- if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) {
- tcx.sess.err(&format!(
+
+ if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) {
+ return Err(format!(
"unable to copy {} to {}: {}",
- source_file.display(),
- obj_out.display(),
+ source_file_regular.display(),
+ obj_out_regular.display(),
err
));
}
+ let obj_out_global_asm =
+ crate::global_asm::add_file_stem_postfix(obj_out_regular.clone(), ".asm");
+ let has_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") {
+ let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, asm_o);
+ if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm)
+ {
+ return Err(format!(
+ "unable to copy {} to {}: {}",
+ source_file_regular.display(),
+ obj_out_regular.display(),
+ err
+ ));
+ }
+ true
+ } else {
+ false
+ };
- work_products.insert(cgu.work_product_id(), work_product);
-
- CompiledModule {
- name: cgu.name().to_string(),
- kind: ModuleKind::Regular,
- object: Some(obj_out),
- dwarf_object: None,
- bytecode: None,
- }
+ Ok(ModuleCodegenResult {
+ module_regular: CompiledModule {
+ name: cgu.name().to_string(),
+ kind: ModuleKind::Regular,
+ object: Some(obj_out_regular),
+ dwarf_object: None,
+ bytecode: None,
+ },
+ module_global_asm: if has_global_asm {
+ Some(CompiledModule {
+ name: cgu.name().to_string(),
+ kind: ModuleKind::Regular,
+ object: Some(obj_out_global_asm),
+ dwarf_object: None,
+ bytecode: None,
+ })
+ } else {
+ None
+ },
+ existing_work_product: Some((cgu.work_product_id(), work_product)),
+ })
}
fn module_codegen(
tcx: TyCtxt<'_>,
- (backend_config, cgu_name): (BackendConfig, rustc_span::Symbol),
-) -> ModuleCodegenResult {
- let cgu = tcx.codegen_unit(cgu_name);
- let mono_items = cgu.items_in_deterministic_order(tcx);
-
- let isa = crate::build_isa(tcx.sess, &backend_config);
- let mut module = make_module(tcx.sess, isa, cgu_name.as_str().to_string());
-
- let mut cx = crate::CodegenCx::new(
- tcx,
- backend_config.clone(),
- module.isa(),
- tcx.sess.opts.debuginfo != DebugInfo::None,
- cgu_name,
- );
- super::predefine_mono_items(tcx, &mut module, &mono_items);
- for (mono_item, _) in mono_items {
- match mono_item {
- MonoItem::Fn(inst) => {
- cx.tcx
- .sess
- .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst));
- }
- MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id),
- MonoItem::GlobalAsm(item_id) => {
- let item = cx.tcx.hir().item(item_id);
- if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind {
- if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
- cx.global_asm.push_str("\n.intel_syntax noprefix\n");
- } else {
- cx.global_asm.push_str("\n.att_syntax\n");
- }
- for piece in asm.template {
- match *piece {
- InlineAsmTemplatePiece::String(ref s) => cx.global_asm.push_str(s),
- InlineAsmTemplatePiece::Placeholder { .. } => todo!(),
- }
- }
- cx.global_asm.push_str("\n.att_syntax\n\n");
- } else {
- bug!("Expected GlobalAsm found {:?}", item);
+ (backend_config, global_asm_config, cgu_name, token): (
+ BackendConfig,
+ Arc<GlobalAsmConfig>,
+ rustc_span::Symbol,
+ ConcurrencyLimiterToken,
+ ),
+) -> OngoingModuleCodegen {
+ let (cgu_name, mut cx, mut module, codegened_functions) = tcx.sess.time("codegen cgu", || {
+ let cgu = tcx.codegen_unit(cgu_name);
+ let mono_items = cgu.items_in_deterministic_order(tcx);
+
+ let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string());
+
+ let mut cx = crate::CodegenCx::new(
+ tcx,
+ backend_config.clone(),
+ module.isa(),
+ tcx.sess.opts.debuginfo != DebugInfo::None,
+ cgu_name,
+ );
+ super::predefine_mono_items(tcx, &mut module, &mono_items);
+ let mut codegened_functions = vec![];
+ for (mono_item, _) in mono_items {
+ match mono_item {
+ MonoItem::Fn(inst) => {
+ tcx.sess.time("codegen fn", || {
+ let codegened_function = crate::base::codegen_fn(
+ tcx,
+ &mut cx,
+ Function::new(),
+ &mut module,
+ inst,
+ );
+ codegened_functions.push(codegened_function);
+ });
+ }
+ MonoItem::Static(def_id) => {
+ crate::constant::codegen_static(tcx, &mut module, def_id)
+ }
+ MonoItem::GlobalAsm(item_id) => {
+ crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id);
}
}
}
- }
- crate::main_shim::maybe_create_entry_wrapper(
- tcx,
- &mut module,
- &mut cx.unwind_context,
- false,
- cgu.is_primary(),
- );
-
- let debug_context = cx.debug_context;
- let unwind_context = cx.unwind_context;
- let codegen_result = tcx.sess.time("write object file", || {
- emit_module(
+ crate::main_shim::maybe_create_entry_wrapper(
tcx,
- &backend_config,
- cgu.name().as_str().to_string(),
- ModuleKind::Regular,
- module,
- debug_context,
- unwind_context,
- )
+ &mut module,
+ &mut cx.unwind_context,
+ false,
+ cgu.is_primary(),
+ );
+
+ let cgu_name = cgu.name().as_str().to_owned();
+
+ (cgu_name, cx, module, codegened_functions)
});
- codegen_global_asm(tcx, cgu.name().as_str(), &cx.global_asm);
+ OngoingModuleCodegen::Async(std::thread::spawn(move || {
+ cx.profiler.clone().verbose_generic_activity("compile functions").run(|| {
+ let mut cached_context = Context::new();
+ for codegened_func in codegened_functions {
+ crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func);
+ }
+ });
- codegen_result
+ let global_asm_object_file =
+ cx.profiler.verbose_generic_activity("compile assembly").run(|| {
+ crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm)
+ })?;
+
+ let codegen_result = cx.profiler.verbose_generic_activity("write object file").run(|| {
+ emit_cgu(
+ &global_asm_config.output_filenames,
+ &cx.profiler,
+ cgu_name,
+ module,
+ cx.debug_context,
+ cx.unwind_context,
+ global_asm_object_file,
+ )
+ });
+ std::mem::drop(token);
+ codegen_result
+ }))
}
pub(crate) fn run_aot(
@@ -189,9 +342,7 @@ pub(crate) fn run_aot(
backend_config: BackendConfig,
metadata: EncodedMetadata,
need_metadata_module: bool,
-) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
- let mut work_products = FxHashMap::default();
-
+) -> Box<OngoingCodegen> {
let cgus = if tcx.sess.opts.output_types.should_codegen() {
tcx.collect_and_partition_mono_items(()).1
} else {
@@ -206,62 +357,69 @@ pub(crate) fn run_aot(
}
}
+ let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx));
+
+ let mut concurrency_limiter = ConcurrencyLimiter::new(tcx.sess, cgus.len());
+
let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
cgus.iter()
.map(|cgu| {
- let cgu_reuse = determine_cgu_reuse(tcx, cgu);
+ let cgu_reuse = if backend_config.disable_incr_cache {
+ CguReuse::No
+ } else {
+ determine_cgu_reuse(tcx, cgu)
+ };
tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse);
match cgu_reuse {
- _ if backend_config.disable_incr_cache => {}
- CguReuse::No => {}
- CguReuse::PreLto => {
- return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products);
+ CguReuse::No => {
+ let dep_node = cgu.codegen_dep_node(tcx);
+ tcx.dep_graph
+ .with_task(
+ dep_node,
+ tcx,
+ (
+ backend_config.clone(),
+ global_asm_config.clone(),
+ cgu.name(),
+ concurrency_limiter.acquire(),
+ ),
+ module_codegen,
+ Some(rustc_middle::dep_graph::hash_result),
+ )
+ .0
+ }
+ CguReuse::PreLto => unreachable!(),
+ CguReuse::PostLto => {
+ concurrency_limiter.job_already_done();
+ OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, &*cgu))
}
- CguReuse::PostLto => unreachable!(),
- }
-
- let dep_node = cgu.codegen_dep_node(tcx);
- let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task(
- dep_node,
- tcx,
- (backend_config.clone(), cgu.name()),
- module_codegen,
- Some(rustc_middle::dep_graph::hash_result),
- );
-
- if let Some((id, product)) = work_product {
- work_products.insert(id, product);
}
-
- module
})
.collect::<Vec<_>>()
});
tcx.sess.abort_if_errors();
- let isa = crate::build_isa(tcx.sess, &backend_config);
- let mut allocator_module = make_module(tcx.sess, isa, "allocator_shim".to_string());
- assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type());
+ let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string());
let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true);
let created_alloc_shim =
crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
let allocator_module = if created_alloc_shim {
- let ModuleCodegenResult(module, work_product) = emit_module(
- tcx,
- &backend_config,
- "allocator_shim".to_string(),
+ let mut product = allocator_module.finish();
+ allocator_unwind_context.emit(&mut product);
+
+ match emit_module(
+ tcx.output_filenames(()),
+ &tcx.sess.prof,
+ product.object,
ModuleKind::Allocator,
- allocator_module,
- None,
- allocator_unwind_context,
- );
- if let Some((id, product)) = work_product {
- work_products.insert(id, product);
+ "allocator_shim".to_owned(),
+ ) {
+ Ok(allocator_module) => Some(allocator_module),
+ Err(err) => tcx.sess.fatal(err),
}
- Some(module)
} else {
None
};
@@ -308,102 +466,14 @@ pub(crate) fn run_aot(
}
.to_owned();
- Box::new((
- CodegenResults {
- modules,
- allocator_module,
- metadata_module,
- metadata,
- crate_info: CrateInfo::new(tcx, target_cpu),
- },
- work_products,
- ))
-}
-
-fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
- use std::io::Write;
- use std::process::{Command, Stdio};
-
- if global_asm.is_empty() {
- return;
- }
-
- if cfg!(not(feature = "inline_asm"))
- || tcx.sess.target.is_like_osx
- || tcx.sess.target.is_like_windows
- {
- if global_asm.contains("__rust_probestack") {
- return;
- }
-
- // FIXME fix linker error on macOS
- if cfg!(not(feature = "inline_asm")) {
- tcx.sess.fatal(
- "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift",
- );
- } else {
- tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows");
- }
- }
-
- let assembler = crate::toolchain::get_toolchain_binary(tcx.sess, "as");
- let linker = crate::toolchain::get_toolchain_binary(tcx.sess, "ld");
-
- // Remove all LLVM style comments
- let global_asm = global_asm
- .lines()
- .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
- .collect::<Vec<_>>()
- .join("\n");
-
- let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name));
-
- // Assemble `global_asm`
- let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
- let mut child = Command::new(assembler)
- .arg("-o")
- .arg(&global_asm_object_file)
- .stdin(Stdio::piped())
- .spawn()
- .expect("Failed to spawn `as`.");
- child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
- let status = child.wait().expect("Failed to wait for `as`.");
- if !status.success() {
- tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm));
- }
-
- // Link the global asm and main object file together
- let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main");
- std::fs::rename(&output_object_file, &main_object_file).unwrap();
- let status = Command::new(linker)
- .arg("-r") // Create a new object file
- .arg("-o")
- .arg(output_object_file)
- .arg(&main_object_file)
- .arg(&global_asm_object_file)
- .status()
- .unwrap();
- if !status.success() {
- tcx.sess.fatal(&format!(
- "Failed to link `{}` and `{}` together",
- main_object_file.display(),
- global_asm_object_file.display(),
- ));
- }
-
- std::fs::remove_file(global_asm_object_file).unwrap();
- std::fs::remove_file(main_object_file).unwrap();
-}
-
-fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf {
- let mut new_filename = path.file_stem().unwrap().to_owned();
- new_filename.push(postfix);
- if let Some(extension) = path.extension() {
- new_filename.push(".");
- new_filename.push(extension);
- }
- path.set_file_name(new_filename);
- path
+ Box::new(OngoingCodegen {
+ modules,
+ allocator_module,
+ metadata_module,
+ metadata,
+ crate_info: CrateInfo::new(tcx, target_cpu),
+ concurrency_limiter,
+ })
}
// Adapted from https://github.com/rust-lang/rust/blob/303d8aff6092709edd4dbd35b1c88e9aa40bf6d8/src/librustc_codegen_ssa/base.rs#L922-L953
@@ -432,5 +502,5 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR
cgu.name()
);
- if tcx.try_mark_green(&dep_node) { CguReuse::PreLto } else { CguReuse::No }
+ if tcx.try_mark_green(&dep_node) { CguReuse::PostLto } else { CguReuse::No }
}
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index a56a91000..0e77e4004 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -61,11 +61,11 @@ impl UnsafeMessage {
}
}
-fn create_jit_module<'tcx>(
- tcx: TyCtxt<'tcx>,
+fn create_jit_module(
+ tcx: TyCtxt<'_>,
backend_config: &BackendConfig,
hotswap: bool,
-) -> (JITModule, CodegenCx<'tcx>) {
+) -> (JITModule, CodegenCx) {
let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string());
let imported_symbols = load_imported_symbols_for_jit(tcx.sess, crate_info);
@@ -111,6 +111,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
&backend_config,
matches!(backend_config.codegen_mode, CodegenMode::JitLazy),
);
+ let mut cached_context = Context::new();
let (_, cgus) = tcx.collect_and_partition_mono_items(());
let mono_items = cgus
@@ -128,11 +129,19 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
MonoItem::Fn(inst) => match backend_config.codegen_mode {
CodegenMode::Aot => unreachable!(),
CodegenMode::Jit => {
- cx.tcx.sess.time("codegen fn", || {
- crate::base::codegen_fn(&mut cx, &mut jit_module, inst)
+ tcx.sess.time("codegen fn", || {
+ crate::base::codegen_and_compile_fn(
+ tcx,
+ &mut cx,
+ &mut cached_context,
+ &mut jit_module,
+ inst,
+ )
});
}
- CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst),
+ CodegenMode::JitLazy => {
+ codegen_shim(tcx, &mut cx, &mut cached_context, &mut jit_module, inst)
+ }
},
MonoItem::Static(def_id) => {
crate::constant::codegen_static(tcx, &mut jit_module, def_id);
@@ -259,7 +268,15 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) ->
false,
Symbol::intern("dummy_cgu_name"),
);
- tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
+ tcx.sess.time("codegen fn", || {
+ crate::base::codegen_and_compile_fn(
+ tcx,
+ &mut cx,
+ &mut Context::new(),
+ jit_module,
+ instance,
+ )
+ });
assert!(cx.global_asm.is_empty());
jit_module.finalize_definitions();
@@ -334,9 +351,13 @@ fn load_imported_symbols_for_jit(
imported_symbols
}
-fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) {
- let tcx = cx.tcx;
-
+fn codegen_shim<'tcx>(
+ tcx: TyCtxt<'tcx>,
+ cx: &mut CodegenCx,
+ cached_context: &mut Context,
+ module: &mut JITModule,
+ inst: Instance<'tcx>,
+) {
let pointer_type = module.target_config().pointer_type();
let name = tcx.symbol_name(inst).name;
@@ -357,8 +378,9 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
)
.unwrap();
- cx.cached_context.clear();
- let trampoline = &mut cx.cached_context.func;
+ let context = cached_context;
+ context.clear();
+ let trampoline = &mut context.func;
trampoline.signature = sig.clone();
let mut builder_ctx = FunctionBuilderContext::new();
@@ -381,5 +403,6 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
trampoline_builder.ins().return_(&ret_vals);
- module.define_function(func_id, &mut cx.cached_context).unwrap();
+ module.define_function(func_id, context).unwrap();
+ cx.unwind_context.add_function(func_id, context, module.isa());
}
diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs
new file mode 100644
index 000000000..dcbcaba30
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs
@@ -0,0 +1,114 @@
+//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a
+//! standalone executable.
+
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::{Command, Stdio};
+use std::sync::Arc;
+
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
+use rustc_hir::ItemId;
+use rustc_session::config::{OutputFilenames, OutputType};
+
+use crate::prelude::*;
+
+pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) {
+ let item = tcx.hir().item(item_id);
+ if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind {
+ if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
+ global_asm.push_str("\n.intel_syntax noprefix\n");
+ } else {
+ global_asm.push_str("\n.att_syntax\n");
+ }
+ for piece in asm.template {
+ match *piece {
+ InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s),
+ InlineAsmTemplatePiece::Placeholder { .. } => todo!(),
+ }
+ }
+ global_asm.push_str("\n.att_syntax\n\n");
+ } else {
+ bug!("Expected GlobalAsm found {:?}", item);
+ }
+}
+
+#[derive(Debug)]
+pub(crate) struct GlobalAsmConfig {
+ asm_enabled: bool,
+ assembler: PathBuf,
+ pub(crate) output_filenames: Arc<OutputFilenames>,
+}
+
+impl GlobalAsmConfig {
+ pub(crate) fn new(tcx: TyCtxt<'_>) -> Self {
+ let asm_enabled = cfg!(feature = "inline_asm") && !tcx.sess.target.is_like_windows;
+
+ GlobalAsmConfig {
+ asm_enabled,
+ assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"),
+ output_filenames: tcx.output_filenames(()).clone(),
+ }
+ }
+}
+
+pub(crate) fn compile_global_asm(
+ config: &GlobalAsmConfig,
+ cgu_name: &str,
+ global_asm: &str,
+) -> Result<Option<PathBuf>, String> {
+ if global_asm.is_empty() {
+ return Ok(None);
+ }
+
+ if !config.asm_enabled {
+ if global_asm.contains("__rust_probestack") {
+ return Ok(None);
+ }
+
+ // FIXME fix linker error on macOS
+ if cfg!(not(feature = "inline_asm")) {
+ return Err(
+ "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift"
+ .to_owned(),
+ );
+ } else {
+ return Err("asm! and global_asm! are not yet supported on Windows".to_owned());
+ }
+ }
+
+ // Remove all LLVM style comments
+ let global_asm = global_asm
+ .lines()
+ .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
+ .collect::<Vec<_>>()
+ .join("\n");
+
+ let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name));
+
+ // Assemble `global_asm`
+ let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
+ let mut child = Command::new(&config.assembler)
+ .arg("-o")
+ .arg(&global_asm_object_file)
+ .stdin(Stdio::piped())
+ .spawn()
+ .expect("Failed to spawn `as`.");
+ child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
+ let status = child.wait().expect("Failed to wait for `as`.");
+ if !status.success() {
+ return Err(format!("Failed to assemble `{}`", global_asm));
+ }
+
+ Ok(Some(global_asm_object_file))
+}
+
+pub(crate) fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf {
+ let mut new_filename = path.file_stem().unwrap().to_owned();
+ new_filename.push(postfix);
+ if let Some(extension) = path.extension() {
+ new_filename.push(".");
+ new_filename.push(extension);
+ }
+ path.set_file_name(new_filename);
+ path
+}
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index 241de5e36..8b3d475cb 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -15,15 +15,19 @@ pub(crate) fn codegen_inline_asm<'tcx>(
template: &[InlineAsmTemplatePiece],
operands: &[InlineAsmOperand<'tcx>],
options: InlineAsmOptions,
+ destination: Option<mir::BasicBlock>,
) {
// FIXME add .eh_frame unwind info directives
if !template.is_empty() {
+ // Used by panic_abort
if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
- let true_ = fx.bcx.ins().iconst(types::I32, 1);
- fx.bcx.ins().trapnz(true_, TrapCode::User(1));
+ fx.bcx.ins().trap(TrapCode::User(1));
return;
- } else if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string())
+ }
+
+ // Used by stdarch
+ if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string())
&& matches!(
template[1],
InlineAsmTemplatePiece::Placeholder {
@@ -47,51 +51,46 @@ pub(crate) fn codegen_inline_asm<'tcx>(
{
assert_eq!(operands.len(), 4);
let (leaf, eax_place) = match operands[1] {
- InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
- assert_eq!(
- reg,
- InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax))
- );
- (
- crate::base::codegen_operand(fx, in_value).load_scalar(fx),
- crate::base::codegen_place(fx, out_place.unwrap()),
- )
- }
+ InlineAsmOperand::InOut {
+ reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
+ late: true,
+ ref in_value,
+ out_place: Some(out_place),
+ } => (
+ crate::base::codegen_operand(fx, in_value).load_scalar(fx),
+ crate::base::codegen_place(fx, out_place),
+ ),
_ => unreachable!(),
};
let ebx_place = match operands[0] {
- InlineAsmOperand::Out { reg, late: true, place } => {
- assert_eq!(
- reg,
+ InlineAsmOperand::Out {
+ reg:
InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86(
- X86InlineAsmRegClass::reg
- ))
- );
- crate::base::codegen_place(fx, place.unwrap())
- }
+ X86InlineAsmRegClass::reg,
+ )),
+ late: true,
+ place: Some(place),
+ } => crate::base::codegen_place(fx, place),
_ => unreachable!(),
};
let (sub_leaf, ecx_place) = match operands[2] {
- InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
- assert_eq!(
- reg,
- InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx))
- );
- (
- crate::base::codegen_operand(fx, in_value).load_scalar(fx),
- crate::base::codegen_place(fx, out_place.unwrap()),
- )
- }
+ InlineAsmOperand::InOut {
+ reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
+ late: true,
+ ref in_value,
+ out_place: Some(out_place),
+ } => (
+ crate::base::codegen_operand(fx, in_value).load_scalar(fx),
+ crate::base::codegen_place(fx, out_place),
+ ),
_ => unreachable!(),
};
let edx_place = match operands[3] {
- InlineAsmOperand::Out { reg, late: true, place } => {
- assert_eq!(
- reg,
- InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx))
- );
- crate::base::codegen_place(fx, place.unwrap())
- }
+ InlineAsmOperand::Out {
+ reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
+ late: true,
+ place: Some(place),
+ } => crate::base::codegen_place(fx, place),
_ => unreachable!(),
};
@@ -101,12 +100,99 @@ pub(crate) fn codegen_inline_asm<'tcx>(
ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
+ let destination_block = fx.get_block(destination.unwrap());
+ fx.bcx.ins().jump(destination_block, &[]);
return;
- } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
+ }
+
+ // Used by compiler-builtins
+ if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
// ___chkstk, ___chkstk_ms and __alloca are only used on Windows
crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
+ return;
} else if fx.tcx.symbol_name(fx.instance).name == "__alloca" {
crate::trap::trap_unimplemented(fx, "Alloca is not supported");
+ return;
+ }
+
+ // Used by measureme
+ if template[0] == InlineAsmTemplatePiece::String("xor %eax, %eax".to_string())
+ && template[1] == InlineAsmTemplatePiece::String("\n".to_string())
+ && template[2] == InlineAsmTemplatePiece::String("mov %rbx, ".to_string())
+ && matches!(
+ template[3],
+ InlineAsmTemplatePiece::Placeholder {
+ operand_idx: 0,
+ modifier: Some('r'),
+ span: _
+ }
+ )
+ && template[4] == InlineAsmTemplatePiece::String("\n".to_string())
+ && template[5] == InlineAsmTemplatePiece::String("cpuid".to_string())
+ && template[6] == InlineAsmTemplatePiece::String("\n".to_string())
+ && template[7] == InlineAsmTemplatePiece::String("mov ".to_string())
+ && matches!(
+ template[8],
+ InlineAsmTemplatePiece::Placeholder {
+ operand_idx: 0,
+ modifier: Some('r'),
+ span: _
+ }
+ )
+ && template[9] == InlineAsmTemplatePiece::String(", %rbx".to_string())
+ {
+ let destination_block = fx.get_block(destination.unwrap());
+ fx.bcx.ins().jump(destination_block, &[]);
+ return;
+ } else if template[0] == InlineAsmTemplatePiece::String("rdpmc".to_string()) {
+ // Return zero dummy values for all performance counters
+ match operands[0] {
+ InlineAsmOperand::In {
+ reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
+ value: _,
+ } => {}
+ _ => unreachable!(),
+ };
+ let lo = match operands[1] {
+ InlineAsmOperand::Out {
+ reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
+ late: true,
+ place: Some(place),
+ } => crate::base::codegen_place(fx, place),
+ _ => unreachable!(),
+ };
+ let hi = match operands[2] {
+ InlineAsmOperand::Out {
+ reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
+ late: true,
+ place: Some(place),
+ } => crate::base::codegen_place(fx, place),
+ _ => unreachable!(),
+ };
+
+ let u32_layout = fx.layout_of(fx.tcx.types.u32);
+ let zero = fx.bcx.ins().iconst(types::I32, 0);
+ lo.write_cvalue(fx, CValue::by_val(zero, u32_layout));
+ hi.write_cvalue(fx, CValue::by_val(zero, u32_layout));
+
+ let destination_block = fx.get_block(destination.unwrap());
+ fx.bcx.ins().jump(destination_block, &[]);
+ return;
+ } else if template[0] == InlineAsmTemplatePiece::String("lock xadd ".to_string())
+ && matches!(
+ template[1],
+ InlineAsmTemplatePiece::Placeholder { operand_idx: 1, modifier: None, span: _ }
+ )
+ && template[2] == InlineAsmTemplatePiece::String(", (".to_string())
+ && matches!(
+ template[3],
+ InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: None, span: _ }
+ )
+ && template[4] == InlineAsmTemplatePiece::String(")".to_string())
+ {
+ let destination_block = fx.get_block(destination.unwrap());
+ fx.bcx.ins().jump(destination_block, &[]);
+ return;
}
}
@@ -175,6 +261,16 @@ pub(crate) fn codegen_inline_asm<'tcx>(
}
call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
+
+ match destination {
+ Some(destination) => {
+ let destination_block = fx.get_block(destination);
+ fx.bcx.ins().jump(destination_block, &[]);
+ }
+ None => {
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+ }
+ }
}
struct InlineAssemblyGenerator<'a, 'tcx> {
@@ -637,7 +733,7 @@ fn call_inline_asm<'tcx>(
inputs: Vec<(Size, Value)>,
outputs: Vec<(Size, CPlace<'tcx>)>,
) {
- let stack_slot = fx.bcx.func.create_stack_slot(StackSlotData {
+ let stack_slot = fx.bcx.func.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
size: u32::try_from(slot_size.bytes()).unwrap(),
});
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
index d02dfd93c..5120b89c4 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
@@ -62,7 +62,7 @@ pub(crate) fn codegen_cpuid_call<'tcx>(
fx.bcx.ins().jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
fx.bcx.switch_to_block(unsupported_leaf);
- crate::trap::trap_unreachable(
+ crate::trap::trap_unimplemented(
fx,
"__cpuid_count arch intrinsic doesn't yet support specified leaf",
);
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
index 869670c8c..a799dca93 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
@@ -139,6 +139,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
.sess
.warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
crate::trap::trap_unimplemented(fx, intrinsic);
+ return;
}
}
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index b2a83e1d4..2e4ca594f 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -44,7 +44,7 @@ fn report_atomic_type_validation_error<'tcx>(
),
);
// Prevent verifier error
- crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
@@ -53,7 +53,7 @@ pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx
_ => unreachable!(),
};
- match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
+ match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
// Cranelift currently only implements icmp for 128bit vectors.
Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
_ => None,
@@ -203,7 +203,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
sym::transmute => {
crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
}
- _ => unimplemented!("unsupported instrinsic {}", intrinsic),
+ _ => unimplemented!("unsupported intrinsic {}", intrinsic),
}
return;
};
@@ -301,7 +301,44 @@ fn codegen_float_intrinsic_call<'tcx>(
_ => unreachable!(),
};
- let res = fx.easy_call(name, &args, ty);
+ let layout = fx.layout_of(ty);
+ let res = match intrinsic {
+ sym::fmaf32 | sym::fmaf64 => {
+ let a = args[0].load_scalar(fx);
+ let b = args[1].load_scalar(fx);
+ let c = args[2].load_scalar(fx);
+ CValue::by_val(fx.bcx.ins().fma(a, b, c), layout)
+ }
+ sym::copysignf32 | sym::copysignf64 => {
+ let a = args[0].load_scalar(fx);
+ let b = args[1].load_scalar(fx);
+ CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout)
+ }
+ sym::fabsf32
+ | sym::fabsf64
+ | sym::floorf32
+ | sym::floorf64
+ | sym::ceilf32
+ | sym::ceilf64
+ | sym::truncf32
+ | sym::truncf64 => {
+ let a = args[0].load_scalar(fx);
+
+ let val = match intrinsic {
+ sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a),
+ sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a),
+ sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a),
+ sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a),
+ _ => unreachable!(),
+ };
+
+ CValue::by_val(val, layout)
+ }
+ // These intrinsics aren't supported natively by Cranelift.
+ // Lower them to a libcall.
+ _ => fx.easy_call(name, &args, ty),
+ };
+
ret.write_cvalue(fx, res);
true
@@ -320,9 +357,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
let usize_layout = fx.layout_of(fx.tcx.types.usize);
match intrinsic {
- sym::assume => {
- intrinsic_args!(fx, args => (_a); intrinsic);
- }
sym::likely | sym::unlikely => {
intrinsic_args!(fx, args => (a); intrinsic);
@@ -540,6 +574,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
}
+ sym::ptr_mask => {
+ intrinsic_args!(fx, args => (ptr, mask); intrinsic);
+ let ptr = ptr.load_scalar(fx);
+ let mask = mask.load_scalar(fx);
+ fx.bcx.ins().band(ptr, mask);
+ }
+
sym::transmute => {
intrinsic_args!(fx, args => (from); intrinsic);
@@ -775,20 +816,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
ret.write_cvalue(fx, val);
}
- sym::ptr_guaranteed_eq => {
+ sym::ptr_guaranteed_cmp => {
intrinsic_args!(fx, args => (a, b); intrinsic);
let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b);
ret.write_cvalue(fx, val);
}
- sym::ptr_guaranteed_ne => {
- intrinsic_args!(fx, args => (a, b); intrinsic);
-
- let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b);
- ret.write_cvalue(fx, val);
- }
-
sym::caller_location => {
intrinsic_args!(fx, args => (); intrinsic);
@@ -818,8 +852,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
// special case for compiler-builtins to avoid having to patch it
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
- let ret_block = fx.get_block(destination.unwrap());
- fx.bcx.ins().jump(ret_block, &[]);
return;
} else {
fx.tcx
@@ -851,8 +883,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
// special case for compiler-builtins to avoid having to patch it
crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
- let ret_block = fx.get_block(destination.unwrap());
- fx.bcx.ins().jump(ret_block, &[]);
return;
} else {
fx.tcx
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index 30e3d1125..1f358b1bb 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -14,7 +14,7 @@ fn report_simd_type_validation_error(
) {
fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
// Prevent verifier error
- crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
}
pub(super) fn codegen_simd_intrinsic_call<'tcx>(
@@ -157,7 +157,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
),
);
// Prevent verifier error
- crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+ fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
return;
}
}
@@ -186,7 +186,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let size = Size::from_bytes(
4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */
);
- alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
+ alloc
+ .inner()
+ .get_bytes_strip_provenance(fx, alloc_range(offset, size))
+ .unwrap()
}
_ => unreachable!("{:?}", idx_const),
};
@@ -274,12 +277,17 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
idx_const
} else {
fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
- let res = crate::trap::trap_unimplemented_ret_value(
+ let trap_block = fx.bcx.create_block();
+ let dummy_block = fx.bcx.create_block();
+ let true_ = fx.bcx.ins().iconst(types::I8, 1);
+ fx.bcx.ins().brnz(true_, trap_block, &[]);
+ fx.bcx.ins().jump(dummy_block, &[]);
+ fx.bcx.switch_to_block(trap_block);
+ crate::trap::trap_unimplemented(
fx,
- ret.layout(),
"Index argument for `simd_extract` is not a constant",
);
- ret.write_cvalue(fx, res);
+ fx.bcx.switch_to_block(dummy_block);
return;
};
@@ -392,21 +400,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
let layout = a.layout();
let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+ let res_lane_layout = fx.layout_of(lane_ty);
for lane in 0..lane_count {
- let a_lane = a.value_lane(fx, lane);
- let b_lane = b.value_lane(fx, lane);
- let c_lane = c.value_lane(fx, lane);
+ let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+ let b_lane = b.value_lane(fx, lane).load_scalar(fx);
+ let c_lane = c.value_lane(fx, lane).load_scalar(fx);
- let res_lane = match lane_ty.kind() {
- ty::Float(FloatTy::F32) => {
- fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty)
- }
- ty::Float(FloatTy::F64) => {
- fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty)
- }
- _ => unreachable!(),
- };
+ let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane);
+ let res_lane = CValue::by_val(res_lane, res_lane_layout);
ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
}
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index bb0793b1d..913414e76 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -4,6 +4,7 @@
#![warn(unused_lifetimes)]
#![warn(unreachable_pub)]
+extern crate jobserver;
#[macro_use]
extern crate rustc_middle;
extern crate rustc_ast;
@@ -25,10 +26,12 @@ extern crate rustc_target;
extern crate rustc_driver;
use std::any::Any;
-use std::cell::Cell;
+use std::cell::{Cell, RefCell};
+use std::sync::Arc;
use rustc_codegen_ssa::traits::CodegenBackend;
use rustc_codegen_ssa::CodegenResults;
+use rustc_data_structures::profiling::SelfProfilerRef;
use rustc_errors::ErrorGuaranteed;
use rustc_metadata::EncodedMetadata;
use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
@@ -51,11 +54,13 @@ mod cast;
mod codegen_i128;
mod common;
mod compiler_builtins;
+mod concurrency_limiter;
mod config;
mod constant;
mod debuginfo;
mod discriminant;
mod driver;
+mod global_asm;
mod inline_asm;
mod intrinsics;
mod linkage;
@@ -119,19 +124,20 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
/// The codegen context holds any information shared between the codegen of individual functions
/// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
-struct CodegenCx<'tcx> {
- tcx: TyCtxt<'tcx>,
+struct CodegenCx {
+ profiler: SelfProfilerRef,
+ output_filenames: Arc<OutputFilenames>,
+ should_write_ir: bool,
global_asm: String,
inline_asm_index: Cell<usize>,
- cached_context: Context,
- debug_context: Option<DebugContext<'tcx>>,
+ debug_context: Option<DebugContext>,
unwind_context: UnwindContext,
cgu_name: Symbol,
}
-impl<'tcx> CodegenCx<'tcx> {
+impl CodegenCx {
fn new(
- tcx: TyCtxt<'tcx>,
+ tcx: TyCtxt<'_>,
backend_config: BackendConfig,
isa: &dyn TargetIsa,
debug_info: bool,
@@ -147,10 +153,11 @@ impl<'tcx> CodegenCx<'tcx> {
None
};
CodegenCx {
- tcx,
+ profiler: tcx.prof.clone(),
+ output_filenames: tcx.output_filenames(()).clone(),
+ should_write_ir: crate::pretty_clif::should_write_ir(tcx),
global_asm: String::new(),
inline_asm_index: Cell::new(0),
- cached_context: Context::new(),
debug_context,
unwind_context,
cgu_name,
@@ -159,7 +166,7 @@ impl<'tcx> CodegenCx<'tcx> {
}
pub struct CraneliftCodegenBackend {
- pub config: Option<BackendConfig>,
+ pub config: RefCell<Option<BackendConfig>>,
}
impl CodegenBackend for CraneliftCodegenBackend {
@@ -169,6 +176,13 @@ impl CodegenBackend for CraneliftCodegenBackend {
Lto::No | Lto::ThinLocal => {}
Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
}
+
+ let mut config = self.config.borrow_mut();
+ if config.is_none() {
+ let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args)
+ .unwrap_or_else(|err| sess.fatal(&err));
+ *config = Some(new_config);
+ }
}
fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec<rustc_span::Symbol> {
@@ -186,15 +200,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
need_metadata_module: bool,
) -> Box<dyn Any> {
tcx.sess.abort_if_errors();
- let config = if let Some(config) = self.config.clone() {
- config
- } else {
- if !tcx.sess.unstable_options() && !tcx.sess.opts.cg.llvm_args.is_empty() {
- tcx.sess.fatal("`-Z unstable-options` must be passed to allow configuring cg_clif");
- }
- BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
- .unwrap_or_else(|err| tcx.sess.fatal(&err))
- };
+ let config = self.config.borrow().clone().unwrap();
match config.codegen_mode {
CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
CodegenMode::Jit | CodegenMode::JitLazy => {
@@ -210,12 +216,13 @@ impl CodegenBackend for CraneliftCodegenBackend {
fn join_codegen(
&self,
ongoing_codegen: Box<dyn Any>,
- _sess: &Session,
+ sess: &Session,
_outputs: &OutputFilenames,
) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> {
- Ok(*ongoing_codegen
- .downcast::<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)>()
- .unwrap())
+ Ok(ongoing_codegen
+ .downcast::<driver::aot::OngoingCodegen>()
+ .unwrap()
+ .join(sess, self.config.borrow().as_ref().unwrap()))
}
fn link(
@@ -312,5 +319,5 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
/// This is the entrypoint for a hot plugged rustc_codegen_cranelift
#[no_mangle]
pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
- Box::new(CraneliftCodegenBackend { config: None })
+ Box::new(CraneliftCodegenBackend { config: RefCell::new(None) })
}
diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs
index c67b6e98b..3c024a84d 100644
--- a/compiler/rustc_codegen_cranelift/src/main_shim.rs
+++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs
@@ -1,7 +1,7 @@
use rustc_hir::LangItem;
use rustc_middle::ty::subst::GenericArg;
use rustc_middle::ty::AssocKind;
-use rustc_session::config::EntryFnType;
+use rustc_session::config::{sigpipe, EntryFnType};
use rustc_span::symbol::Ident;
use crate::prelude::*;
@@ -15,12 +15,12 @@ pub(crate) fn maybe_create_entry_wrapper(
is_jit: bool,
is_primary_cgu: bool,
) {
- let (main_def_id, is_main_fn) = match tcx.entry_fn(()) {
+ let (main_def_id, (is_main_fn, sigpipe)) = match tcx.entry_fn(()) {
Some((def_id, entry_ty)) => (
def_id,
match entry_ty {
- EntryFnType::Main => true,
- EntryFnType::Start => false,
+ EntryFnType::Main { sigpipe } => (true, sigpipe),
+ EntryFnType::Start => (false, sigpipe::DEFAULT),
},
),
None => return,
@@ -35,7 +35,7 @@ pub(crate) fn maybe_create_entry_wrapper(
return;
}
- create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn);
+ create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn, sigpipe);
fn create_entry_fn(
tcx: TyCtxt<'_>,
@@ -44,6 +44,7 @@ pub(crate) fn maybe_create_entry_wrapper(
rust_main_def_id: DefId,
ignore_lang_start_wrapper: bool,
is_main_fn: bool,
+ sigpipe: u8,
) {
let main_ret_ty = tcx.fn_sig(rust_main_def_id).output();
// Given that `main()` has no arguments,
@@ -83,6 +84,7 @@ pub(crate) fn maybe_create_entry_wrapper(
bcx.switch_to_block(block);
let arg_argc = bcx.append_block_param(block, m.target_config().pointer_type());
let arg_argv = bcx.append_block_param(block, m.target_config().pointer_type());
+ let arg_sigpipe = bcx.ins().iconst(types::I8, sigpipe as i64);
let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
@@ -143,7 +145,8 @@ pub(crate) fn maybe_create_entry_wrapper(
let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref);
let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func);
- let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]);
+ let call_inst =
+ bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv, arg_sigpipe]);
bcx.inst_results(call_inst)[0]
} else {
// using user-defined start fn
diff --git a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
index d1f89adb3..0df7e8229 100644
--- a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
@@ -1,20 +1,3 @@
//! Various optimizations specific to cg_clif
-use cranelift_codegen::isa::TargetIsa;
-
-use crate::prelude::*;
-
pub(crate) mod peephole;
-
-pub(crate) fn optimize_function<'tcx>(
- tcx: TyCtxt<'tcx>,
- isa: &dyn TargetIsa,
- instance: Instance<'tcx>,
- ctx: &mut Context,
- clif_comments: &mut crate::pretty_clif::CommentWriter,
-) {
- // FIXME classify optimizations over opt levels once we have more
-
- crate::pretty_clif::write_clif_file(tcx, "preopt", isa, instance, &ctx.func, &*clif_comments);
- crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
-}
diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
index 1d1ec2168..a7af16268 100644
--- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
@@ -62,7 +62,7 @@ use cranelift_codegen::{
};
use rustc_middle::ty::layout::FnAbiOf;
-use rustc_session::config::OutputType;
+use rustc_session::config::{OutputFilenames, OutputType};
use crate::prelude::*;
@@ -205,15 +205,11 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
}
pub(crate) fn write_ir_file(
- tcx: TyCtxt<'_>,
- name: impl FnOnce() -> String,
+ output_filenames: &OutputFilenames,
+ name: &str,
write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
) {
- if !should_write_ir(tcx) {
- return;
- }
-
- let clif_output_dir = tcx.output_filenames(()).with_extension("clif");
+ let clif_output_dir = output_filenames.with_extension("clif");
match std::fs::create_dir(&clif_output_dir) {
Ok(()) => {}
@@ -221,44 +217,43 @@ pub(crate) fn write_ir_file(
res @ Err(_) => res.unwrap(),
}
- let clif_file_name = clif_output_dir.join(name());
+ let clif_file_name = clif_output_dir.join(name);
let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
if let Err(err) = res {
- tcx.sess.warn(&format!("error writing ir file: {}", err));
+ // Using early_warn as no Session is available here
+ rustc_session::early_warn(
+ rustc_session::config::ErrorOutputType::default(),
+ &format!("error writing ir file: {}", err),
+ );
}
}
-pub(crate) fn write_clif_file<'tcx>(
- tcx: TyCtxt<'tcx>,
+pub(crate) fn write_clif_file(
+ output_filenames: &OutputFilenames,
+ symbol_name: &str,
postfix: &str,
isa: &dyn cranelift_codegen::isa::TargetIsa,
- instance: Instance<'tcx>,
func: &cranelift_codegen::ir::Function,
mut clif_comments: &CommentWriter,
) {
// FIXME work around filename too long errors
- write_ir_file(
- tcx,
- || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
- |file| {
- let mut clif = String::new();
- cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func)
- .unwrap();
+ write_ir_file(output_filenames, &format!("{}.{}.clif", symbol_name, postfix), |file| {
+ let mut clif = String::new();
+ cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func).unwrap();
- for flag in isa.flags().iter() {
- writeln!(file, "set {}", flag)?;
- }
- write!(file, "target {}", isa.triple().architecture.to_string())?;
- for isa_flag in isa.isa_flags().iter() {
- write!(file, " {}", isa_flag)?;
- }
- writeln!(file, "\n")?;
- writeln!(file)?;
- file.write_all(clif.as_bytes())?;
- Ok(())
- },
- );
+ for flag in isa.flags().iter() {
+ writeln!(file, "set {}", flag)?;
+ }
+ write!(file, "target {}", isa.triple().architecture.to_string())?;
+ for isa_flag in isa.isa_flags().iter() {
+ write!(file, " {}", isa_flag)?;
+ }
+ writeln!(file, "\n")?;
+ writeln!(file)?;
+ file.write_all(clif.as_bytes())?;
+ Ok(())
+ });
}
impl fmt::Debug for FunctionCx<'_, '_, '_> {
diff --git a/compiler/rustc_codegen_cranelift/src/toolchain.rs b/compiler/rustc_codegen_cranelift/src/toolchain.rs
index f86236ef3..b6b465e1f 100644
--- a/compiler/rustc_codegen_cranelift/src/toolchain.rs
+++ b/compiler/rustc_codegen_cranelift/src/toolchain.rs
@@ -8,10 +8,8 @@ use rustc_session::Session;
/// Tries to infer the path of a binary for the target toolchain from the linker name.
pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf {
let (mut linker, _linker_flavor) = linker_and_flavor(sess);
- let linker_file_name = linker
- .file_name()
- .and_then(|name| name.to_str())
- .unwrap_or_else(|| sess.fatal("couldn't extract file name from specified linker"));
+ let linker_file_name =
+ linker.file_name().unwrap().to_str().expect("linker filename should be valid UTF-8");
if linker_file_name == "ld.lld" {
if tool != "ld" {
diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs
index 923269c4d..82a2ec579 100644
--- a/compiler/rustc_codegen_cranelift/src/trap.rs
+++ b/compiler/rustc_codegen_cranelift/src/trap.rs
@@ -25,33 +25,10 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
fx.bcx.ins().call(puts, &[msg_ptr]);
}
-/// Use this for example when a function call should never return. This will fill the current block,
-/// so you can **not** add instructions to it afterwards.
-///
-/// Trap code: user65535
-pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
- codegen_print(fx, msg.as_ref());
- fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
-}
/// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen.
-/// Unlike `trap_unreachable` this will not fill the current block, so you **must** add instructions
-/// to it afterwards.
///
/// Trap code: user65535
pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
codegen_print(fx, msg.as_ref());
- let true_ = fx.bcx.ins().iconst(types::I32, 1);
- fx.bcx.ins().trapnz(true_, TrapCode::User(!0));
-}
-
-/// Like `trap_unimplemented` but returns a fake value of the specified type.
-///
-/// Trap code: user65535
-pub(crate) fn trap_unimplemented_ret_value<'tcx>(
- fx: &mut FunctionCx<'_, '_, 'tcx>,
- dest_layout: TyAndLayout<'tcx>,
- msg: impl AsRef<str>,
-) -> CValue<'tcx> {
- trap_unimplemented(fx, msg);
- CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout)
+ fx.bcx.ins().trap(TrapCode::User(!0));
}
diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs
index 052ca0a08..dd9d891dd 100644
--- a/compiler/rustc_codegen_cranelift/src/unsize.rs
+++ b/compiler/rustc_codegen_cranelift/src/unsize.rs
@@ -29,6 +29,7 @@ pub(crate) fn unsized_info<'tcx>(
let old_info =
old_info.expect("unsized_info: missing old info for trait upcasting coercion");
if data_a.principal_def_id() == data_b.principal_def_id() {
+ // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables.
return old_info;
}
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index 45ae2bd8f..cfaadca94 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -122,7 +122,7 @@ impl<'tcx> CValue<'tcx> {
let clif_ty = match layout.abi {
Abi::Scalar(scalar) => scalar_to_clif_type(fx.tcx, scalar),
Abi::Vector { element, count } => scalar_to_clif_type(fx.tcx, element)
- .by(u16::try_from(count).unwrap())
+ .by(u32::try_from(count).unwrap())
.unwrap(),
_ => unreachable!("{:?}", layout.ty),
};
@@ -330,7 +330,7 @@ impl<'tcx> CPlace<'tcx> {
.fatal(&format!("values of type {} are too big to store on the stack", layout.ty));
}
- let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
+ let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
@@ -472,7 +472,7 @@ impl<'tcx> CPlace<'tcx> {
}
_ if src_ty.is_vector() || dst_ty.is_vector() => {
// FIXME do something more efficient for transmutes between vectors and integers.
- let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
+ let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
kind: StackSlotKind::ExplicitSlot,
// FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
// specify stack slot alignment.
@@ -519,7 +519,7 @@ impl<'tcx> CPlace<'tcx> {
if let ty::Array(element, len) = dst_layout.ty.kind() {
// Can only happen for vector types
let len =
- u16::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
+ u32::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap();
let data = match from.0 {
@@ -614,7 +614,7 @@ impl<'tcx> CPlace<'tcx> {
dst_align,
src_align,
true,
- MemFlags::trusted(),
+ flags,
);
}
CValueInner::ByRef(_, Some(_)) => todo!(),
@@ -815,7 +815,8 @@ pub(crate) fn assert_assignable<'tcx>(
);
// fn(&T) -> for<'l> fn(&'l T) is allowed
}
- (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => {
+ (&ty::Dynamic(from_traits, _, _from_kind), &ty::Dynamic(to_traits, _, _to_kind)) => {
+ // FIXME(dyn-star): Do the right thing with DynKinds
for (from, to) in from_traits.iter().zip(to_traits) {
let from =
fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);