Adding debian version 1.65.0+dfsg1-2.debian/1.65.0+dfsg1-2

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

30 files changed, 1323 insertions, 1032 deletions

diff --git a/compiler/rustc_codegen_cranelift/src/abi/comments.rs b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
index 37d2679c1..7f4619b5c 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/comments.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/comments.rs
@@ -24,7 +24,7 @@ pub(super) fn add_arg_comment<'tcx>(
     local: Option<mir::Local>,
     local_field: Option<usize>,
     params: &[Value],
-    arg_abi_mode: PassMode,
+    arg_abi_mode: &PassMode,
     arg_layout: TyAndLayout<'tcx>,
 ) {
     if !fx.clif_comments.enabled() {
diff --git a/compiler/rustc_codegen_cranelift/src/abi/mod.rs b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
index 815450f68..0497c2570 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/mod.rs
@@ -342,7 +342,7 @@ pub(crate) fn codegen_terminator_call<'tcx>(
 
     let ret_place = codegen_place(fx, destination);
 
-    // Handle special calls like instrinsics and empty drop glue.
+    // Handle special calls like intrinsics and empty drop glue.
     let instance = if let ty::FnDef(def_id, substs) = *fn_ty.kind() {
         let instance = ty::Instance::resolve(fx.tcx, ty::ParamEnv::reveal_all(), def_id, substs)
             .unwrap()
diff --git a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
index 6c10baa53..96e25d3a8 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/pass_mode.rs
@@ -23,7 +23,7 @@ fn reg_to_abi_param(reg: Reg) -> AbiParam {
         (RegKind::Integer, 9..=16) => types::I128,
         (RegKind::Float, 4) => types::F32,
         (RegKind::Float, 8) => types::F64,
-        (RegKind::Vector, size) => types::I8.by(u16::try_from(size).unwrap()).unwrap(),
+        (RegKind::Vector, size) => types::I8.by(u32::try_from(size).unwrap()).unwrap(),
         _ => unreachable!("{:?}", reg),
     };
     AbiParam::new(clif_ty)
@@ -38,7 +38,7 @@ fn apply_arg_attrs_to_abi_param(mut param: AbiParam, arg_attrs: ArgAttributes) -
     param
 }
 
-fn cast_target_to_abi_params(cast: CastTarget) -> SmallVec<[AbiParam; 2]> {
+fn cast_target_to_abi_params(cast: &CastTarget) -> SmallVec<[AbiParam; 2]> {
     let (rest_count, rem_bytes) = if cast.rest.unit.size.bytes() == 0 {
         (0, 0)
     } else {
@@ -100,7 +100,10 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 }
                 _ => unreachable!("{:?}", self.layout.abi),
             },
-            PassMode::Cast(cast) => cast_target_to_abi_params(cast),
+            PassMode::Cast(ref cast, pad_i32) => {
+                assert!(!pad_i32, "padding support not yet implemented");
+                cast_target_to_abi_params(cast)
+            }
             PassMode::Indirect { attrs, extra_attrs: None, on_stack } => {
                 if on_stack {
                     // Abi requires aligning struct size to pointer size
@@ -145,7 +148,9 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
                 }
                 _ => unreachable!("{:?}", self.layout.abi),
             },
-            PassMode::Cast(cast) => (None, cast_target_to_abi_params(cast).into_iter().collect()),
+            PassMode::Cast(ref cast, _) => {
+                (None, cast_target_to_abi_params(cast).into_iter().collect())
+            }
             PassMode::Indirect { attrs: _, extra_attrs: None, on_stack } => {
                 assert!(!on_stack);
                 (Some(AbiParam::special(pointer_ty(tcx), ArgumentPurpose::StructReturn)), vec![])
@@ -160,7 +165,7 @@ impl<'tcx> ArgAbiExt<'tcx> for ArgAbi<'tcx, Ty<'tcx>> {
 pub(super) fn to_casted_value<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     arg: CValue<'tcx>,
-    cast: CastTarget,
+    cast: &CastTarget,
 ) -> SmallVec<[Value; 2]> {
     let (ptr, meta) = arg.force_stack(fx);
     assert!(meta.is_none());
@@ -179,12 +184,12 @@ pub(super) fn from_casted_value<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     block_params: &[Value],
     layout: TyAndLayout<'tcx>,
-    cast: CastTarget,
+    cast: &CastTarget,
 ) -> CValue<'tcx> {
     let abi_params = cast_target_to_abi_params(cast);
     let abi_param_size: u32 = abi_params.iter().map(|param| param.value_type.bytes()).sum();
     let layout_size = u32::try_from(layout.size.bytes()).unwrap();
-    let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
+    let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
         kind: StackSlotKind::ExplicitSlot,
         // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
         // specify stack slot alignment.
@@ -193,7 +198,7 @@ pub(super) fn from_casted_value<'tcx>(
         // larger alignment than the integer.
         size: (std::cmp::max(abi_param_size, layout_size) + 15) / 16 * 16,
     });
-    let ptr = Pointer::new(fx.bcx.ins().stack_addr(pointer_ty(fx.tcx), stack_slot, 0));
+    let ptr = Pointer::stack_slot(stack_slot);
     let mut offset = 0;
     let mut block_params_iter = block_params.iter().copied();
     for param in abi_params {
@@ -224,7 +229,7 @@ pub(super) fn adjust_arg_for_abi<'tcx>(
             let (a, b) = arg.load_scalar_pair(fx);
             smallvec![a, b]
         }
-        PassMode::Cast(cast) => to_casted_value(fx, arg, cast),
+        PassMode::Cast(ref cast, _) => to_casted_value(fx, arg, cast),
         PassMode::Indirect { .. } => {
             if is_owned {
                 match arg.force_stack(fx) {
@@ -268,7 +273,7 @@ pub(super) fn cvalue_for_param<'tcx>(
         local,
         local_field,
         &block_params,
-        arg_abi.mode,
+        &arg_abi.mode,
         arg_abi.layout,
     );
 
@@ -282,7 +287,9 @@ pub(super) fn cvalue_for_param<'tcx>(
             assert_eq!(block_params.len(), 2, "{:?}", block_params);
             Some(CValue::by_val_pair(block_params[0], block_params[1], arg_abi.layout))
         }
-        PassMode::Cast(cast) => Some(from_casted_value(fx, &block_params, arg_abi.layout, cast)),
+        PassMode::Cast(ref cast, _) => {
+            Some(from_casted_value(fx, &block_params, arg_abi.layout, cast))
+        }
         PassMode::Indirect { attrs: _, extra_attrs: None, on_stack: _ } => {
             assert_eq!(block_params.len(), 1, "{:?}", block_params);
             Some(CValue::by_ref(Pointer::new(block_params[0]), arg_abi.layout))
diff --git a/compiler/rustc_codegen_cranelift/src/abi/returning.rs b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
index ff3bb2dfd..aaa141876 100644
--- a/compiler/rustc_codegen_cranelift/src/abi/returning.rs
+++ b/compiler/rustc_codegen_cranelift/src/abi/returning.rs
@@ -13,7 +13,7 @@ pub(super) fn codegen_return_param<'tcx>(
     block_params_iter: &mut impl Iterator<Item = Value>,
 ) -> CPlace<'tcx> {
     let (ret_place, ret_param): (_, SmallVec<[_; 2]>) = match fx.fn_abi.as_ref().unwrap().ret.mode {
-        PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => {
+        PassMode::Ignore | PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => {
             let is_ssa = ssa_analyzed[RETURN_PLACE] == crate::analyze::SsaKind::Ssa;
             (
                 super::make_local_place(
@@ -44,7 +44,7 @@ pub(super) fn codegen_return_param<'tcx>(
         Some(RETURN_PLACE),
         None,
         &ret_param,
-        fx.fn_abi.as_ref().unwrap().ret.mode,
+        &fx.fn_abi.as_ref().unwrap().ret.mode,
         fx.fn_abi.as_ref().unwrap().ret.layout,
     );
 
@@ -75,7 +75,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
         PassMode::Indirect { attrs: _, extra_attrs: Some(_), on_stack: _ } => {
             unreachable!("unsized return value")
         }
-        PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(_) => (None, None),
+        PassMode::Direct(_) | PassMode::Pair(_, _) | PassMode::Cast(..) => (None, None),
     };
 
     let call_inst = f(fx, return_ptr);
@@ -92,7 +92,7 @@ pub(super) fn codegen_with_call_return_arg<'tcx>(
             ret_place
                 .write_cvalue(fx, CValue::by_val_pair(ret_val_a, ret_val_b, ret_arg_abi.layout));
         }
-        PassMode::Cast(cast) => {
+        PassMode::Cast(ref cast, _) => {
             let results =
                 fx.bcx.inst_results(call_inst).iter().copied().collect::<SmallVec<[Value; 2]>>();
             let result =
@@ -131,7 +131,7 @@ pub(crate) fn codegen_return(fx: &mut FunctionCx<'_, '_, '_>) {
             let (ret_val_a, ret_val_b) = place.to_cvalue(fx).load_scalar_pair(fx);
             fx.bcx.ins().return_(&[ret_val_a, ret_val_b]);
         }
-        PassMode::Cast(cast) => {
+        PassMode::Cast(ref cast, _) => {
             let place = fx.get_local_place(RETURN_PLACE);
             let ret_val = place.to_cvalue(fx);
             let ret_vals = super::pass_mode::to_casted_value(fx, ret_val, cast);
diff --git a/compiler/rustc_codegen_cranelift/src/analyze.rs b/compiler/rustc_codegen_cranelift/src/analyze.rs
index 35b89358b..0cbb9f3ec 100644
--- a/compiler/rustc_codegen_cranelift/src/analyze.rs
+++ b/compiler/rustc_codegen_cranelift/src/analyze.rs
@@ -26,7 +26,7 @@ pub(crate) fn analyze(fx: &FunctionCx<'_, '_, '_>) -> IndexVec<Local, SsaKind> {
         })
         .collect::<IndexVec<Local, SsaKind>>();
 
-    for bb in fx.mir.basic_blocks().iter() {
+    for bb in fx.mir.basic_blocks.iter() {
         for stmt in bb.statements.iter() {
             match &stmt.kind {
                 Assign(place_and_rval) => match &place_and_rval.1 {
diff --git a/compiler/rustc_codegen_cranelift/src/archive.rs b/compiler/rustc_codegen_cranelift/src/archive.rs
index b4c790961..fb5313f17 100644
--- a/compiler/rustc_codegen_cranelift/src/archive.rs
+++ b/compiler/rustc_codegen_cranelift/src/archive.rs
@@ -38,6 +38,7 @@ impl ArchiveBuilderBuilder for ArArchiveBuilderBuilder {
         _lib_name: &str,
         _dll_imports: &[rustc_session::cstore::DllImport],
         _tmpdir: &Path,
+        _is_direct_dependency: bool,
     ) -> PathBuf {
         bug!("creating dll imports is not supported");
     }
diff --git a/compiler/rustc_codegen_cranelift/src/base.rs b/compiler/rustc_codegen_cranelift/src/base.rs
index 122e103ff..399474d79 100644
--- a/compiler/rustc_codegen_cranelift/src/base.rs
+++ b/compiler/rustc_codegen_cranelift/src/base.rs
@@ -6,21 +6,43 @@ use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::FnAbiOf;
 use rustc_middle::ty::print::with_no_trimmed_paths;
 
-use indexmap::IndexSet;
-
 use crate::constant::ConstantCx;
+use crate::debuginfo::FunctionDebugContext;
 use crate::prelude::*;
 use crate::pretty_clif::CommentWriter;
 
-pub(crate) fn codegen_fn<'tcx>(
-    cx: &mut crate::CodegenCx<'tcx>,
+pub(crate) struct CodegenedFunction {
+    symbol_name: String,
+    func_id: FuncId,
+    func: Function,
+    clif_comments: CommentWriter,
+    func_debug_cx: Option<FunctionDebugContext>,
+}
+
+#[cfg_attr(not(feature = "jit"), allow(dead_code))]
+pub(crate) fn codegen_and_compile_fn<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    cx: &mut crate::CodegenCx,
+    cached_context: &mut Context,
     module: &mut dyn Module,
     instance: Instance<'tcx>,
 ) {
-    let tcx = cx.tcx;
-
     let _inst_guard =
         crate::PrintOnPanic(|| format!("{:?} {}", instance, tcx.symbol_name(instance).name));
+
+    let cached_func = std::mem::replace(&mut cached_context.func, Function::new());
+    let codegened_func = codegen_fn(tcx, cx, cached_func, module, instance);
+
+    compile_fn(cx, cached_context, module, codegened_func);
+}
+
+pub(crate) fn codegen_fn<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    cx: &mut crate::CodegenCx,
+    cached_func: Function,
+    module: &mut dyn Module,
+    instance: Instance<'tcx>,
+) -> CodegenedFunction {
     debug_assert!(!instance.substs.needs_infer());
 
     let mir = tcx.instance_mir(instance.def);
@@ -34,15 +56,14 @@ pub(crate) fn codegen_fn<'tcx>(
     });
 
     // Declare function
-    let symbol_name = tcx.symbol_name(instance);
+    let symbol_name = tcx.symbol_name(instance).name.to_string();
     let sig = get_function_sig(tcx, module.isa().triple(), instance);
-    let func_id = module.declare_function(symbol_name.name, Linkage::Local, &sig).unwrap();
-
-    cx.cached_context.clear();
+    let func_id = module.declare_function(&symbol_name, Linkage::Local, &sig).unwrap();
 
     // Make the FunctionBuilder
     let mut func_ctx = FunctionBuilderContext::new();
-    let mut func = std::mem::replace(&mut cx.cached_context.func, Function::new());
+    let mut func = cached_func;
+    func.clear();
     func.name = ExternalName::user(0, func_id.as_u32());
     func.signature = sig;
     func.collect_debug_info();
@@ -52,13 +73,19 @@ pub(crate) fn codegen_fn<'tcx>(
     // Predefine blocks
     let start_block = bcx.create_block();
     let block_map: IndexVec<BasicBlock, Block> =
-        (0..mir.basic_blocks().len()).map(|_| bcx.create_block()).collect();
+        (0..mir.basic_blocks.len()).map(|_| bcx.create_block()).collect();
 
     // Make FunctionCx
     let target_config = module.target_config();
     let pointer_type = target_config.pointer_type();
     let clif_comments = crate::pretty_clif::CommentWriter::new(tcx, instance);
 
+    let func_debug_cx = if let Some(debug_context) = &mut cx.debug_context {
+        Some(debug_context.define_function(tcx, &symbol_name, mir.span))
+    } else {
+        None
+    };
+
     let mut fx = FunctionCx {
         cx,
         module,
@@ -66,6 +93,7 @@ pub(crate) fn codegen_fn<'tcx>(
         target_config,
         pointer_type,
         constants_cx: ConstantCx::new(),
+        func_debug_cx,
 
         instance,
         symbol_name,
@@ -78,81 +106,48 @@ pub(crate) fn codegen_fn<'tcx>(
         caller_location: None, // set by `codegen_fn_prelude`
 
         clif_comments,
-        source_info_set: indexmap::IndexSet::new(),
+        last_source_file: None,
         next_ssa_var: 0,
     };
 
-    let arg_uninhabited = fx
-        .mir
-        .args_iter()
-        .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
-
-    if !crate::constant::check_constants(&mut fx) {
-        fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
-        fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
-        crate::trap::trap_unreachable(&mut fx, "compilation should have been aborted");
-    } else if arg_uninhabited {
-        fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
-        fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
-        fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
-    } else {
-        tcx.sess.time("codegen clif ir", || {
-            tcx.sess
-                .time("codegen prelude", || crate::abi::codegen_fn_prelude(&mut fx, start_block));
-            codegen_fn_content(&mut fx);
-        });
-    }
+    tcx.sess.time("codegen clif ir", || codegen_fn_body(&mut fx, start_block));
 
     // Recover all necessary data from fx, before accessing func will prevent future access to it.
-    let instance = fx.instance;
+    let symbol_name = fx.symbol_name;
     let clif_comments = fx.clif_comments;
-    let source_info_set = fx.source_info_set;
-    let local_map = fx.local_map;
+    let func_debug_cx = fx.func_debug_cx;
 
     fx.constants_cx.finalize(fx.tcx, &mut *fx.module);
 
-    crate::pretty_clif::write_clif_file(
-        tcx,
-        "unopt",
-        module.isa(),
-        instance,
-        &func,
-        &clif_comments,
-    );
+    if cx.should_write_ir {
+        crate::pretty_clif::write_clif_file(
+            tcx.output_filenames(()),
+            &symbol_name,
+            "unopt",
+            module.isa(),
+            &func,
+            &clif_comments,
+        );
+    }
 
     // Verify function
     verify_func(tcx, &clif_comments, &func);
 
-    compile_fn(
-        cx,
-        module,
-        instance,
-        symbol_name.name,
-        func_id,
-        func,
-        clif_comments,
-        source_info_set,
-        local_map,
-    );
+    CodegenedFunction { symbol_name, func_id, func, clif_comments, func_debug_cx }
 }
 
-fn compile_fn<'tcx>(
-    cx: &mut crate::CodegenCx<'tcx>,
+pub(crate) fn compile_fn(
+    cx: &mut crate::CodegenCx,
+    cached_context: &mut Context,
     module: &mut dyn Module,
-    instance: Instance<'tcx>,
-    symbol_name: &str,
-    func_id: FuncId,
-    func: Function,
-    mut clif_comments: CommentWriter,
-    source_info_set: IndexSet<SourceInfo>,
-    local_map: IndexVec<mir::Local, CPlace<'tcx>>,
+    codegened_func: CodegenedFunction,
 ) {
-    let tcx = cx.tcx;
+    let clif_comments = codegened_func.clif_comments;
 
     // Store function in context
-    let context = &mut cx.cached_context;
+    let context = cached_context;
     context.clear();
-    context.func = func;
+    context.func = codegened_func.func;
 
     // If the return block is not reachable, then the SSA builder may have inserted an `iconst.i128`
     // instruction, which doesn't have an encoding.
@@ -164,17 +159,6 @@ fn compile_fn<'tcx>(
     // invalidate it when it would change.
     context.domtree.clear();
 
-    // Perform rust specific optimizations
-    tcx.sess.time("optimize clif ir", || {
-        crate::optimize::optimize_function(
-            tcx,
-            module.isa(),
-            instance,
-            context,
-            &mut clif_comments,
-        );
-    });
-
     #[cfg(any())] // This is never true
     let _clif_guard = {
         use std::fmt::Write;
@@ -203,46 +187,44 @@ fn compile_fn<'tcx>(
     };
 
     // Define function
-    tcx.sess.time("define function", || {
-        context.want_disasm = crate::pretty_clif::should_write_ir(tcx);
-        module.define_function(func_id, context).unwrap();
+    cx.profiler.verbose_generic_activity("define function").run(|| {
+        context.want_disasm = cx.should_write_ir;
+        module.define_function(codegened_func.func_id, context).unwrap();
     });
 
-    // Write optimized function to file for debugging
-    crate::pretty_clif::write_clif_file(
-        tcx,
-        "opt",
-        module.isa(),
-        instance,
-        &context.func,
-        &clif_comments,
-    );
+    if cx.should_write_ir {
+        // Write optimized function to file for debugging
+        crate::pretty_clif::write_clif_file(
+            &cx.output_filenames,
+            &codegened_func.symbol_name,
+            "opt",
+            module.isa(),
+            &context.func,
+            &clif_comments,
+        );
 
-    if let Some(disasm) = &context.mach_compile_result.as_ref().unwrap().disasm {
-        crate::pretty_clif::write_ir_file(
-            tcx,
-            || format!("{}.vcode", tcx.symbol_name(instance).name),
-            |file| file.write_all(disasm.as_bytes()),
-        )
+        if let Some(disasm) = &context.compiled_code().unwrap().disasm {
+            crate::pretty_clif::write_ir_file(
+                &cx.output_filenames,
+                &format!("{}.vcode", codegened_func.symbol_name),
+                |file| file.write_all(disasm.as_bytes()),
+            )
+        }
     }
 
     // Define debuginfo for function
     let isa = module.isa();
     let debug_context = &mut cx.debug_context;
     let unwind_context = &mut cx.unwind_context;
-    tcx.sess.time("generate debug info", || {
+    cx.profiler.verbose_generic_activity("generate debug info").run(|| {
         if let Some(debug_context) = debug_context {
-            debug_context.define_function(
-                instance,
-                func_id,
-                symbol_name,
-                isa,
+            codegened_func.func_debug_cx.unwrap().finalize(
+                debug_context,
+                codegened_func.func_id,
                 context,
-                &source_info_set,
-                local_map,
             );
         }
-        unwind_context.add_function(func_id, &context, isa);
+        unwind_context.add_function(codegened_func.func_id, &context, isa);
     });
 }
 
@@ -268,8 +250,28 @@ pub(crate) fn verify_func(
     });
 }
 
-fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
-    for (bb, bb_data) in fx.mir.basic_blocks().iter_enumerated() {
+fn codegen_fn_body(fx: &mut FunctionCx<'_, '_, '_>, start_block: Block) {
+    if !crate::constant::check_constants(fx) {
+        fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+        fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
+        // compilation should have been aborted
+        fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+        return;
+    }
+
+    let arg_uninhabited = fx
+        .mir
+        .args_iter()
+        .any(|arg| fx.layout_of(fx.monomorphize(fx.mir.local_decls[arg].ty)).abi.is_uninhabited());
+    if arg_uninhabited {
+        fx.bcx.append_block_params_for_function_params(fx.block_map[START_BLOCK]);
+        fx.bcx.switch_to_block(fx.block_map[START_BLOCK]);
+        fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+        return;
+    }
+    fx.tcx.sess.time("codegen prelude", || crate::abi::codegen_fn_prelude(fx, start_block));
+
+    for (bb, bb_data) in fx.mir.basic_blocks.iter_enumerated() {
         let block = fx.get_block(bb);
         fx.bcx.switch_to_block(block);
 
@@ -457,17 +459,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                     template,
                     operands,
                     *options,
+                    *destination,
                 );
-
-                match *destination {
-                    Some(destination) => {
-                        let destination_block = fx.get_block(destination);
-                        fx.bcx.ins().jump(destination_block, &[]);
-                    }
-                    None => {
-                        fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
-                    }
-                }
             }
             TerminatorKind::Resume | TerminatorKind::Abort => {
                 // FIXME implement unwinding
@@ -708,12 +701,14 @@ fn codegen_stmt<'tcx>(
                     let operand = codegen_operand(fx, operand);
                     operand.unsize_value(fx, lval);
                 }
+                Rvalue::Cast(CastKind::DynStar, _, _) => {
+                    // FIXME(dyn-star)
+                    unimplemented!()
+                }
                 Rvalue::Discriminant(place) => {
                     let place = codegen_place(fx, place);
                     let value = place.to_cvalue(fx);
-                    let discr =
-                        crate::discriminant::codegen_get_discriminant(fx, value, dest_layout);
-                    lval.write_cvalue(fx, discr);
+                    crate::discriminant::codegen_get_discriminant(fx, lval, value, dest_layout);
                 }
                 Rvalue::Repeat(ref operand, times) => {
                     let operand = codegen_operand(fx, operand);
@@ -803,20 +798,31 @@ fn codegen_stmt<'tcx>(
         | StatementKind::AscribeUserType(..) => {}
 
         StatementKind::Coverage { .. } => fx.tcx.sess.fatal("-Zcoverage is unimplemented"),
-        StatementKind::CopyNonOverlapping(inner) => {
-            let dst = codegen_operand(fx, &inner.dst);
-            let pointee = dst
-                .layout()
-                .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
-                .expect("Expected pointer");
-            let dst = dst.load_scalar(fx);
-            let src = codegen_operand(fx, &inner.src).load_scalar(fx);
-            let count = codegen_operand(fx, &inner.count).load_scalar(fx);
-            let elem_size: u64 = pointee.size.bytes();
-            let bytes =
-                if elem_size != 1 { fx.bcx.ins().imul_imm(count, elem_size as i64) } else { count };
-            fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
-        }
+        StatementKind::Intrinsic(ref intrinsic) => match &**intrinsic {
+            // We ignore `assume` intrinsics, they are only useful for optimizations
+            NonDivergingIntrinsic::Assume(_) => {}
+            NonDivergingIntrinsic::CopyNonOverlapping(mir::CopyNonOverlapping {
+                src,
+                dst,
+                count,
+            }) => {
+                let dst = codegen_operand(fx, dst);
+                let pointee = dst
+                    .layout()
+                    .pointee_info_at(fx, rustc_target::abi::Size::ZERO)
+                    .expect("Expected pointer");
+                let dst = dst.load_scalar(fx);
+                let src = codegen_operand(fx, src).load_scalar(fx);
+                let count = codegen_operand(fx, count).load_scalar(fx);
+                let elem_size: u64 = pointee.size.bytes();
+                let bytes = if elem_size != 1 {
+                    fx.bcx.ins().imul_imm(count, elem_size as i64)
+                } else {
+                    count
+                };
+                fx.bcx.call_memcpy(fx.target_config, dst, src, bytes);
+            }
+        },
     }
 }
 
@@ -934,8 +940,11 @@ pub(crate) fn codegen_panic_inner<'tcx>(
     args: &[Value],
     span: Span,
 ) {
-    let def_id =
-        fx.tcx.lang_items().require(lang_item).unwrap_or_else(|s| fx.tcx.sess.span_fatal(span, &s));
+    let def_id = fx
+        .tcx
+        .lang_items()
+        .require(lang_item)
+        .unwrap_or_else(|e| fx.tcx.sess.span_fatal(span, e.to_string()));
 
     let instance = Instance::mono(fx.tcx, def_id).polymorphize(fx.tcx);
     let symbol_name = fx.tcx.symbol_name(instance).name;
diff --git a/compiler/rustc_codegen_cranelift/src/common.rs b/compiler/rustc_codegen_cranelift/src/common.rs
index f9dc1b516..589594465 100644
--- a/compiler/rustc_codegen_cranelift/src/common.rs
+++ b/compiler/rustc_codegen_cranelift/src/common.rs
@@ -1,14 +1,18 @@
 use cranelift_codegen::isa::TargetFrontendConfig;
+use gimli::write::FileId;
+
+use rustc_data_structures::sync::Lrc;
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::layout::{
     FnAbiError, FnAbiOfHelpers, FnAbiRequest, LayoutError, LayoutOfHelpers,
 };
-use rustc_middle::ty::SymbolName;
+use rustc_span::SourceFile;
 use rustc_target::abi::call::FnAbi;
 use rustc_target::abi::{Integer, Primitive};
 use rustc_target::spec::{HasTargetSpec, Target};
 
 use crate::constant::ConstantCx;
+use crate::debuginfo::FunctionDebugContext;
 use crate::prelude::*;
 
 pub(crate) fn pointer_ty(tcx: TyCtxt<'_>) -> types::Type {
@@ -74,7 +78,7 @@ fn clif_type_from_ty<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> Option<types::Typ
                 _ => unreachable!(),
             };
 
-            match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
+            match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
                 // Cranelift currently only implements icmp for 128bit vectors.
                 Some(vector_ty) if vector_ty.bits() == 128 => vector_ty,
                 _ => return None,
@@ -232,15 +236,16 @@ pub(crate) fn type_sign(ty: Ty<'_>) -> bool {
 }
 
 pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
-    pub(crate) cx: &'clif mut crate::CodegenCx<'tcx>,
+    pub(crate) cx: &'clif mut crate::CodegenCx,
     pub(crate) module: &'m mut dyn Module,
     pub(crate) tcx: TyCtxt<'tcx>,
     pub(crate) target_config: TargetFrontendConfig, // Cached from module
     pub(crate) pointer_type: Type,                  // Cached from module
     pub(crate) constants_cx: ConstantCx,
+    pub(crate) func_debug_cx: Option<FunctionDebugContext>,
 
     pub(crate) instance: Instance<'tcx>,
-    pub(crate) symbol_name: SymbolName<'tcx>,
+    pub(crate) symbol_name: String,
     pub(crate) mir: &'tcx Body<'tcx>,
     pub(crate) fn_abi: Option<&'tcx FnAbi<'tcx, Ty<'tcx>>>,
 
@@ -252,7 +257,11 @@ pub(crate) struct FunctionCx<'m, 'clif, 'tcx: 'm> {
     pub(crate) caller_location: Option<CValue<'tcx>>,
 
     pub(crate) clif_comments: crate::pretty_clif::CommentWriter,
-    pub(crate) source_info_set: indexmap::IndexSet<SourceInfo>,
+
+    /// Last accessed source file and it's debuginfo file id.
+    ///
+    /// For optimization purposes only
+    pub(crate) last_source_file: Option<(Lrc<SourceFile>, FileId)>,
 
     /// This should only be accessed by `CPlace::new_var`.
     pub(crate) next_ssa_var: u32,
@@ -336,8 +345,31 @@ impl<'tcx> FunctionCx<'_, '_, 'tcx> {
     }
 
     pub(crate) fn set_debug_loc(&mut self, source_info: mir::SourceInfo) {
-        let (index, _) = self.source_info_set.insert_full(source_info);
-        self.bcx.set_srcloc(SourceLoc::new(index as u32));
+        if let Some(debug_context) = &mut self.cx.debug_context {
+            let (file, line, column) =
+                DebugContext::get_span_loc(self.tcx, self.mir.span, source_info.span);
+
+            // add_source_file is very slow.
+            // Optimize for the common case of the current file not being changed.
+            let mut cached_file_id = None;
+            if let Some((ref last_source_file, last_file_id)) = self.last_source_file {
+                // If the allocations are not equal, the files may still be equal, but that
+                // doesn't matter, as this is just an optimization.
+                if rustc_data_structures::sync::Lrc::ptr_eq(last_source_file, &file) {
+                    cached_file_id = Some(last_file_id);
+                }
+            }
+
+            let file_id = if let Some(file_id) = cached_file_id {
+                file_id
+            } else {
+                debug_context.add_source_file(&file)
+            };
+
+            let source_loc =
+                self.func_debug_cx.as_mut().unwrap().add_dbg_loc(file_id, line, column);
+            self.bcx.set_srcloc(source_loc);
+        }
     }
 
     // Note: must be kept in sync with get_caller_location from cg_ssa
diff --git a/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
new file mode 100644
index 000000000..dfde97920
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/src/concurrency_limiter.rs
@@ -0,0 +1,168 @@
+use std::sync::{Arc, Condvar, Mutex};
+
+use rustc_session::Session;
+
+use jobserver::HelperThread;
+
+// FIXME don't panic when a worker thread panics
+
+pub(super) struct ConcurrencyLimiter {
+    helper_thread: Option<HelperThread>,
+    state: Arc<Mutex<state::ConcurrencyLimiterState>>,
+    available_token_condvar: Arc<Condvar>,
+}
+
+impl ConcurrencyLimiter {
+    pub(super) fn new(sess: &Session, pending_jobs: usize) -> Self {
+        let state = Arc::new(Mutex::new(state::ConcurrencyLimiterState::new(pending_jobs)));
+        let available_token_condvar = Arc::new(Condvar::new());
+
+        let state_helper = state.clone();
+        let available_token_condvar_helper = available_token_condvar.clone();
+        let helper_thread = sess
+            .jobserver
+            .clone()
+            .into_helper_thread(move |token| {
+                let mut state = state_helper.lock().unwrap();
+                state.add_new_token(token.unwrap());
+                available_token_condvar_helper.notify_one();
+            })
+            .unwrap();
+        ConcurrencyLimiter {
+            helper_thread: Some(helper_thread),
+            state,
+            available_token_condvar: Arc::new(Condvar::new()),
+        }
+    }
+
+    pub(super) fn acquire(&mut self) -> ConcurrencyLimiterToken {
+        let mut state = self.state.lock().unwrap();
+        loop {
+            state.assert_invariants();
+
+            if state.try_start_job() {
+                return ConcurrencyLimiterToken {
+                    state: self.state.clone(),
+                    available_token_condvar: self.available_token_condvar.clone(),
+                };
+            }
+
+            self.helper_thread.as_mut().unwrap().request_token();
+            state = self.available_token_condvar.wait(state).unwrap();
+        }
+    }
+
+    pub(super) fn job_already_done(&mut self) {
+        let mut state = self.state.lock().unwrap();
+        state.job_already_done();
+    }
+}
+
+impl Drop for ConcurrencyLimiter {
+    fn drop(&mut self) {
+        //
+        self.helper_thread.take();
+
+        // Assert that all jobs have finished
+        let state = Mutex::get_mut(Arc::get_mut(&mut self.state).unwrap()).unwrap();
+        state.assert_done();
+    }
+}
+
+#[derive(Debug)]
+pub(super) struct ConcurrencyLimiterToken {
+    state: Arc<Mutex<state::ConcurrencyLimiterState>>,
+    available_token_condvar: Arc<Condvar>,
+}
+
+impl Drop for ConcurrencyLimiterToken {
+    fn drop(&mut self) {
+        let mut state = self.state.lock().unwrap();
+        state.job_finished();
+        self.available_token_condvar.notify_one();
+    }
+}
+
+mod state {
+    use jobserver::Acquired;
+
+    #[derive(Debug)]
+    pub(super) struct ConcurrencyLimiterState {
+        pending_jobs: usize,
+        active_jobs: usize,
+
+        // None is used to represent the implicit token, Some to represent explicit tokens
+        tokens: Vec<Option<Acquired>>,
+    }
+
+    impl ConcurrencyLimiterState {
+        pub(super) fn new(pending_jobs: usize) -> Self {
+            ConcurrencyLimiterState { pending_jobs, active_jobs: 0, tokens: vec![None] }
+        }
+
+        pub(super) fn assert_invariants(&self) {
+            // There must be no excess active jobs
+            assert!(self.active_jobs <= self.pending_jobs);
+
+            // There may not be more active jobs than there are tokens
+            assert!(self.active_jobs <= self.tokens.len());
+        }
+
+        pub(super) fn assert_done(&self) {
+            assert_eq!(self.pending_jobs, 0);
+            assert_eq!(self.active_jobs, 0);
+        }
+
+        pub(super) fn add_new_token(&mut self, token: Acquired) {
+            self.tokens.push(Some(token));
+            self.drop_excess_capacity();
+        }
+
+        pub(super) fn try_start_job(&mut self) -> bool {
+            if self.active_jobs < self.tokens.len() {
+                // Using existing token
+                self.job_started();
+                return true;
+            }
+
+            false
+        }
+
+        pub(super) fn job_started(&mut self) {
+            self.assert_invariants();
+            self.active_jobs += 1;
+            self.drop_excess_capacity();
+            self.assert_invariants();
+        }
+
+        pub(super) fn job_finished(&mut self) {
+            self.assert_invariants();
+            self.pending_jobs -= 1;
+            self.active_jobs -= 1;
+            self.assert_invariants();
+            self.drop_excess_capacity();
+            self.assert_invariants();
+        }
+
+        pub(super) fn job_already_done(&mut self) {
+            self.assert_invariants();
+            self.pending_jobs -= 1;
+            self.assert_invariants();
+            self.drop_excess_capacity();
+            self.assert_invariants();
+        }
+
+        fn drop_excess_capacity(&mut self) {
+            self.assert_invariants();
+
+            // Drop all tokens that can never be used anymore
+            self.tokens.truncate(std::cmp::max(self.pending_jobs, 1));
+
+            // Keep some excess tokens to satisfy requests faster
+            const MAX_EXTRA_CAPACITY: usize = 2;
+            self.tokens.truncate(std::cmp::max(self.active_jobs + MAX_EXTRA_CAPACITY, 1));
+
+            self.assert_invariants();
+        }
+    }
+}
diff --git a/compiler/rustc_codegen_cranelift/src/constant.rs b/compiler/rustc_codegen_cranelift/src/constant.rs
index 7f7fd0e9c..6b4ed9b9d 100644
--- a/compiler/rustc_codegen_cranelift/src/constant.rs
+++ b/compiler/rustc_codegen_cranelift/src/constant.rs
@@ -41,36 +41,30 @@ impl ConstantCx {
 pub(crate) fn check_constants(fx: &mut FunctionCx<'_, '_, '_>) -> bool {
     let mut all_constants_ok = true;
     for constant in &fx.mir.required_consts {
-        let const_ = match fx.monomorphize(constant.literal) {
-            ConstantKind::Ty(ct) => ct,
+        let unevaluated = match fx.monomorphize(constant.literal) {
+            ConstantKind::Ty(ct) => match ct.kind() {
+                ConstKind::Unevaluated(uv) => uv.expand(),
+                ConstKind::Value(_) => continue,
+                ConstKind::Param(_)
+                | ConstKind::Infer(_)
+                | ConstKind::Bound(_, _)
+                | ConstKind::Placeholder(_)
+                | ConstKind::Error(_) => unreachable!("{:?}", ct),
+            },
+            ConstantKind::Unevaluated(uv, _) => uv,
             ConstantKind::Val(..) => continue,
         };
-        match const_.kind() {
-            ConstKind::Value(_) => {}
-            ConstKind::Unevaluated(unevaluated) => {
-                if let Err(err) =
-                    fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None)
-                {
-                    all_constants_ok = false;
-                    match err {
-                        ErrorHandled::Reported(_) | ErrorHandled::Linted => {
-                            fx.tcx.sess.span_err(constant.span, "erroneous constant encountered");
-                        }
-                        ErrorHandled::TooGeneric => {
-                            span_bug!(
-                                constant.span,
-                                "codgen encountered polymorphic constant: {:?}",
-                                err
-                            );
-                        }
-                    }
+
+        if let Err(err) = fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {
+            all_constants_ok = false;
+            match err {
+                ErrorHandled::Reported(_) | ErrorHandled::Linted => {
+                    fx.tcx.sess.span_err(constant.span, "erroneous constant encountered");
+                }
+                ErrorHandled::TooGeneric => {
+                    span_bug!(constant.span, "codegen encountered polymorphic constant: {:?}", err);
                 }
             }
-            ConstKind::Param(_)
-            | ConstKind::Infer(_)
-            | ConstKind::Bound(_, _)
-            | ConstKind::Placeholder(_)
-            | ConstKind::Error(_) => unreachable!("{:?}", const_),
         }
     }
     all_constants_ok
@@ -122,36 +116,28 @@ pub(crate) fn codegen_constant<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
     constant: &Constant<'tcx>,
 ) -> CValue<'tcx> {
-    let const_ = match fx.monomorphize(constant.literal) {
-        ConstantKind::Ty(ct) => ct,
-        ConstantKind::Val(val, ty) => return codegen_const_value(fx, val, ty),
-    };
-    let const_val = match const_.kind() {
-        ConstKind::Value(valtree) => fx.tcx.valtree_to_const_val((const_.ty(), valtree)),
-        ConstKind::Unevaluated(ty::Unevaluated { def, substs, promoted })
+    let (const_val, ty) = match fx.monomorphize(constant.literal) {
+        ConstantKind::Ty(const_) => unreachable!("{:?}", const_),
+        ConstantKind::Unevaluated(ty::Unevaluated { def, substs, promoted }, ty)
             if fx.tcx.is_static(def.did) =>
         {
             assert!(substs.is_empty());
             assert!(promoted.is_none());
 
-            return codegen_static_ref(fx, def.did, fx.layout_of(const_.ty())).to_cvalue(fx);
+            return codegen_static_ref(fx, def.did, fx.layout_of(ty)).to_cvalue(fx);
         }
-        ConstKind::Unevaluated(unevaluated) => {
+        ConstantKind::Unevaluated(unevaluated, ty) => {
             match fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), unevaluated, None) {
-                Ok(const_val) => const_val,
+                Ok(const_val) => (const_val, ty),
                 Err(_) => {
                     span_bug!(constant.span, "erroneous constant not captured by required_consts");
                 }
             }
         }
-        ConstKind::Param(_)
-        | ConstKind::Infer(_)
-        | ConstKind::Bound(_, _)
-        | ConstKind::Placeholder(_)
-        | ConstKind::Error(_) => unreachable!("{:?}", const_),
+        ConstantKind::Val(val, ty) => (val, ty),
     };
 
-    codegen_const_value(fx, const_val, const_.ty())
+    codegen_const_value(fx, const_val, ty)
 }
 
 pub(crate) fn codegen_const_value<'tcx>(
@@ -430,7 +416,7 @@ fn define_all_allocs(tcx: TyCtxt<'_>, module: &mut dyn Module, cx: &mut Constant
         let bytes = alloc.inspect_with_uninit_and_ptr_outside_interpreter(0..alloc.len()).to_vec();
         data_ctx.define(bytes.into_boxed_slice());
 
-        for &(offset, alloc_id) in alloc.relocations().iter() {
+        for &(offset, alloc_id) in alloc.provenance().iter() {
             let addend = {
                 let endianness = tcx.data_layout.endian;
                 let offset = offset.bytes() as usize;
@@ -496,6 +482,9 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                 .eval_for_mir(fx.tcx, ParamEnv::reveal_all())
                 .try_to_value(fx.tcx),
             ConstantKind::Val(val, _) => Some(val),
+            ConstantKind::Unevaluated(uv, _) => {
+                fx.tcx.const_eval_resolve(ParamEnv::reveal_all(), uv, None).ok()
+            }
         },
         // FIXME(rust-lang/rust#85105): Casts like `IMM8 as u32` result in the const being stored
         // inside a temporary before being passed to the intrinsic requiring the const argument.
@@ -505,7 +494,7 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                 return None;
             }
             let mut computed_const_val = None;
-            for bb_data in fx.mir.basic_blocks() {
+            for bb_data in fx.mir.basic_blocks.iter() {
                 for stmt in &bb_data.statements {
                     match &stmt.kind {
                         StatementKind::Assign(local_and_rvalue) if &local_and_rvalue.0 == place => {
@@ -536,9 +525,11 @@ pub(crate) fn mir_operand_get_const_val<'tcx>(
                         {
                             return None;
                         }
-                        StatementKind::CopyNonOverlapping(_) => {
-                            return None;
-                        } // conservative handling
+                        StatementKind::Intrinsic(ref intrinsic) => match **intrinsic {
+                            NonDivergingIntrinsic::CopyNonOverlapping(..) => return None,
+                            NonDivergingIntrinsic::Assume(..) => {}
+                        },
+                        // conservative handling
                         StatementKind::Assign(_)
                         | StatementKind::FakeRead(_)
                         | StatementKind::SetDiscriminant { .. }
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
index 589910ede..9583cd2ec 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/emit.rs
@@ -9,7 +9,7 @@ use gimli::{RunTimeEndian, SectionId};
 use super::object::WriteDebugInfo;
 use super::DebugContext;
 
-impl DebugContext<'_> {
+impl DebugContext {
     pub(crate) fn emit(&mut self, product: &mut ObjectProduct) {
         let unit_range_list_id = self.dwarf.unit.ranges.add(self.unit_range_list.clone());
         let root = self.dwarf.unit.root();
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
index bbcb95913..463de6a91 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/line_info.rs
@@ -3,8 +3,10 @@
 use std::ffi::OsStr;
 use std::path::{Component, Path};
 
+use crate::debuginfo::FunctionDebugContext;
 use crate::prelude::*;
 
+use rustc_data_structures::sync::Lrc;
 use rustc_span::{
     FileName, Pos, SourceFile, SourceFileAndLine, SourceFileHash, SourceFileHashAlgorithm,
 };
@@ -14,7 +16,6 @@ use cranelift_codegen::MachSrcLoc;
 
 use gimli::write::{
     Address, AttributeValue, FileId, FileInfo, LineProgram, LineString, LineStringTable,
-    UnitEntryId,
 };
 
 // OPTIMIZATION: It is cheaper to do this in one pass than using `.parent()` and `.file_name()`.
@@ -47,9 +48,9 @@ fn osstr_as_utf8_bytes(path: &OsStr) -> &[u8] {
     }
 }
 
-pub(crate) const MD5_LEN: usize = 16;
+const MD5_LEN: usize = 16;
 
-pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
+fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
     if hash.kind == SourceFileHashAlgorithm::Md5 {
         let mut buf = [0u8; MD5_LEN];
         buf.copy_from_slice(hash.hash_bytes());
@@ -59,160 +60,132 @@ pub(crate) fn make_file_info(hash: SourceFileHash) -> Option<FileInfo> {
     }
 }
 
-fn line_program_add_file(
-    line_program: &mut LineProgram,
-    line_strings: &mut LineStringTable,
-    file: &SourceFile,
-) -> FileId {
-    match &file.name {
-        FileName::Real(path) => {
-            let (dir_path, file_name) = split_path_dir_and_file(path.remapped_path_if_available());
-            let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str());
-            let file_name = osstr_as_utf8_bytes(file_name);
-
-            let dir_id = if !dir_name.is_empty() {
-                let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings);
-                line_program.add_directory(dir_name)
-            } else {
-                line_program.default_directory()
-            };
-            let file_name = LineString::new(file_name, line_program.encoding(), line_strings);
+impl DebugContext {
+    pub(crate) fn get_span_loc(
+        tcx: TyCtxt<'_>,
+        function_span: Span,
+        span: Span,
+    ) -> (Lrc<SourceFile>, u64, u64) {
+        // Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131
+        // In order to have a good line stepping behavior in debugger, we overwrite debug
+        // locations of macro expansions with that of the outermost expansion site (when the macro is
+        // annotated with `#[collapse_debuginfo]` or when `-Zdebug-macros` is provided).
+        let span = if tcx.should_collapse_debuginfo(span) {
+            span
+        } else {
+            // Walk up the macro expansion chain until we reach a non-expanded span.
+            // We also stop at the function body level because no line stepping can occur
+            // at the level above that.
+            rustc_span::hygiene::walk_chain(span, function_span.ctxt())
+        };
 
-            let info = make_file_info(file.src_hash);
+        match tcx.sess.source_map().lookup_line(span.lo()) {
+            Ok(SourceFileAndLine { sf: file, line }) => {
+                let line_pos = file.line_begin_pos(span.lo());
 
-            line_program.file_has_md5 &= info.is_some();
-            line_program.add_file(file_name, dir_id, info)
+                (
+                    file,
+                    u64::try_from(line).unwrap() + 1,
+                    u64::from((span.lo() - line_pos).to_u32()) + 1,
+                )
+            }
+            Err(file) => (file, 0, 0),
         }
-        // FIXME give more appropriate file names
-        filename => {
-            let dir_id = line_program.default_directory();
-            let dummy_file_name = LineString::new(
-                filename.prefer_remapped().to_string().into_bytes(),
-                line_program.encoding(),
-                line_strings,
-            );
-            line_program.add_file(dummy_file_name, dir_id, None)
+    }
+
+    pub(crate) fn add_source_file(&mut self, source_file: &SourceFile) -> FileId {
+        let line_program: &mut LineProgram = &mut self.dwarf.unit.line_program;
+        let line_strings: &mut LineStringTable = &mut self.dwarf.line_strings;
+
+        match &source_file.name {
+            FileName::Real(path) => {
+                let (dir_path, file_name) =
+                    split_path_dir_and_file(path.remapped_path_if_available());
+                let dir_name = osstr_as_utf8_bytes(dir_path.as_os_str());
+                let file_name = osstr_as_utf8_bytes(file_name);
+
+                let dir_id = if !dir_name.is_empty() {
+                    let dir_name = LineString::new(dir_name, line_program.encoding(), line_strings);
+                    line_program.add_directory(dir_name)
+                } else {
+                    line_program.default_directory()
+                };
+                let file_name = LineString::new(file_name, line_program.encoding(), line_strings);
+
+                let info = make_file_info(source_file.src_hash);
+
+                line_program.file_has_md5 &= info.is_some();
+                line_program.add_file(file_name, dir_id, info)
+            }
+            // FIXME give more appropriate file names
+            filename => {
+                let dir_id = line_program.default_directory();
+                let dummy_file_name = LineString::new(
+                    filename.prefer_remapped().to_string().into_bytes(),
+                    line_program.encoding(),
+                    line_strings,
+                );
+                line_program.add_file(dummy_file_name, dir_id, None)
+            }
         }
     }
 }
 
-impl<'tcx> DebugContext<'tcx> {
-    pub(super) fn emit_location(&mut self, entry_id: UnitEntryId, span: Span) {
-        let loc = self.tcx.sess.source_map().lookup_char_pos(span.lo());
-
-        let file_id = line_program_add_file(
-            &mut self.dwarf.unit.line_program,
-            &mut self.dwarf.line_strings,
-            &loc.file,
-        );
-
-        let entry = self.dwarf.unit.get_mut(entry_id);
-
-        entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
-        entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(loc.line as u64));
-        entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(loc.col.to_usize() as u64));
+impl FunctionDebugContext {
+    pub(crate) fn add_dbg_loc(&mut self, file_id: FileId, line: u64, column: u64) -> SourceLoc {
+        let (index, _) = self.source_loc_set.insert_full((file_id, line, column));
+        SourceLoc::new(u32::try_from(index).unwrap())
     }
 
     pub(super) fn create_debug_lines(
         &mut self,
+        debug_context: &mut DebugContext,
         symbol: usize,
-        entry_id: UnitEntryId,
         context: &Context,
-        function_span: Span,
-        source_info_set: &indexmap::IndexSet<SourceInfo>,
     ) -> CodeOffset {
-        let tcx = self.tcx;
-        let line_program = &mut self.dwarf.unit.line_program;
-
-        let line_strings = &mut self.dwarf.line_strings;
-        let mut last_span = None;
-        let mut last_file = None;
-        let mut create_row_for_span = |line_program: &mut LineProgram, span: Span| {
-            if let Some(last_span) = last_span {
-                if span == last_span {
-                    line_program.generate_row();
-                    return;
-                }
-            }
-            last_span = Some(span);
-
-            // Based on https://github.com/rust-lang/rust/blob/e369d87b015a84653343032833d65d0545fd3f26/src/librustc_codegen_ssa/mir/mod.rs#L116-L131
-            // In order to have a good line stepping behavior in debugger, we overwrite debug
-            // locations of macro expansions with that of the outermost expansion site
-            // (unless the crate is being compiled with `-Z debug-macros`).
-            let span = if !span.from_expansion() || tcx.sess.opts.unstable_opts.debug_macros {
-                span
-            } else {
-                // Walk up the macro expansion chain until we reach a non-expanded span.
-                // We also stop at the function body level because no line stepping can occur
-                // at the level above that.
-                rustc_span::hygiene::walk_chain(span, function_span.ctxt())
+        let create_row_for_span =
+            |debug_context: &mut DebugContext, source_loc: (FileId, u64, u64)| {
+                let (file_id, line, col) = source_loc;
+
+                debug_context.dwarf.unit.line_program.row().file = file_id;
+                debug_context.dwarf.unit.line_program.row().line = line;
+                debug_context.dwarf.unit.line_program.row().column = col;
+                debug_context.dwarf.unit.line_program.generate_row();
             };
 
-            let (file, line, col) = match tcx.sess.source_map().lookup_line(span.lo()) {
-                Ok(SourceFileAndLine { sf: file, line }) => {
-                    let line_pos = file.line_begin_pos(span.lo());
-
-                    (
-                        file,
-                        u64::try_from(line).unwrap() + 1,
-                        u64::from((span.lo() - line_pos).to_u32()) + 1,
-                    )
-                }
-                Err(file) => (file, 0, 0),
-            };
-
-            // line_program_add_file is very slow.
-            // Optimize for the common case of the current file not being changed.
-            let current_file_changed = if let Some(last_file) = &last_file {
-                // If the allocations are not equal, then the files may still be equal, but that
-                // is not a problem, as this is just an optimization.
-                !rustc_data_structures::sync::Lrc::ptr_eq(last_file, &file)
-            } else {
-                true
-            };
-            if current_file_changed {
-                let file_id = line_program_add_file(line_program, line_strings, &file);
-                line_program.row().file = file_id;
-                last_file = Some(file);
-            }
-
-            line_program.row().line = line;
-            line_program.row().column = col;
-            line_program.generate_row();
-        };
-
-        line_program.begin_sequence(Some(Address::Symbol { symbol, addend: 0 }));
+        debug_context
+            .dwarf
+            .unit
+            .line_program
+            .begin_sequence(Some(Address::Symbol { symbol, addend: 0 }));
 
         let mut func_end = 0;
 
-        let mcr = context.mach_compile_result.as_ref().unwrap();
+        let mcr = context.compiled_code().unwrap();
         for &MachSrcLoc { start, end, loc } in mcr.buffer.get_srclocs_sorted() {
-            line_program.row().address_offset = u64::from(start);
+            debug_context.dwarf.unit.line_program.row().address_offset = u64::from(start);
             if !loc.is_default() {
-                let source_info = *source_info_set.get_index(loc.bits() as usize).unwrap();
-                create_row_for_span(line_program, source_info.span);
+                let source_loc = *self.source_loc_set.get_index(loc.bits() as usize).unwrap();
+                create_row_for_span(debug_context, source_loc);
             } else {
-                create_row_for_span(line_program, function_span);
+                create_row_for_span(debug_context, self.function_source_loc);
             }
             func_end = end;
         }
 
-        line_program.end_sequence(u64::from(func_end));
+        debug_context.dwarf.unit.line_program.end_sequence(u64::from(func_end));
 
         let func_end = mcr.buffer.total_size();
 
         assert_ne!(func_end, 0);
 
-        let entry = self.dwarf.unit.get_mut(entry_id);
+        let entry = debug_context.dwarf.unit.get_mut(self.entry_id);
         entry.set(
             gimli::DW_AT_low_pc,
             AttributeValue::Address(Address::Symbol { symbol, addend: 0 }),
         );
         entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(func_end)));
 
-        self.emit_location(entry_id, function_span);
-
         func_end
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
index 693092ba5..c55db2017 100644
--- a/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/debuginfo/mod.rs
@@ -7,35 +7,34 @@ mod unwind;
 
 use crate::prelude::*;
 
-use rustc_index::vec::IndexVec;
-
-use cranelift_codegen::entity::EntityRef;
-use cranelift_codegen::ir::{Endianness, LabelValueLoc, ValueLabel};
+use cranelift_codegen::ir::Endianness;
 use cranelift_codegen::isa::TargetIsa;
-use cranelift_codegen::ValueLocRange;
 
 use gimli::write::{
-    Address, AttributeValue, DwarfUnit, Expression, LineProgram, LineString, Location,
-    LocationList, Range, RangeList, UnitEntryId,
+    Address, AttributeValue, DwarfUnit, FileId, LineProgram, LineString, Range, RangeList,
+    UnitEntryId,
 };
-use gimli::{Encoding, Format, LineEncoding, RunTimeEndian, X86_64};
+use gimli::{Encoding, Format, LineEncoding, RunTimeEndian};
+use indexmap::IndexSet;
 
 pub(crate) use emit::{DebugReloc, DebugRelocName};
 pub(crate) use unwind::UnwindContext;
 
-pub(crate) struct DebugContext<'tcx> {
-    tcx: TyCtxt<'tcx>,
-
+pub(crate) struct DebugContext {
     endian: RunTimeEndian,
 
     dwarf: DwarfUnit,
     unit_range_list: RangeList,
+}
 
-    types: FxHashMap<Ty<'tcx>, UnitEntryId>,
+pub(crate) struct FunctionDebugContext {
+    entry_id: UnitEntryId,
+    function_source_loc: (FileId, u64, u64),
+    source_loc_set: indexmap::IndexSet<(FileId, u64, u64)>,
 }
 
-impl<'tcx> DebugContext<'tcx> {
-    pub(crate) fn new(tcx: TyCtxt<'tcx>, isa: &dyn TargetIsa) -> Self {
+impl DebugContext {
+    pub(crate) fn new(tcx: TyCtxt<'_>, isa: &dyn TargetIsa) -> Self {
         let encoding = Encoding {
             format: Format::Dwarf32,
             // FIXME this should be configurable
@@ -101,127 +100,18 @@ impl<'tcx> DebugContext<'tcx> {
             root.set(gimli::DW_AT_low_pc, AttributeValue::Address(Address::Constant(0)));
         }
 
-        DebugContext {
-            tcx,
-
-            endian,
-
-            dwarf,
-            unit_range_list: RangeList(Vec::new()),
-
-            types: FxHashMap::default(),
-        }
-    }
-
-    fn dwarf_ty(&mut self, ty: Ty<'tcx>) -> UnitEntryId {
-        if let Some(type_id) = self.types.get(&ty) {
-            return *type_id;
-        }
-
-        let new_entry = |dwarf: &mut DwarfUnit, tag| dwarf.unit.add(dwarf.unit.root(), tag);
-
-        let primitive = |dwarf: &mut DwarfUnit, ate| {
-            let type_id = new_entry(dwarf, gimli::DW_TAG_base_type);
-            let type_entry = dwarf.unit.get_mut(type_id);
-            type_entry.set(gimli::DW_AT_encoding, AttributeValue::Encoding(ate));
-            type_id
-        };
-
-        let name = format!("{}", ty);
-        let layout = self.tcx.layout_of(ParamEnv::reveal_all().and(ty)).unwrap();
-
-        let type_id = match ty.kind() {
-            ty::Bool => primitive(&mut self.dwarf, gimli::DW_ATE_boolean),
-            ty::Char => primitive(&mut self.dwarf, gimli::DW_ATE_UTF),
-            ty::Uint(_) => primitive(&mut self.dwarf, gimli::DW_ATE_unsigned),
-            ty::Int(_) => primitive(&mut self.dwarf, gimli::DW_ATE_signed),
-            ty::Float(_) => primitive(&mut self.dwarf, gimli::DW_ATE_float),
-            ty::Ref(_, pointee_ty, _mutbl)
-            | ty::RawPtr(ty::TypeAndMut { ty: pointee_ty, mutbl: _mutbl }) => {
-                let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_pointer_type);
-
-                // Ensure that type is inserted before recursing to avoid duplicates
-                self.types.insert(ty, type_id);
-
-                let pointee = self.dwarf_ty(*pointee_ty);
-
-                let type_entry = self.dwarf.unit.get_mut(type_id);
-
-                //type_entry.set(gimli::DW_AT_mutable, AttributeValue::Flag(mutbl == rustc_hir::Mutability::Mut));
-                type_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(pointee));
-
-                type_id
-            }
-            ty::Adt(adt_def, _substs) if adt_def.is_struct() && !layout.is_unsized() => {
-                let type_id = new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type);
-
-                // Ensure that type is inserted before recursing to avoid duplicates
-                self.types.insert(ty, type_id);
-
-                let variant = adt_def.non_enum_variant();
-
-                for (field_idx, field_def) in variant.fields.iter().enumerate() {
-                    let field_offset = layout.fields.offset(field_idx);
-                    let field_layout = layout.field(
-                        &layout::LayoutCx { tcx: self.tcx, param_env: ParamEnv::reveal_all() },
-                        field_idx,
-                    );
-
-                    let field_type = self.dwarf_ty(field_layout.ty);
-
-                    let field_id = self.dwarf.unit.add(type_id, gimli::DW_TAG_member);
-                    let field_entry = self.dwarf.unit.get_mut(field_id);
-
-                    field_entry.set(
-                        gimli::DW_AT_name,
-                        AttributeValue::String(field_def.name.as_str().to_string().into_bytes()),
-                    );
-                    field_entry.set(
-                        gimli::DW_AT_data_member_location,
-                        AttributeValue::Udata(field_offset.bytes()),
-                    );
-                    field_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(field_type));
-                }
-
-                type_id
-            }
-            _ => new_entry(&mut self.dwarf, gimli::DW_TAG_structure_type),
-        };
-
-        let type_entry = self.dwarf.unit.get_mut(type_id);
-
-        type_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
-        type_entry.set(gimli::DW_AT_byte_size, AttributeValue::Udata(layout.size.bytes()));
-
-        self.types.insert(ty, type_id);
-
-        type_id
-    }
-
-    fn define_local(&mut self, scope: UnitEntryId, name: String, ty: Ty<'tcx>) -> UnitEntryId {
-        let dw_ty = self.dwarf_ty(ty);
-
-        let var_id = self.dwarf.unit.add(scope, gimli::DW_TAG_variable);
-        let var_entry = self.dwarf.unit.get_mut(var_id);
-
-        var_entry.set(gimli::DW_AT_name, AttributeValue::String(name.into_bytes()));
-        var_entry.set(gimli::DW_AT_type, AttributeValue::UnitRef(dw_ty));
-
-        var_id
+        DebugContext { endian, dwarf, unit_range_list: RangeList(Vec::new()) }
     }
 
     pub(crate) fn define_function(
         &mut self,
-        instance: Instance<'tcx>,
-        func_id: FuncId,
+        tcx: TyCtxt<'_>,
         name: &str,
-        isa: &dyn TargetIsa,
-        context: &Context,
-        source_info_set: &indexmap::IndexSet<SourceInfo>,
-        local_map: IndexVec<mir::Local, CPlace<'tcx>>,
-    ) {
-        let symbol = func_id.as_u32() as usize;
-        let mir = self.tcx.instance_mir(instance.def);
+        function_span: Span,
+    ) -> FunctionDebugContext {
+        let (file, line, column) = DebugContext::get_span_loc(tcx, function_span, function_span);
+
+        let file_id = self.add_source_file(&file);
 
         // FIXME: add to appropriate scope instead of root
         let scope = self.dwarf.unit.root();
@@ -233,14 +123,35 @@ impl<'tcx> DebugContext<'tcx> {
         entry.set(gimli::DW_AT_name, AttributeValue::StringRef(name_id));
         entry.set(gimli::DW_AT_linkage_name, AttributeValue::StringRef(name_id));
 
-        let end = self.create_debug_lines(symbol, entry_id, context, mir.span, source_info_set);
+        entry.set(gimli::DW_AT_decl_file, AttributeValue::FileIndex(Some(file_id)));
+        entry.set(gimli::DW_AT_decl_line, AttributeValue::Udata(line));
+        entry.set(gimli::DW_AT_decl_column, AttributeValue::Udata(column));
 
-        self.unit_range_list.0.push(Range::StartLength {
+        FunctionDebugContext {
+            entry_id,
+            function_source_loc: (file_id, line, column),
+            source_loc_set: IndexSet::new(),
+        }
+    }
+}
+
+impl FunctionDebugContext {
+    pub(crate) fn finalize(
+        mut self,
+        debug_context: &mut DebugContext,
+        func_id: FuncId,
+        context: &Context,
+    ) {
+        let symbol = func_id.as_u32() as usize;
+
+        let end = self.create_debug_lines(debug_context, symbol, context);
+
+        debug_context.unit_range_list.0.push(Range::StartLength {
             begin: Address::Symbol { symbol, addend: 0 },
             length: u64::from(end),
         });
 
-        let func_entry = self.dwarf.unit.get_mut(entry_id);
+        let func_entry = debug_context.dwarf.unit.get_mut(self.entry_id);
         // Gdb requires both DW_AT_low_pc and DW_AT_high_pc. Otherwise the DW_TAG_subprogram is skipped.
         func_entry.set(
             gimli::DW_AT_low_pc,
@@ -248,110 +159,5 @@ impl<'tcx> DebugContext<'tcx> {
         );
         // Using Udata for DW_AT_high_pc requires at least DWARF4
         func_entry.set(gimli::DW_AT_high_pc, AttributeValue::Udata(u64::from(end)));
-
-        // FIXME make it more reliable and implement scopes before re-enabling this.
-        if false {
-            let value_labels_ranges = std::collections::HashMap::new(); // FIXME
-
-            for (local, _local_decl) in mir.local_decls.iter_enumerated() {
-                let ty = self.tcx.subst_and_normalize_erasing_regions(
-                    instance.substs,
-                    ty::ParamEnv::reveal_all(),
-                    mir.local_decls[local].ty,
-                );
-                let var_id = self.define_local(entry_id, format!("{:?}", local), ty);
-
-                let location = place_location(
-                    self,
-                    isa,
-                    symbol,
-                    &local_map,
-                    &value_labels_ranges,
-                    Place { local, projection: ty::List::empty() },
-                );
-
-                let var_entry = self.dwarf.unit.get_mut(var_id);
-                var_entry.set(gimli::DW_AT_location, location);
-            }
-        }
-
-        // FIXME create locals for all entries in mir.var_debug_info
-    }
-}
-
-fn place_location<'tcx>(
-    debug_context: &mut DebugContext<'tcx>,
-    isa: &dyn TargetIsa,
-    symbol: usize,
-    local_map: &IndexVec<mir::Local, CPlace<'tcx>>,
-    #[allow(rustc::default_hash_types)] value_labels_ranges: &std::collections::HashMap<
-        ValueLabel,
-        Vec<ValueLocRange>,
-    >,
-    place: Place<'tcx>,
-) -> AttributeValue {
-    assert!(place.projection.is_empty()); // FIXME implement them
-
-    match local_map[place.local].inner() {
-        CPlaceInner::Var(_local, var) => {
-            let value_label = cranelift_codegen::ir::ValueLabel::new(var.index());
-            if let Some(value_loc_ranges) = value_labels_ranges.get(&value_label) {
-                let loc_list = LocationList(
-                    value_loc_ranges
-                        .iter()
-                        .map(|value_loc_range| Location::StartEnd {
-                            begin: Address::Symbol {
-                                symbol,
-                                addend: i64::from(value_loc_range.start),
-                            },
-                            end: Address::Symbol { symbol, addend: i64::from(value_loc_range.end) },
-                            data: translate_loc(isa, value_loc_range.loc).unwrap(),
-                        })
-                        .collect(),
-                );
-                let loc_list_id = debug_context.dwarf.unit.locations.add(loc_list);
-
-                AttributeValue::LocationListRef(loc_list_id)
-            } else {
-                // FIXME set value labels for unused locals
-
-                AttributeValue::Exprloc(Expression::new())
-            }
-        }
-        CPlaceInner::VarPair(_, _, _) => {
-            // FIXME implement this
-
-            AttributeValue::Exprloc(Expression::new())
-        }
-        CPlaceInner::VarLane(_, _, _) => {
-            // FIXME implement this
-
-            AttributeValue::Exprloc(Expression::new())
-        }
-        CPlaceInner::Addr(_, _) => {
-            // FIXME implement this (used by arguments and returns)
-
-            AttributeValue::Exprloc(Expression::new())
-
-            // For PointerBase::Stack:
-            //AttributeValue::Exprloc(translate_loc(ValueLoc::Stack(*stack_slot)).unwrap())
-        }
-    }
-}
-
-// Adapted from https://github.com/CraneStation/wasmtime/blob/5a1845b4caf7a5dba8eda1fef05213a532ed4259/crates/debug/src/transform/expression.rs#L59-L137
-fn translate_loc(isa: &dyn TargetIsa, loc: LabelValueLoc) -> Option<Expression> {
-    match loc {
-        LabelValueLoc::Reg(reg) => {
-            let machine_reg = isa.map_regalloc_reg_to_dwarf(reg).unwrap();
-            let mut expr = Expression::new();
-            expr.op_reg(gimli::Register(machine_reg));
-            Some(expr)
-        }
-        LabelValueLoc::SPOffset(offset) => {
-            let mut expr = Expression::new();
-            expr.op_breg(X86_64::RSP, offset);
-            Some(expr)
-        }
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/discriminant.rs b/compiler/rustc_codegen_cranelift/src/discriminant.rs
index f619bb5ed..97b395bcd 100644
--- a/compiler/rustc_codegen_cranelift/src/discriminant.rs
+++ b/compiler/rustc_codegen_cranelift/src/discriminant.rs
@@ -42,10 +42,10 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
         Variants::Multiple {
             tag: _,
             tag_field,
-            tag_encoding: TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start },
+            tag_encoding: TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start },
             variants: _,
         } => {
-            if variant_index != dataful_variant {
+            if variant_index != untagged_variant {
                 let niche = place.place_field(fx, mir::Field::new(tag_field));
                 let niche_value = variant_index.as_u32() - niche_variants.start().as_u32();
                 let niche_value = ty::ScalarInt::try_from_uint(
@@ -62,16 +62,14 @@ pub(crate) fn codegen_set_discriminant<'tcx>(
 
 pub(crate) fn codegen_get_discriminant<'tcx>(
     fx: &mut FunctionCx<'_, '_, 'tcx>,
+    dest: CPlace<'tcx>,
     value: CValue<'tcx>,
     dest_layout: TyAndLayout<'tcx>,
-) -> CValue<'tcx> {
+) {
     let layout = value.layout();
 
-    if layout.abi == Abi::Uninhabited {
-        let true_ = fx.bcx.ins().iconst(types::I32, 1);
-        fx.bcx.ins().trapnz(true_, TrapCode::UnreachableCodeReached);
-        // Return a dummy value
-        return CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout);
+    if layout.abi.is_uninhabited() {
+        return;
     }
 
     let (tag_scalar, tag_field, tag_encoding) = match &layout.variants {
@@ -89,7 +87,9 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
             } else {
                 ty::ScalarInt::try_from_uint(discr_val, dest_layout.size).unwrap()
             };
-            return CValue::const_val(fx, dest_layout, discr_val);
+            let res = CValue::const_val(fx, dest_layout, discr_val);
+            dest.write_cvalue(fx, res);
+            return;
         }
         Variants::Multiple { tag, tag_field, tag_encoding, variants: _ } => {
             (tag, *tag_field, tag_encoding)
@@ -110,9 +110,10 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
                 _ => false,
             };
             let val = clif_intcast(fx, tag, cast_to, signed);
-            CValue::by_val(val, dest_layout)
+            let res = CValue::by_val(val, dest_layout);
+            dest.write_cvalue(fx, res);
         }
-        TagEncoding::Niche { dataful_variant, ref niche_variants, niche_start } => {
+        TagEncoding::Niche { untagged_variant, ref niche_variants, niche_start } => {
             // Rebase from niche values to discriminants, and check
             // whether the result is in range for the niche variants.
 
@@ -168,9 +169,11 @@ pub(crate) fn codegen_get_discriminant<'tcx>(
                 fx.bcx.ins().iadd_imm(relative_discr, i64::from(niche_variants.start().as_u32()))
             };
 
-            let dataful_variant = fx.bcx.ins().iconst(cast_to, i64::from(dataful_variant.as_u32()));
-            let discr = fx.bcx.ins().select(is_niche, niche_discr, dataful_variant);
-            CValue::by_val(discr, dest_layout)
+            let untagged_variant =
+                fx.bcx.ins().iconst(cast_to, i64::from(untagged_variant.as_u32()));
+            let discr = fx.bcx.ins().select(is_niche, niche_discr, untagged_variant);
+            let res = CValue::by_val(discr, dest_layout);
+            dest.write_cvalue(fx, res);
         }
     }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/driver/aot.rs b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
index 3cd1ef563..8eabe1cbc 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/aot.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/aot.rs
@@ -1,33 +1,129 @@
 //! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a
 //! standalone executable.
 
+use std::fs::File;
 use std::path::PathBuf;
+use std::sync::Arc;
+use std::thread::JoinHandle;
 
-use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
 use rustc_codegen_ssa::back::metadata::create_compressed_metadata_file;
 use rustc_codegen_ssa::{CodegenResults, CompiledModule, CrateInfo, ModuleKind};
+use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
 use rustc_middle::mir::mono::{CodegenUnit, MonoItem};
 use rustc_session::cgu_reuse_tracker::CguReuse;
-use rustc_session::config::{DebugInfo, OutputType};
+use rustc_session::config::{DebugInfo, OutputFilenames, OutputType};
 use rustc_session::Session;
 
-use cranelift_codegen::isa::TargetIsa;
 use cranelift_object::{ObjectBuilder, ObjectModule};
 
+use crate::concurrency_limiter::{ConcurrencyLimiter, ConcurrencyLimiterToken};
+use crate::global_asm::GlobalAsmConfig;
 use crate::{prelude::*, BackendConfig};
 
-struct ModuleCodegenResult(CompiledModule, Option<(WorkProductId, WorkProduct)>);
+struct ModuleCodegenResult {
+    module_regular: CompiledModule,
+    module_global_asm: Option<CompiledModule>,
+    existing_work_product: Option<(WorkProductId, WorkProduct)>,
+}
+
+enum OngoingModuleCodegen {
+    Sync(Result<ModuleCodegenResult, String>),
+    Async(JoinHandle<Result<ModuleCodegenResult, String>>),
+}
 
-impl<HCX> HashStable<HCX> for ModuleCodegenResult {
+impl<HCX> HashStable<HCX> for OngoingModuleCodegen {
     fn hash_stable(&self, _: &mut HCX, _: &mut StableHasher) {
         // do nothing
     }
 }
 
-fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectModule {
+pub(crate) struct OngoingCodegen {
+    modules: Vec<OngoingModuleCodegen>,
+    allocator_module: Option<CompiledModule>,
+    metadata_module: Option<CompiledModule>,
+    metadata: EncodedMetadata,
+    crate_info: CrateInfo,
+    concurrency_limiter: ConcurrencyLimiter,
+}
+
+impl OngoingCodegen {
+    pub(crate) fn join(
+        self,
+        sess: &Session,
+        backend_config: &BackendConfig,
+    ) -> (CodegenResults, FxHashMap<WorkProductId, WorkProduct>) {
+        let mut work_products = FxHashMap::default();
+        let mut modules = vec![];
+
+        for module_codegen in self.modules {
+            let module_codegen_result = match module_codegen {
+                OngoingModuleCodegen::Sync(module_codegen_result) => module_codegen_result,
+                OngoingModuleCodegen::Async(join_handle) => match join_handle.join() {
+                    Ok(module_codegen_result) => module_codegen_result,
+                    Err(panic) => std::panic::resume_unwind(panic),
+                },
+            };
+
+            let module_codegen_result = match module_codegen_result {
+                Ok(module_codegen_result) => module_codegen_result,
+                Err(err) => sess.fatal(&err),
+            };
+            let ModuleCodegenResult { module_regular, module_global_asm, existing_work_product } =
+                module_codegen_result;
+
+            if let Some((work_product_id, work_product)) = existing_work_product {
+                work_products.insert(work_product_id, work_product);
+            } else {
+                let work_product = if backend_config.disable_incr_cache {
+                    None
+                } else if let Some(module_global_asm) = &module_global_asm {
+                    rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
+                        sess,
+                        &module_regular.name,
+                        &[
+                            ("o", &module_regular.object.as_ref().unwrap()),
+                            ("asm.o", &module_global_asm.object.as_ref().unwrap()),
+                        ],
+                    )
+                } else {
+                    rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
+                        sess,
+                        &module_regular.name,
+                        &[("o", &module_regular.object.as_ref().unwrap())],
+                    )
+                };
+                if let Some((work_product_id, work_product)) = work_product {
+                    work_products.insert(work_product_id, work_product);
+                }
+            }
+
+            modules.push(module_regular);
+            if let Some(module_global_asm) = module_global_asm {
+                modules.push(module_global_asm);
+            }
+        }
+
+        drop(self.concurrency_limiter);
+
+        (
+            CodegenResults {
+                modules,
+                allocator_module: self.allocator_module,
+                metadata_module: self.metadata_module,
+                metadata: self.metadata,
+                crate_info: self.crate_info,
+            },
+            work_products,
+        )
+    }
+}
+
+fn make_module(sess: &Session, backend_config: &BackendConfig, name: String) -> ObjectModule {
+    let isa = crate::build_isa(sess, backend_config);
+
     let mut builder =
         ObjectBuilder::new(isa, name + ".o", cranelift_module::default_libcall_names()).unwrap();
     // Unlike cg_llvm, cg_clif defaults to disabling -Zfunction-sections. For cg_llvm binary size
@@ -37,15 +133,15 @@ fn make_module(sess: &Session, isa: Box<dyn TargetIsa>, name: String) -> ObjectM
     ObjectModule::new(builder)
 }
 
-fn emit_module(
-    tcx: TyCtxt<'_>,
-    backend_config: &BackendConfig,
+fn emit_cgu(
+    output_filenames: &OutputFilenames,
+    prof: &SelfProfilerRef,
     name: String,
-    kind: ModuleKind,
     module: ObjectModule,
-    debug: Option<DebugContext<'_>>,
+    debug: Option<DebugContext>,
     unwind_context: UnwindContext,
-) -> ModuleCodegenResult {
+    global_asm_object_file: Option<PathBuf>,
+) -> Result<ModuleCodegenResult, String> {
     let mut product = module.finish();
 
     if let Some(mut debug) = debug {
@@ -54,134 +150,191 @@ fn emit_module(
 
     unwind_context.emit(&mut product);
 
-    let tmp_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(&name));
-    let obj = product.object.write().unwrap();
+    let module_regular =
+        emit_module(output_filenames, prof, product.object, ModuleKind::Regular, name.clone())?;
+
+    Ok(ModuleCodegenResult {
+        module_regular,
+        module_global_asm: global_asm_object_file.map(|global_asm_object_file| CompiledModule {
+            name: format!("{name}.asm"),
+            kind: ModuleKind::Regular,
+            object: Some(global_asm_object_file),
+            dwarf_object: None,
+            bytecode: None,
+        }),
+        existing_work_product: None,
+    })
+}
 
-    tcx.sess.prof.artifact_size("object_file", name.clone(), obj.len().try_into().unwrap());
+fn emit_module(
+    output_filenames: &OutputFilenames,
+    prof: &SelfProfilerRef,
+    object: cranelift_object::object::write::Object<'_>,
+    kind: ModuleKind,
+    name: String,
+) -> Result<CompiledModule, String> {
+    let tmp_file = output_filenames.temp_path(OutputType::Object, Some(&name));
+    let mut file = match File::create(&tmp_file) {
+        Ok(file) => file,
+        Err(err) => return Err(format!("error creating object file: {}", err)),
+    };
 
-    if let Err(err) = std::fs::write(&tmp_file, obj) {
-        tcx.sess.fatal(&format!("error writing object file: {}", err));
+    if let Err(err) = object.write_stream(&mut file) {
+        return Err(format!("error writing object file: {}", err));
     }
 
-    let work_product = if backend_config.disable_incr_cache {
-        None
-    } else {
-        rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
-            tcx.sess,
-            &name,
-            &[("o", &tmp_file)],
-        )
-    };
+    prof.artifact_size("object_file", &*name, file.metadata().unwrap().len());
 
-    ModuleCodegenResult(
-        CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None },
-        work_product,
-    )
+    Ok(CompiledModule { name, kind, object: Some(tmp_file), dwarf_object: None, bytecode: None })
 }
 
 fn reuse_workproduct_for_cgu(
     tcx: TyCtxt<'_>,
     cgu: &CodegenUnit<'_>,
-    work_products: &mut FxHashMap<WorkProductId, WorkProduct>,
-) -> CompiledModule {
+) -> Result<ModuleCodegenResult, String> {
     let work_product = cgu.previous_work_product(tcx);
-    let obj_out = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str()));
-    let source_file = rustc_incremental::in_incr_comp_dir_sess(
+    let obj_out_regular =
+        tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str()));
+    let source_file_regular = rustc_incremental::in_incr_comp_dir_sess(
         &tcx.sess,
         &work_product.saved_files.get("o").expect("no saved object file in work product"),
     );
-    if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) {
-        tcx.sess.err(&format!(
+
+    if let Err(err) = rustc_fs_util::link_or_copy(&source_file_regular, &obj_out_regular) {
+        return Err(format!(
             "unable to copy {} to {}: {}",
-            source_file.display(),
-            obj_out.display(),
+            source_file_regular.display(),
+            obj_out_regular.display(),
             err
         ));
     }
+    let obj_out_global_asm =
+        crate::global_asm::add_file_stem_postfix(obj_out_regular.clone(), ".asm");
+    let has_global_asm = if let Some(asm_o) = work_product.saved_files.get("asm.o") {
+        let source_file_global_asm = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, asm_o);
+        if let Err(err) = rustc_fs_util::link_or_copy(&source_file_global_asm, &obj_out_global_asm)
+        {
+            return Err(format!(
+                "unable to copy {} to {}: {}",
+                source_file_regular.display(),
+                obj_out_regular.display(),
+                err
+            ));
+        }
+        true
+    } else {
+        false
+    };
 
-    work_products.insert(cgu.work_product_id(), work_product);
-
-    CompiledModule {
-        name: cgu.name().to_string(),
-        kind: ModuleKind::Regular,
-        object: Some(obj_out),
-        dwarf_object: None,
-        bytecode: None,
-    }
+    Ok(ModuleCodegenResult {
+        module_regular: CompiledModule {
+            name: cgu.name().to_string(),
+            kind: ModuleKind::Regular,
+            object: Some(obj_out_regular),
+            dwarf_object: None,
+            bytecode: None,
+        },
+        module_global_asm: if has_global_asm {
+            Some(CompiledModule {
+                name: cgu.name().to_string(),
+                kind: ModuleKind::Regular,
+                object: Some(obj_out_global_asm),
+                dwarf_object: None,
+                bytecode: None,
+            })
+        } else {
+            None
+        },
+        existing_work_product: Some((cgu.work_product_id(), work_product)),
+    })
 }
 
 fn module_codegen(
     tcx: TyCtxt<'_>,
-    (backend_config, cgu_name): (BackendConfig, rustc_span::Symbol),
-) -> ModuleCodegenResult {
-    let cgu = tcx.codegen_unit(cgu_name);
-    let mono_items = cgu.items_in_deterministic_order(tcx);
-
-    let isa = crate::build_isa(tcx.sess, &backend_config);
-    let mut module = make_module(tcx.sess, isa, cgu_name.as_str().to_string());
-
-    let mut cx = crate::CodegenCx::new(
-        tcx,
-        backend_config.clone(),
-        module.isa(),
-        tcx.sess.opts.debuginfo != DebugInfo::None,
-        cgu_name,
-    );
-    super::predefine_mono_items(tcx, &mut module, &mono_items);
-    for (mono_item, _) in mono_items {
-        match mono_item {
-            MonoItem::Fn(inst) => {
-                cx.tcx
-                    .sess
-                    .time("codegen fn", || crate::base::codegen_fn(&mut cx, &mut module, inst));
-            }
-            MonoItem::Static(def_id) => crate::constant::codegen_static(tcx, &mut module, def_id),
-            MonoItem::GlobalAsm(item_id) => {
-                let item = cx.tcx.hir().item(item_id);
-                if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind {
-                    if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
-                        cx.global_asm.push_str("\n.intel_syntax noprefix\n");
-                    } else {
-                        cx.global_asm.push_str("\n.att_syntax\n");
-                    }
-                    for piece in asm.template {
-                        match *piece {
-                            InlineAsmTemplatePiece::String(ref s) => cx.global_asm.push_str(s),
-                            InlineAsmTemplatePiece::Placeholder { .. } => todo!(),
-                        }
-                    }
-                    cx.global_asm.push_str("\n.att_syntax\n\n");
-                } else {
-                    bug!("Expected GlobalAsm found {:?}", item);
+    (backend_config, global_asm_config, cgu_name, token): (
+        BackendConfig,
+        Arc<GlobalAsmConfig>,
+        rustc_span::Symbol,
+        ConcurrencyLimiterToken,
+    ),
+) -> OngoingModuleCodegen {
+    let (cgu_name, mut cx, mut module, codegened_functions) = tcx.sess.time("codegen cgu", || {
+        let cgu = tcx.codegen_unit(cgu_name);
+        let mono_items = cgu.items_in_deterministic_order(tcx);
+
+        let mut module = make_module(tcx.sess, &backend_config, cgu_name.as_str().to_string());
+
+        let mut cx = crate::CodegenCx::new(
+            tcx,
+            backend_config.clone(),
+            module.isa(),
+            tcx.sess.opts.debuginfo != DebugInfo::None,
+            cgu_name,
+        );
+        super::predefine_mono_items(tcx, &mut module, &mono_items);
+        let mut codegened_functions = vec![];
+        for (mono_item, _) in mono_items {
+            match mono_item {
+                MonoItem::Fn(inst) => {
+                    tcx.sess.time("codegen fn", || {
+                        let codegened_function = crate::base::codegen_fn(
+                            tcx,
+                            &mut cx,
+                            Function::new(),
+                            &mut module,
+                            inst,
+                        );
+                        codegened_functions.push(codegened_function);
+                    });
+                }
+                MonoItem::Static(def_id) => {
+                    crate::constant::codegen_static(tcx, &mut module, def_id)
+                }
+                MonoItem::GlobalAsm(item_id) => {
+                    crate::global_asm::codegen_global_asm_item(tcx, &mut cx.global_asm, item_id);
                 }
             }
         }
-    }
-    crate::main_shim::maybe_create_entry_wrapper(
-        tcx,
-        &mut module,
-        &mut cx.unwind_context,
-        false,
-        cgu.is_primary(),
-    );
-
-    let debug_context = cx.debug_context;
-    let unwind_context = cx.unwind_context;
-    let codegen_result = tcx.sess.time("write object file", || {
-        emit_module(
+        crate::main_shim::maybe_create_entry_wrapper(
             tcx,
-            &backend_config,
-            cgu.name().as_str().to_string(),
-            ModuleKind::Regular,
-            module,
-            debug_context,
-            unwind_context,
-        )
+            &mut module,
+            &mut cx.unwind_context,
+            false,
+            cgu.is_primary(),
+        );
+
+        let cgu_name = cgu.name().as_str().to_owned();
+
+        (cgu_name, cx, module, codegened_functions)
     });
 
-    codegen_global_asm(tcx, cgu.name().as_str(), &cx.global_asm);
+    OngoingModuleCodegen::Async(std::thread::spawn(move || {
+        cx.profiler.clone().verbose_generic_activity("compile functions").run(|| {
+            let mut cached_context = Context::new();
+            for codegened_func in codegened_functions {
+                crate::base::compile_fn(&mut cx, &mut cached_context, &mut module, codegened_func);
+            }
+        });
 
-    codegen_result
+        let global_asm_object_file =
+            cx.profiler.verbose_generic_activity("compile assembly").run(|| {
+                crate::global_asm::compile_global_asm(&global_asm_config, &cgu_name, &cx.global_asm)
+            })?;
+
+        let codegen_result = cx.profiler.verbose_generic_activity("write object file").run(|| {
+            emit_cgu(
+                &global_asm_config.output_filenames,
+                &cx.profiler,
+                cgu_name,
+                module,
+                cx.debug_context,
+                cx.unwind_context,
+                global_asm_object_file,
+            )
+        });
+        std::mem::drop(token);
+        codegen_result
+    }))
 }
 
 pub(crate) fn run_aot(
@@ -189,9 +342,7 @@ pub(crate) fn run_aot(
     backend_config: BackendConfig,
     metadata: EncodedMetadata,
     need_metadata_module: bool,
-) -> Box<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)> {
-    let mut work_products = FxHashMap::default();
-
+) -> Box<OngoingCodegen> {
     let cgus = if tcx.sess.opts.output_types.should_codegen() {
         tcx.collect_and_partition_mono_items(()).1
     } else {
@@ -206,62 +357,69 @@ pub(crate) fn run_aot(
         }
     }
 
+    let global_asm_config = Arc::new(crate::global_asm::GlobalAsmConfig::new(tcx));
+
+    let mut concurrency_limiter = ConcurrencyLimiter::new(tcx.sess, cgus.len());
+
     let modules = super::time(tcx, backend_config.display_cg_time, "codegen mono items", || {
         cgus.iter()
             .map(|cgu| {
-                let cgu_reuse = determine_cgu_reuse(tcx, cgu);
+                let cgu_reuse = if backend_config.disable_incr_cache {
+                    CguReuse::No
+                } else {
+                    determine_cgu_reuse(tcx, cgu)
+                };
                 tcx.sess.cgu_reuse_tracker.set_actual_reuse(cgu.name().as_str(), cgu_reuse);
 
                 match cgu_reuse {
-                    _ if backend_config.disable_incr_cache => {}
-                    CguReuse::No => {}
-                    CguReuse::PreLto => {
-                        return reuse_workproduct_for_cgu(tcx, &*cgu, &mut work_products);
+                    CguReuse::No => {
+                        let dep_node = cgu.codegen_dep_node(tcx);
+                        tcx.dep_graph
+                            .with_task(
+                                dep_node,
+                                tcx,
+                                (
+                                    backend_config.clone(),
+                                    global_asm_config.clone(),
+                                    cgu.name(),
+                                    concurrency_limiter.acquire(),
+                                ),
+                                module_codegen,
+                                Some(rustc_middle::dep_graph::hash_result),
+                            )
+                            .0
+                    }
+                    CguReuse::PreLto => unreachable!(),
+                    CguReuse::PostLto => {
+                        concurrency_limiter.job_already_done();
+                        OngoingModuleCodegen::Sync(reuse_workproduct_for_cgu(tcx, &*cgu))
                     }
-                    CguReuse::PostLto => unreachable!(),
-                }
-
-                let dep_node = cgu.codegen_dep_node(tcx);
-                let (ModuleCodegenResult(module, work_product), _) = tcx.dep_graph.with_task(
-                    dep_node,
-                    tcx,
-                    (backend_config.clone(), cgu.name()),
-                    module_codegen,
-                    Some(rustc_middle::dep_graph::hash_result),
-                );
-
-                if let Some((id, product)) = work_product {
-                    work_products.insert(id, product);
                 }
-
-                module
             })
             .collect::<Vec<_>>()
     });
 
     tcx.sess.abort_if_errors();
 
-    let isa = crate::build_isa(tcx.sess, &backend_config);
-    let mut allocator_module = make_module(tcx.sess, isa, "allocator_shim".to_string());
-    assert_eq!(pointer_ty(tcx), allocator_module.target_config().pointer_type());
+    let mut allocator_module = make_module(tcx.sess, &backend_config, "allocator_shim".to_string());
     let mut allocator_unwind_context = UnwindContext::new(allocator_module.isa(), true);
     let created_alloc_shim =
         crate::allocator::codegen(tcx, &mut allocator_module, &mut allocator_unwind_context);
 
     let allocator_module = if created_alloc_shim {
-        let ModuleCodegenResult(module, work_product) = emit_module(
-            tcx,
-            &backend_config,
-            "allocator_shim".to_string(),
+        let mut product = allocator_module.finish();
+        allocator_unwind_context.emit(&mut product);
+
+        match emit_module(
+            tcx.output_filenames(()),
+            &tcx.sess.prof,
+            product.object,
             ModuleKind::Allocator,
-            allocator_module,
-            None,
-            allocator_unwind_context,
-        );
-        if let Some((id, product)) = work_product {
-            work_products.insert(id, product);
+            "allocator_shim".to_owned(),
+        ) {
+            Ok(allocator_module) => Some(allocator_module),
+            Err(err) => tcx.sess.fatal(err),
         }
-        Some(module)
     } else {
         None
     };
@@ -308,102 +466,14 @@ pub(crate) fn run_aot(
     }
     .to_owned();
 
-    Box::new((
-        CodegenResults {
-            modules,
-            allocator_module,
-            metadata_module,
-            metadata,
-            crate_info: CrateInfo::new(tcx, target_cpu),
-        },
-        work_products,
-    ))
-}
-
-fn codegen_global_asm(tcx: TyCtxt<'_>, cgu_name: &str, global_asm: &str) {
-    use std::io::Write;
-    use std::process::{Command, Stdio};
-
-    if global_asm.is_empty() {
-        return;
-    }
-
-    if cfg!(not(feature = "inline_asm"))
-        || tcx.sess.target.is_like_osx
-        || tcx.sess.target.is_like_windows
-    {
-        if global_asm.contains("__rust_probestack") {
-            return;
-        }
-
-        // FIXME fix linker error on macOS
-        if cfg!(not(feature = "inline_asm")) {
-            tcx.sess.fatal(
-                "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift",
-            );
-        } else {
-            tcx.sess.fatal("asm! and global_asm! are not yet supported on macOS and Windows");
-        }
-    }
-
-    let assembler = crate::toolchain::get_toolchain_binary(tcx.sess, "as");
-    let linker = crate::toolchain::get_toolchain_binary(tcx.sess, "ld");
-
-    // Remove all LLVM style comments
-    let global_asm = global_asm
-        .lines()
-        .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
-        .collect::<Vec<_>>()
-        .join("\n");
-
-    let output_object_file = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu_name));
-
-    // Assemble `global_asm`
-    let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
-    let mut child = Command::new(assembler)
-        .arg("-o")
-        .arg(&global_asm_object_file)
-        .stdin(Stdio::piped())
-        .spawn()
-        .expect("Failed to spawn `as`.");
-    child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
-    let status = child.wait().expect("Failed to wait for `as`.");
-    if !status.success() {
-        tcx.sess.fatal(&format!("Failed to assemble `{}`", global_asm));
-    }
-
-    // Link the global asm and main object file together
-    let main_object_file = add_file_stem_postfix(output_object_file.clone(), ".main");
-    std::fs::rename(&output_object_file, &main_object_file).unwrap();
-    let status = Command::new(linker)
-        .arg("-r") // Create a new object file
-        .arg("-o")
-        .arg(output_object_file)
-        .arg(&main_object_file)
-        .arg(&global_asm_object_file)
-        .status()
-        .unwrap();
-    if !status.success() {
-        tcx.sess.fatal(&format!(
-            "Failed to link `{}` and `{}` together",
-            main_object_file.display(),
-            global_asm_object_file.display(),
-        ));
-    }
-
-    std::fs::remove_file(global_asm_object_file).unwrap();
-    std::fs::remove_file(main_object_file).unwrap();
-}
-
-fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf {
-    let mut new_filename = path.file_stem().unwrap().to_owned();
-    new_filename.push(postfix);
-    if let Some(extension) = path.extension() {
-        new_filename.push(".");
-        new_filename.push(extension);
-    }
-    path.set_file_name(new_filename);
-    path
+    Box::new(OngoingCodegen {
+        modules,
+        allocator_module,
+        metadata_module,
+        metadata,
+        crate_info: CrateInfo::new(tcx, target_cpu),
+        concurrency_limiter,
+    })
 }
 
 // Adapted from https://github.com/rust-lang/rust/blob/303d8aff6092709edd4dbd35b1c88e9aa40bf6d8/src/librustc_codegen_ssa/base.rs#L922-L953
@@ -432,5 +502,5 @@ fn determine_cgu_reuse<'tcx>(tcx: TyCtxt<'tcx>, cgu: &CodegenUnit<'tcx>) -> CguR
         cgu.name()
     );
 
-    if tcx.try_mark_green(&dep_node) { CguReuse::PreLto } else { CguReuse::No }
+    if tcx.try_mark_green(&dep_node) { CguReuse::PostLto } else { CguReuse::No }
 }
diff --git a/compiler/rustc_codegen_cranelift/src/driver/jit.rs b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
index a56a91000..0e77e4004 100644
--- a/compiler/rustc_codegen_cranelift/src/driver/jit.rs
+++ b/compiler/rustc_codegen_cranelift/src/driver/jit.rs
@@ -61,11 +61,11 @@ impl UnsafeMessage {
     }
 }
 
-fn create_jit_module<'tcx>(
-    tcx: TyCtxt<'tcx>,
+fn create_jit_module(
+    tcx: TyCtxt<'_>,
     backend_config: &BackendConfig,
     hotswap: bool,
-) -> (JITModule, CodegenCx<'tcx>) {
+) -> (JITModule, CodegenCx) {
     let crate_info = CrateInfo::new(tcx, "dummy_target_cpu".to_string());
     let imported_symbols = load_imported_symbols_for_jit(tcx.sess, crate_info);
 
@@ -111,6 +111,7 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
         &backend_config,
         matches!(backend_config.codegen_mode, CodegenMode::JitLazy),
     );
+    let mut cached_context = Context::new();
 
     let (_, cgus) = tcx.collect_and_partition_mono_items(());
     let mono_items = cgus
@@ -128,11 +129,19 @@ pub(crate) fn run_jit(tcx: TyCtxt<'_>, backend_config: BackendConfig) -> ! {
                 MonoItem::Fn(inst) => match backend_config.codegen_mode {
                     CodegenMode::Aot => unreachable!(),
                     CodegenMode::Jit => {
-                        cx.tcx.sess.time("codegen fn", || {
-                            crate::base::codegen_fn(&mut cx, &mut jit_module, inst)
+                        tcx.sess.time("codegen fn", || {
+                            crate::base::codegen_and_compile_fn(
+                                tcx,
+                                &mut cx,
+                                &mut cached_context,
+                                &mut jit_module,
+                                inst,
+                            )
                         });
                     }
-                    CodegenMode::JitLazy => codegen_shim(&mut cx, &mut jit_module, inst),
+                    CodegenMode::JitLazy => {
+                        codegen_shim(tcx, &mut cx, &mut cached_context, &mut jit_module, inst)
+                    }
                 },
                 MonoItem::Static(def_id) => {
                     crate::constant::codegen_static(tcx, &mut jit_module, def_id);
@@ -259,7 +268,15 @@ fn jit_fn(instance_ptr: *const Instance<'static>, trampoline_ptr: *const u8) ->
                 false,
                 Symbol::intern("dummy_cgu_name"),
             );
-            tcx.sess.time("codegen fn", || crate::base::codegen_fn(&mut cx, jit_module, instance));
+            tcx.sess.time("codegen fn", || {
+                crate::base::codegen_and_compile_fn(
+                    tcx,
+                    &mut cx,
+                    &mut Context::new(),
+                    jit_module,
+                    instance,
+                )
+            });
 
             assert!(cx.global_asm.is_empty());
             jit_module.finalize_definitions();
@@ -334,9 +351,13 @@ fn load_imported_symbols_for_jit(
     imported_symbols
 }
 
-fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: Instance<'tcx>) {
-    let tcx = cx.tcx;
-
+fn codegen_shim<'tcx>(
+    tcx: TyCtxt<'tcx>,
+    cx: &mut CodegenCx,
+    cached_context: &mut Context,
+    module: &mut JITModule,
+    inst: Instance<'tcx>,
+) {
     let pointer_type = module.target_config().pointer_type();
 
     let name = tcx.symbol_name(inst).name;
@@ -357,8 +378,9 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
         )
         .unwrap();
 
-    cx.cached_context.clear();
-    let trampoline = &mut cx.cached_context.func;
+    let context = cached_context;
+    context.clear();
+    let trampoline = &mut context.func;
     trampoline.signature = sig.clone();
 
     let mut builder_ctx = FunctionBuilderContext::new();
@@ -381,5 +403,6 @@ fn codegen_shim<'tcx>(cx: &mut CodegenCx<'tcx>, module: &mut JITModule, inst: In
     let ret_vals = trampoline_builder.func.dfg.inst_results(call_inst).to_vec();
     trampoline_builder.ins().return_(&ret_vals);
 
-    module.define_function(func_id, &mut cx.cached_context).unwrap();
+    module.define_function(func_id, context).unwrap();
+    cx.unwind_context.add_function(func_id, context, module.isa());
 }
diff --git a/compiler/rustc_codegen_cranelift/src/global_asm.rs b/compiler/rustc_codegen_cranelift/src/global_asm.rs
new file mode 100644
index 000000000..dcbcaba30
--- /dev/null
+++ b/compiler/rustc_codegen_cranelift/src/global_asm.rs
@@ -0,0 +1,114 @@
+//! The AOT driver uses [`cranelift_object`] to write object files suitable for linking into a
+//! standalone executable.
+
+use std::io::Write;
+use std::path::PathBuf;
+use std::process::{Command, Stdio};
+use std::sync::Arc;
+
+use rustc_ast::{InlineAsmOptions, InlineAsmTemplatePiece};
+use rustc_hir::ItemId;
+use rustc_session::config::{OutputFilenames, OutputType};
+
+use crate::prelude::*;
+
+pub(crate) fn codegen_global_asm_item(tcx: TyCtxt<'_>, global_asm: &mut String, item_id: ItemId) {
+    let item = tcx.hir().item(item_id);
+    if let rustc_hir::ItemKind::GlobalAsm(asm) = item.kind {
+        if !asm.options.contains(InlineAsmOptions::ATT_SYNTAX) {
+            global_asm.push_str("\n.intel_syntax noprefix\n");
+        } else {
+            global_asm.push_str("\n.att_syntax\n");
+        }
+        for piece in asm.template {
+            match *piece {
+                InlineAsmTemplatePiece::String(ref s) => global_asm.push_str(s),
+                InlineAsmTemplatePiece::Placeholder { .. } => todo!(),
+            }
+        }
+        global_asm.push_str("\n.att_syntax\n\n");
+    } else {
+        bug!("Expected GlobalAsm found {:?}", item);
+    }
+}
+
+#[derive(Debug)]
+pub(crate) struct GlobalAsmConfig {
+    asm_enabled: bool,
+    assembler: PathBuf,
+    pub(crate) output_filenames: Arc<OutputFilenames>,
+}
+
+impl GlobalAsmConfig {
+    pub(crate) fn new(tcx: TyCtxt<'_>) -> Self {
+        let asm_enabled = cfg!(feature = "inline_asm") && !tcx.sess.target.is_like_windows;
+
+        GlobalAsmConfig {
+            asm_enabled,
+            assembler: crate::toolchain::get_toolchain_binary(tcx.sess, "as"),
+            output_filenames: tcx.output_filenames(()).clone(),
+        }
+    }
+}
+
+pub(crate) fn compile_global_asm(
+    config: &GlobalAsmConfig,
+    cgu_name: &str,
+    global_asm: &str,
+) -> Result<Option<PathBuf>, String> {
+    if global_asm.is_empty() {
+        return Ok(None);
+    }
+
+    if !config.asm_enabled {
+        if global_asm.contains("__rust_probestack") {
+            return Ok(None);
+        }
+
+        // FIXME fix linker error on macOS
+        if cfg!(not(feature = "inline_asm")) {
+            return Err(
+                "asm! and global_asm! support is disabled while compiling rustc_codegen_cranelift"
+                    .to_owned(),
+            );
+        } else {
+            return Err("asm! and global_asm! are not yet supported on Windows".to_owned());
+        }
+    }
+
+    // Remove all LLVM style comments
+    let global_asm = global_asm
+        .lines()
+        .map(|line| if let Some(index) = line.find("//") { &line[0..index] } else { line })
+        .collect::<Vec<_>>()
+        .join("\n");
+
+    let output_object_file = config.output_filenames.temp_path(OutputType::Object, Some(cgu_name));
+
+    // Assemble `global_asm`
+    let global_asm_object_file = add_file_stem_postfix(output_object_file.clone(), ".asm");
+    let mut child = Command::new(&config.assembler)
+        .arg("-o")
+        .arg(&global_asm_object_file)
+        .stdin(Stdio::piped())
+        .spawn()
+        .expect("Failed to spawn `as`.");
+    child.stdin.take().unwrap().write_all(global_asm.as_bytes()).unwrap();
+    let status = child.wait().expect("Failed to wait for `as`.");
+    if !status.success() {
+        return Err(format!("Failed to assemble `{}`", global_asm));
+    }
+
+    Ok(Some(global_asm_object_file))
+}
+
+pub(crate) fn add_file_stem_postfix(mut path: PathBuf, postfix: &str) -> PathBuf {
+    let mut new_filename = path.file_stem().unwrap().to_owned();
+    new_filename.push(postfix);
+    if let Some(extension) = path.extension() {
+        new_filename.push(".");
+        new_filename.push(extension);
+    }
+    path.set_file_name(new_filename);
+    path
+}
diff --git a/compiler/rustc_codegen_cranelift/src/inline_asm.rs b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
index 241de5e36..8b3d475cb 100644
--- a/compiler/rustc_codegen_cranelift/src/inline_asm.rs
+++ b/compiler/rustc_codegen_cranelift/src/inline_asm.rs
@@ -15,15 +15,19 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     template: &[InlineAsmTemplatePiece],
     operands: &[InlineAsmOperand<'tcx>],
     options: InlineAsmOptions,
+    destination: Option<mir::BasicBlock>,
 ) {
     // FIXME add .eh_frame unwind info directives
 
     if !template.is_empty() {
+        // Used by panic_abort
         if template[0] == InlineAsmTemplatePiece::String("int $$0x29".to_string()) {
-            let true_ = fx.bcx.ins().iconst(types::I32, 1);
-            fx.bcx.ins().trapnz(true_, TrapCode::User(1));
+            fx.bcx.ins().trap(TrapCode::User(1));
             return;
-        } else if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string())
+        }
+
+        // Used by stdarch
+        if template[0] == InlineAsmTemplatePiece::String("movq %rbx, ".to_string())
             && matches!(
                 template[1],
                 InlineAsmTemplatePiece::Placeholder {
@@ -47,51 +51,46 @@ pub(crate) fn codegen_inline_asm<'tcx>(
         {
             assert_eq!(operands.len(), 4);
             let (leaf, eax_place) = match operands[1] {
-                InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
-                    assert_eq!(
-                        reg,
-                        InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax))
-                    );
-                    (
-                        crate::base::codegen_operand(fx, in_value).load_scalar(fx),
-                        crate::base::codegen_place(fx, out_place.unwrap()),
-                    )
-                }
+                InlineAsmOperand::InOut {
+                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
+                    late: true,
+                    ref in_value,
+                    out_place: Some(out_place),
+                } => (
+                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
+                    crate::base::codegen_place(fx, out_place),
+                ),
                 _ => unreachable!(),
             };
             let ebx_place = match operands[0] {
-                InlineAsmOperand::Out { reg, late: true, place } => {
-                    assert_eq!(
-                        reg,
+                InlineAsmOperand::Out {
+                    reg:
                         InlineAsmRegOrRegClass::RegClass(InlineAsmRegClass::X86(
-                            X86InlineAsmRegClass::reg
-                        ))
-                    );
-                    crate::base::codegen_place(fx, place.unwrap())
-                }
+                            X86InlineAsmRegClass::reg,
+                        )),
+                    late: true,
+                    place: Some(place),
+                } => crate::base::codegen_place(fx, place),
                 _ => unreachable!(),
             };
             let (sub_leaf, ecx_place) = match operands[2] {
-                InlineAsmOperand::InOut { reg, late: true, ref in_value, out_place } => {
-                    assert_eq!(
-                        reg,
-                        InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx))
-                    );
-                    (
-                        crate::base::codegen_operand(fx, in_value).load_scalar(fx),
-                        crate::base::codegen_place(fx, out_place.unwrap()),
-                    )
-                }
+                InlineAsmOperand::InOut {
+                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
+                    late: true,
+                    ref in_value,
+                    out_place: Some(out_place),
+                } => (
+                    crate::base::codegen_operand(fx, in_value).load_scalar(fx),
+                    crate::base::codegen_place(fx, out_place),
+                ),
                 _ => unreachable!(),
             };
             let edx_place = match operands[3] {
-                InlineAsmOperand::Out { reg, late: true, place } => {
-                    assert_eq!(
-                        reg,
-                        InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx))
-                    );
-                    crate::base::codegen_place(fx, place.unwrap())
-                }
+                InlineAsmOperand::Out {
+                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
+                    late: true,
+                    place: Some(place),
+                } => crate::base::codegen_place(fx, place),
                 _ => unreachable!(),
             };
 
@@ -101,12 +100,99 @@ pub(crate) fn codegen_inline_asm<'tcx>(
             ebx_place.write_cvalue(fx, CValue::by_val(ebx, fx.layout_of(fx.tcx.types.u32)));
             ecx_place.write_cvalue(fx, CValue::by_val(ecx, fx.layout_of(fx.tcx.types.u32)));
             edx_place.write_cvalue(fx, CValue::by_val(edx, fx.layout_of(fx.tcx.types.u32)));
+            let destination_block = fx.get_block(destination.unwrap());
+            fx.bcx.ins().jump(destination_block, &[]);
             return;
-        } else if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
+        }
+
+        // Used by compiler-builtins
+        if fx.tcx.symbol_name(fx.instance).name.starts_with("___chkstk") {
             // ___chkstk, ___chkstk_ms and __alloca are only used on Windows
             crate::trap::trap_unimplemented(fx, "Stack probes are not supported");
+            return;
         } else if fx.tcx.symbol_name(fx.instance).name == "__alloca" {
             crate::trap::trap_unimplemented(fx, "Alloca is not supported");
+            return;
+        }
+
+        // Used by measureme
+        if template[0] == InlineAsmTemplatePiece::String("xor %eax, %eax".to_string())
+            && template[1] == InlineAsmTemplatePiece::String("\n".to_string())
+            && template[2] == InlineAsmTemplatePiece::String("mov %rbx, ".to_string())
+            && matches!(
+                template[3],
+                InlineAsmTemplatePiece::Placeholder {
+                    operand_idx: 0,
+                    modifier: Some('r'),
+                    span: _
+                }
+            )
+            && template[4] == InlineAsmTemplatePiece::String("\n".to_string())
+            && template[5] == InlineAsmTemplatePiece::String("cpuid".to_string())
+            && template[6] == InlineAsmTemplatePiece::String("\n".to_string())
+            && template[7] == InlineAsmTemplatePiece::String("mov ".to_string())
+            && matches!(
+                template[8],
+                InlineAsmTemplatePiece::Placeholder {
+                    operand_idx: 0,
+                    modifier: Some('r'),
+                    span: _
+                }
+            )
+            && template[9] == InlineAsmTemplatePiece::String(", %rbx".to_string())
+        {
+            let destination_block = fx.get_block(destination.unwrap());
+            fx.bcx.ins().jump(destination_block, &[]);
+            return;
+        } else if template[0] == InlineAsmTemplatePiece::String("rdpmc".to_string()) {
+            // Return zero dummy values for all performance counters
+            match operands[0] {
+                InlineAsmOperand::In {
+                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::cx)),
+                    value: _,
+                } => {}
+                _ => unreachable!(),
+            };
+            let lo = match operands[1] {
+                InlineAsmOperand::Out {
+                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::ax)),
+                    late: true,
+                    place: Some(place),
+                } => crate::base::codegen_place(fx, place),
+                _ => unreachable!(),
+            };
+            let hi = match operands[2] {
+                InlineAsmOperand::Out {
+                    reg: InlineAsmRegOrRegClass::Reg(InlineAsmReg::X86(X86InlineAsmReg::dx)),
+                    late: true,
+                    place: Some(place),
+                } => crate::base::codegen_place(fx, place),
+                _ => unreachable!(),
+            };
+
+            let u32_layout = fx.layout_of(fx.tcx.types.u32);
+            let zero = fx.bcx.ins().iconst(types::I32, 0);
+            lo.write_cvalue(fx, CValue::by_val(zero, u32_layout));
+            hi.write_cvalue(fx, CValue::by_val(zero, u32_layout));
+
+            let destination_block = fx.get_block(destination.unwrap());
+            fx.bcx.ins().jump(destination_block, &[]);
+            return;
+        } else if template[0] == InlineAsmTemplatePiece::String("lock xadd ".to_string())
+            && matches!(
+                template[1],
+                InlineAsmTemplatePiece::Placeholder { operand_idx: 1, modifier: None, span: _ }
+            )
+            && template[2] == InlineAsmTemplatePiece::String(", (".to_string())
+            && matches!(
+                template[3],
+                InlineAsmTemplatePiece::Placeholder { operand_idx: 0, modifier: None, span: _ }
+            )
+            && template[4] == InlineAsmTemplatePiece::String(")".to_string())
+        {
+            let destination_block = fx.get_block(destination.unwrap());
+            fx.bcx.ins().jump(destination_block, &[]);
+            return;
         }
     }
 
@@ -175,6 +261,16 @@ pub(crate) fn codegen_inline_asm<'tcx>(
     }
 
     call_inline_asm(fx, &asm_name, asm_gen.stack_slot_size, inputs, outputs);
+
+    match destination {
+        Some(destination) => {
+            let destination_block = fx.get_block(destination);
+            fx.bcx.ins().jump(destination_block, &[]);
+        }
+        None => {
+            fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
+        }
+    }
 }
 
 struct InlineAssemblyGenerator<'a, 'tcx> {
@@ -637,7 +733,7 @@ fn call_inline_asm<'tcx>(
     inputs: Vec<(Size, Value)>,
     outputs: Vec<(Size, CPlace<'tcx>)>,
 ) {
-    let stack_slot = fx.bcx.func.create_stack_slot(StackSlotData {
+    let stack_slot = fx.bcx.func.create_sized_stack_slot(StackSlotData {
         kind: StackSlotKind::ExplicitSlot,
         size: u32::try_from(slot_size.bytes()).unwrap(),
     });
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
index d02dfd93c..5120b89c4 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/cpuid.rs
@@ -62,7 +62,7 @@ pub(crate) fn codegen_cpuid_call<'tcx>(
     fx.bcx.ins().jump(dest, &[zero, zero, proc_info_ecx, proc_info_edx]);
 
     fx.bcx.switch_to_block(unsupported_leaf);
-    crate::trap::trap_unreachable(
+    crate::trap::trap_unimplemented(
         fx,
         "__cpuid_count arch intrinsic doesn't yet support specified leaf",
     );
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
index 869670c8c..a799dca93 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/llvm.rs
@@ -139,6 +139,7 @@ pub(crate) fn codegen_llvm_intrinsic_call<'tcx>(
                 .sess
                 .warn(&format!("unsupported llvm intrinsic {}; replacing with trap", intrinsic));
             crate::trap::trap_unimplemented(fx, intrinsic);
+            return;
         }
     }
 
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
index b2a83e1d4..2e4ca594f 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/mod.rs
@@ -44,7 +44,7 @@ fn report_atomic_type_validation_error<'tcx>(
         ),
     );
     // Prevent verifier error
-    crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+    fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
 }
 
 pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx>) -> Option<Type> {
@@ -53,7 +53,7 @@ pub(crate) fn clif_vector_type<'tcx>(tcx: TyCtxt<'tcx>, layout: TyAndLayout<'tcx
         _ => unreachable!(),
     };
 
-    match scalar_to_clif_type(tcx, element).by(u16::try_from(count).unwrap()) {
+    match scalar_to_clif_type(tcx, element).by(u32::try_from(count).unwrap()) {
         // Cranelift currently only implements icmp for 128bit vectors.
         Some(vector_ty) if vector_ty.bits() == 128 => Some(vector_ty),
         _ => None,
@@ -203,7 +203,7 @@ pub(crate) fn codegen_intrinsic_call<'tcx>(
             sym::transmute => {
                 crate::base::codegen_panic(fx, "Transmuting to uninhabited type.", source_info);
             }
-            _ => unimplemented!("unsupported instrinsic {}", intrinsic),
+            _ => unimplemented!("unsupported intrinsic {}", intrinsic),
         }
         return;
     };
@@ -301,7 +301,44 @@ fn codegen_float_intrinsic_call<'tcx>(
         _ => unreachable!(),
     };
 
-    let res = fx.easy_call(name, &args, ty);
+    let layout = fx.layout_of(ty);
+    let res = match intrinsic {
+        sym::fmaf32 | sym::fmaf64 => {
+            let a = args[0].load_scalar(fx);
+            let b = args[1].load_scalar(fx);
+            let c = args[2].load_scalar(fx);
+            CValue::by_val(fx.bcx.ins().fma(a, b, c), layout)
+        }
+        sym::copysignf32 | sym::copysignf64 => {
+            let a = args[0].load_scalar(fx);
+            let b = args[1].load_scalar(fx);
+            CValue::by_val(fx.bcx.ins().fcopysign(a, b), layout)
+        }
+        sym::fabsf32
+        | sym::fabsf64
+        | sym::floorf32
+        | sym::floorf64
+        | sym::ceilf32
+        | sym::ceilf64
+        | sym::truncf32
+        | sym::truncf64 => {
+            let a = args[0].load_scalar(fx);
+
+            let val = match intrinsic {
+                sym::fabsf32 | sym::fabsf64 => fx.bcx.ins().fabs(a),
+                sym::floorf32 | sym::floorf64 => fx.bcx.ins().floor(a),
+                sym::ceilf32 | sym::ceilf64 => fx.bcx.ins().ceil(a),
+                sym::truncf32 | sym::truncf64 => fx.bcx.ins().trunc(a),
+                _ => unreachable!(),
+            };
+
+            CValue::by_val(val, layout)
+        }
+        // These intrinsics aren't supported natively by Cranelift.
+        // Lower them to a libcall.
+        _ => fx.easy_call(name, &args, ty),
+    };
+
     ret.write_cvalue(fx, res);
 
     true
@@ -320,9 +357,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
     let usize_layout = fx.layout_of(fx.tcx.types.usize);
 
     match intrinsic {
-        sym::assume => {
-            intrinsic_args!(fx, args => (_a); intrinsic);
-        }
         sym::likely | sym::unlikely => {
             intrinsic_args!(fx, args => (a); intrinsic);
 
@@ -540,6 +574,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, CValue::by_val(res, base.layout()));
         }
 
+        sym::ptr_mask => {
+            intrinsic_args!(fx, args => (ptr, mask); intrinsic);
+            let ptr = ptr.load_scalar(fx);
+            let mask = mask.load_scalar(fx);
+            fx.bcx.ins().band(ptr, mask);
+        }
+
         sym::transmute => {
             intrinsic_args!(fx, args => (from); intrinsic);
 
@@ -775,20 +816,13 @@ fn codegen_regular_intrinsic_call<'tcx>(
             ret.write_cvalue(fx, val);
         }
 
-        sym::ptr_guaranteed_eq => {
+        sym::ptr_guaranteed_cmp => {
             intrinsic_args!(fx, args => (a, b); intrinsic);
 
             let val = crate::num::codegen_ptr_binop(fx, BinOp::Eq, a, b);
             ret.write_cvalue(fx, val);
         }
 
-        sym::ptr_guaranteed_ne => {
-            intrinsic_args!(fx, args => (a, b); intrinsic);
-
-            let val = crate::num::codegen_ptr_binop(fx, BinOp::Ne, a, b);
-            ret.write_cvalue(fx, val);
-        }
-
         sym::caller_location => {
             intrinsic_args!(fx, args => (); intrinsic);
 
@@ -818,8 +852,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
                         // special case for compiler-builtins to avoid having to patch it
                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
-                        let ret_block = fx.get_block(destination.unwrap());
-                        fx.bcx.ins().jump(ret_block, &[]);
                         return;
                     } else {
                         fx.tcx
@@ -851,8 +883,6 @@ fn codegen_regular_intrinsic_call<'tcx>(
                     if fx.tcx.is_compiler_builtins(LOCAL_CRATE) {
                         // special case for compiler-builtins to avoid having to patch it
                         crate::trap::trap_unimplemented(fx, "128bit atomics not yet supported");
-                        let ret_block = fx.get_block(destination.unwrap());
-                        fx.bcx.ins().jump(ret_block, &[]);
                         return;
                     } else {
                         fx.tcx
diff --git a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
index 30e3d1125..1f358b1bb 100644
--- a/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
+++ b/compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs
@@ -14,7 +14,7 @@ fn report_simd_type_validation_error(
 ) {
     fx.tcx.sess.span_err(span, &format!("invalid monomorphization of `{}` intrinsic: expected SIMD input type, found non-SIMD `{}`", intrinsic, ty));
     // Prevent verifier error
-    crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+    fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
 }
 
 pub(super) fn codegen_simd_intrinsic_call<'tcx>(
@@ -157,7 +157,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                             ),
                         );
                         // Prevent verifier error
-                        crate::trap::trap_unreachable(fx, "compilation should not have succeeded");
+                        fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
                         return;
                     }
                 }
@@ -186,7 +186,10 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                         let size = Size::from_bytes(
                             4 * ret_lane_count, /* size_of([u32; ret_lane_count]) */
                         );
-                        alloc.inner().get_bytes(fx, alloc_range(offset, size)).unwrap()
+                        alloc
+                            .inner()
+                            .get_bytes_strip_provenance(fx, alloc_range(offset, size))
+                            .unwrap()
                     }
                     _ => unreachable!("{:?}", idx_const),
                 };
@@ -274,12 +277,17 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 idx_const
             } else {
                 fx.tcx.sess.span_warn(span, "Index argument for `simd_extract` is not a constant");
-                let res = crate::trap::trap_unimplemented_ret_value(
+                let trap_block = fx.bcx.create_block();
+                let dummy_block = fx.bcx.create_block();
+                let true_ = fx.bcx.ins().iconst(types::I8, 1);
+                fx.bcx.ins().brnz(true_, trap_block, &[]);
+                fx.bcx.ins().jump(dummy_block, &[]);
+                fx.bcx.switch_to_block(trap_block);
+                crate::trap::trap_unimplemented(
                     fx,
-                    ret.layout(),
                     "Index argument for `simd_extract` is not a constant",
                 );
-                ret.write_cvalue(fx, res);
+                fx.bcx.switch_to_block(dummy_block);
                 return;
             };
 
@@ -392,21 +400,15 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
 
             let layout = a.layout();
             let (lane_count, lane_ty) = layout.ty.simd_size_and_type(fx.tcx);
+            let res_lane_layout = fx.layout_of(lane_ty);
 
             for lane in 0..lane_count {
-                let a_lane = a.value_lane(fx, lane);
-                let b_lane = b.value_lane(fx, lane);
-                let c_lane = c.value_lane(fx, lane);
+                let a_lane = a.value_lane(fx, lane).load_scalar(fx);
+                let b_lane = b.value_lane(fx, lane).load_scalar(fx);
+                let c_lane = c.value_lane(fx, lane).load_scalar(fx);
 
-                let res_lane = match lane_ty.kind() {
-                    ty::Float(FloatTy::F32) => {
-                        fx.easy_call("fmaf", &[a_lane, b_lane, c_lane], lane_ty)
-                    }
-                    ty::Float(FloatTy::F64) => {
-                        fx.easy_call("fma", &[a_lane, b_lane, c_lane], lane_ty)
-                    }
-                    _ => unreachable!(),
-                };
+                let res_lane = fx.bcx.ins().fma(a_lane, b_lane, c_lane);
+                let res_lane = CValue::by_val(res_lane, res_lane_layout);
 
                 ret.place_lane(fx, lane).write_cvalue(fx, res_lane);
             }
diff --git a/compiler/rustc_codegen_cranelift/src/lib.rs b/compiler/rustc_codegen_cranelift/src/lib.rs
index bb0793b1d..913414e76 100644
--- a/compiler/rustc_codegen_cranelift/src/lib.rs
+++ b/compiler/rustc_codegen_cranelift/src/lib.rs
@@ -4,6 +4,7 @@
 #![warn(unused_lifetimes)]
 #![warn(unreachable_pub)]
 
+extern crate jobserver;
 #[macro_use]
 extern crate rustc_middle;
 extern crate rustc_ast;
@@ -25,10 +26,12 @@ extern crate rustc_target;
 extern crate rustc_driver;
 
 use std::any::Any;
-use std::cell::Cell;
+use std::cell::{Cell, RefCell};
+use std::sync::Arc;
 
 use rustc_codegen_ssa::traits::CodegenBackend;
 use rustc_codegen_ssa::CodegenResults;
+use rustc_data_structures::profiling::SelfProfilerRef;
 use rustc_errors::ErrorGuaranteed;
 use rustc_metadata::EncodedMetadata;
 use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
@@ -51,11 +54,13 @@ mod cast;
 mod codegen_i128;
 mod common;
 mod compiler_builtins;
+mod concurrency_limiter;
 mod config;
 mod constant;
 mod debuginfo;
 mod discriminant;
 mod driver;
+mod global_asm;
 mod inline_asm;
 mod intrinsics;
 mod linkage;
@@ -119,19 +124,20 @@ impl<F: Fn() -> String> Drop for PrintOnPanic<F> {
 
 /// The codegen context holds any information shared between the codegen of individual functions
 /// inside a single codegen unit with the exception of the Cranelift [`Module`](cranelift_module::Module).
-struct CodegenCx<'tcx> {
-    tcx: TyCtxt<'tcx>,
+struct CodegenCx {
+    profiler: SelfProfilerRef,
+    output_filenames: Arc<OutputFilenames>,
+    should_write_ir: bool,
     global_asm: String,
     inline_asm_index: Cell<usize>,
-    cached_context: Context,
-    debug_context: Option<DebugContext<'tcx>>,
+    debug_context: Option<DebugContext>,
     unwind_context: UnwindContext,
     cgu_name: Symbol,
 }
 
-impl<'tcx> CodegenCx<'tcx> {
+impl CodegenCx {
     fn new(
-        tcx: TyCtxt<'tcx>,
+        tcx: TyCtxt<'_>,
         backend_config: BackendConfig,
         isa: &dyn TargetIsa,
         debug_info: bool,
@@ -147,10 +153,11 @@ impl<'tcx> CodegenCx<'tcx> {
             None
         };
         CodegenCx {
-            tcx,
+            profiler: tcx.prof.clone(),
+            output_filenames: tcx.output_filenames(()).clone(),
+            should_write_ir: crate::pretty_clif::should_write_ir(tcx),
             global_asm: String::new(),
             inline_asm_index: Cell::new(0),
-            cached_context: Context::new(),
             debug_context,
             unwind_context,
             cgu_name,
@@ -159,7 +166,7 @@ impl<'tcx> CodegenCx<'tcx> {
 }
 
 pub struct CraneliftCodegenBackend {
-    pub config: Option<BackendConfig>,
+    pub config: RefCell<Option<BackendConfig>>,
 }
 
 impl CodegenBackend for CraneliftCodegenBackend {
@@ -169,6 +176,13 @@ impl CodegenBackend for CraneliftCodegenBackend {
             Lto::No | Lto::ThinLocal => {}
             Lto::Thin | Lto::Fat => sess.warn("LTO is not supported. You may get a linker error."),
         }
+
+        let mut config = self.config.borrow_mut();
+        if config.is_none() {
+            let new_config = BackendConfig::from_opts(&sess.opts.cg.llvm_args)
+                .unwrap_or_else(|err| sess.fatal(&err));
+            *config = Some(new_config);
+        }
     }
 
     fn target_features(&self, _sess: &Session, _allow_unstable: bool) -> Vec<rustc_span::Symbol> {
@@ -186,15 +200,7 @@ impl CodegenBackend for CraneliftCodegenBackend {
         need_metadata_module: bool,
     ) -> Box<dyn Any> {
         tcx.sess.abort_if_errors();
-        let config = if let Some(config) = self.config.clone() {
-            config
-        } else {
-            if !tcx.sess.unstable_options() && !tcx.sess.opts.cg.llvm_args.is_empty() {
-                tcx.sess.fatal("`-Z unstable-options` must be passed to allow configuring cg_clif");
-            }
-            BackendConfig::from_opts(&tcx.sess.opts.cg.llvm_args)
-                .unwrap_or_else(|err| tcx.sess.fatal(&err))
-        };
+        let config = self.config.borrow().clone().unwrap();
         match config.codegen_mode {
             CodegenMode::Aot => driver::aot::run_aot(tcx, config, metadata, need_metadata_module),
             CodegenMode::Jit | CodegenMode::JitLazy => {
@@ -210,12 +216,13 @@ impl CodegenBackend for CraneliftCodegenBackend {
     fn join_codegen(
         &self,
         ongoing_codegen: Box<dyn Any>,
-        _sess: &Session,
+        sess: &Session,
         _outputs: &OutputFilenames,
     ) -> Result<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>), ErrorGuaranteed> {
-        Ok(*ongoing_codegen
-            .downcast::<(CodegenResults, FxHashMap<WorkProductId, WorkProduct>)>()
-            .unwrap())
+        Ok(ongoing_codegen
+            .downcast::<driver::aot::OngoingCodegen>()
+            .unwrap()
+            .join(sess, self.config.borrow().as_ref().unwrap()))
     }
 
     fn link(
@@ -312,5 +319,5 @@ fn build_isa(sess: &Session, backend_config: &BackendConfig) -> Box<dyn isa::Tar
 /// This is the entrypoint for a hot plugged rustc_codegen_cranelift
 #[no_mangle]
 pub fn __rustc_codegen_backend() -> Box<dyn CodegenBackend> {
-    Box::new(CraneliftCodegenBackend { config: None })
+    Box::new(CraneliftCodegenBackend { config: RefCell::new(None) })
 }
diff --git a/compiler/rustc_codegen_cranelift/src/main_shim.rs b/compiler/rustc_codegen_cranelift/src/main_shim.rs
index c67b6e98b..3c024a84d 100644
--- a/compiler/rustc_codegen_cranelift/src/main_shim.rs
+++ b/compiler/rustc_codegen_cranelift/src/main_shim.rs
@@ -1,7 +1,7 @@
 use rustc_hir::LangItem;
 use rustc_middle::ty::subst::GenericArg;
 use rustc_middle::ty::AssocKind;
-use rustc_session::config::EntryFnType;
+use rustc_session::config::{sigpipe, EntryFnType};
 use rustc_span::symbol::Ident;
 
 use crate::prelude::*;
@@ -15,12 +15,12 @@ pub(crate) fn maybe_create_entry_wrapper(
     is_jit: bool,
     is_primary_cgu: bool,
 ) {
-    let (main_def_id, is_main_fn) = match tcx.entry_fn(()) {
+    let (main_def_id, (is_main_fn, sigpipe)) = match tcx.entry_fn(()) {
         Some((def_id, entry_ty)) => (
             def_id,
             match entry_ty {
-                EntryFnType::Main => true,
-                EntryFnType::Start => false,
+                EntryFnType::Main { sigpipe } => (true, sigpipe),
+                EntryFnType::Start => (false, sigpipe::DEFAULT),
             },
         ),
         None => return,
@@ -35,7 +35,7 @@ pub(crate) fn maybe_create_entry_wrapper(
         return;
     }
 
-    create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn);
+    create_entry_fn(tcx, module, unwind_context, main_def_id, is_jit, is_main_fn, sigpipe);
 
     fn create_entry_fn(
         tcx: TyCtxt<'_>,
@@ -44,6 +44,7 @@ pub(crate) fn maybe_create_entry_wrapper(
         rust_main_def_id: DefId,
         ignore_lang_start_wrapper: bool,
         is_main_fn: bool,
+        sigpipe: u8,
     ) {
         let main_ret_ty = tcx.fn_sig(rust_main_def_id).output();
         // Given that `main()` has no arguments,
@@ -83,6 +84,7 @@ pub(crate) fn maybe_create_entry_wrapper(
             bcx.switch_to_block(block);
             let arg_argc = bcx.append_block_param(block, m.target_config().pointer_type());
             let arg_argv = bcx.append_block_param(block, m.target_config().pointer_type());
+            let arg_sigpipe = bcx.ins().iconst(types::I8, sigpipe as i64);
 
             let main_func_ref = m.declare_func_in_func(main_func_id, &mut bcx.func);
 
@@ -143,7 +145,8 @@ pub(crate) fn maybe_create_entry_wrapper(
                 let main_val = bcx.ins().func_addr(m.target_config().pointer_type(), main_func_ref);
 
                 let func_ref = m.declare_func_in_func(start_func_id, &mut bcx.func);
-                let call_inst = bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv]);
+                let call_inst =
+                    bcx.ins().call(func_ref, &[main_val, arg_argc, arg_argv, arg_sigpipe]);
                 bcx.inst_results(call_inst)[0]
             } else {
                 // using user-defined start fn
diff --git a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
index d1f89adb3..0df7e8229 100644
--- a/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
+++ b/compiler/rustc_codegen_cranelift/src/optimize/mod.rs
@@ -1,20 +1,3 @@
 //! Various optimizations specific to cg_clif
 
-use cranelift_codegen::isa::TargetIsa;
-
-use crate::prelude::*;
-
 pub(crate) mod peephole;
-
-pub(crate) fn optimize_function<'tcx>(
-    tcx: TyCtxt<'tcx>,
-    isa: &dyn TargetIsa,
-    instance: Instance<'tcx>,
-    ctx: &mut Context,
-    clif_comments: &mut crate::pretty_clif::CommentWriter,
-) {
-    // FIXME classify optimizations over opt levels once we have more
-
-    crate::pretty_clif::write_clif_file(tcx, "preopt", isa, instance, &ctx.func, &*clif_comments);
-    crate::base::verify_func(tcx, &*clif_comments, &ctx.func);
-}
diff --git a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
index 1d1ec2168..a7af16268 100644
--- a/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
+++ b/compiler/rustc_codegen_cranelift/src/pretty_clif.rs
@@ -62,7 +62,7 @@ use cranelift_codegen::{
 };
 
 use rustc_middle::ty::layout::FnAbiOf;
-use rustc_session::config::OutputType;
+use rustc_session::config::{OutputFilenames, OutputType};
 
 use crate::prelude::*;
 
@@ -205,15 +205,11 @@ pub(crate) fn should_write_ir(tcx: TyCtxt<'_>) -> bool {
 }
 
 pub(crate) fn write_ir_file(
-    tcx: TyCtxt<'_>,
-    name: impl FnOnce() -> String,
+    output_filenames: &OutputFilenames,
+    name: &str,
     write: impl FnOnce(&mut dyn Write) -> std::io::Result<()>,
 ) {
-    if !should_write_ir(tcx) {
-        return;
-    }
-
-    let clif_output_dir = tcx.output_filenames(()).with_extension("clif");
+    let clif_output_dir = output_filenames.with_extension("clif");
 
     match std::fs::create_dir(&clif_output_dir) {
         Ok(()) => {}
@@ -221,44 +217,43 @@ pub(crate) fn write_ir_file(
         res @ Err(_) => res.unwrap(),
     }
 
-    let clif_file_name = clif_output_dir.join(name());
+    let clif_file_name = clif_output_dir.join(name);
 
     let res = std::fs::File::create(clif_file_name).and_then(|mut file| write(&mut file));
     if let Err(err) = res {
-        tcx.sess.warn(&format!("error writing ir file: {}", err));
+        // Using early_warn as no Session is available here
+        rustc_session::early_warn(
+            rustc_session::config::ErrorOutputType::default(),
+            &format!("error writing ir file: {}", err),
+        );
     }
 }
 
-pub(crate) fn write_clif_file<'tcx>(
-    tcx: TyCtxt<'tcx>,
+pub(crate) fn write_clif_file(
+    output_filenames: &OutputFilenames,
+    symbol_name: &str,
     postfix: &str,
     isa: &dyn cranelift_codegen::isa::TargetIsa,
-    instance: Instance<'tcx>,
     func: &cranelift_codegen::ir::Function,
     mut clif_comments: &CommentWriter,
 ) {
     // FIXME work around filename too long errors
-    write_ir_file(
-        tcx,
-        || format!("{}.{}.clif", tcx.symbol_name(instance).name, postfix),
-        |file| {
-            let mut clif = String::new();
-            cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func)
-                .unwrap();
+    write_ir_file(output_filenames, &format!("{}.{}.clif", symbol_name, postfix), |file| {
+        let mut clif = String::new();
+        cranelift_codegen::write::decorate_function(&mut clif_comments, &mut clif, func).unwrap();
 
-            for flag in isa.flags().iter() {
-                writeln!(file, "set {}", flag)?;
-            }
-            write!(file, "target {}", isa.triple().architecture.to_string())?;
-            for isa_flag in isa.isa_flags().iter() {
-                write!(file, " {}", isa_flag)?;
-            }
-            writeln!(file, "\n")?;
-            writeln!(file)?;
-            file.write_all(clif.as_bytes())?;
-            Ok(())
-        },
-    );
+        for flag in isa.flags().iter() {
+            writeln!(file, "set {}", flag)?;
+        }
+        write!(file, "target {}", isa.triple().architecture.to_string())?;
+        for isa_flag in isa.isa_flags().iter() {
+            write!(file, " {}", isa_flag)?;
+        }
+        writeln!(file, "\n")?;
+        writeln!(file)?;
+        file.write_all(clif.as_bytes())?;
+        Ok(())
+    });
 }
 
 impl fmt::Debug for FunctionCx<'_, '_, '_> {
diff --git a/compiler/rustc_codegen_cranelift/src/toolchain.rs b/compiler/rustc_codegen_cranelift/src/toolchain.rs
index f86236ef3..b6b465e1f 100644
--- a/compiler/rustc_codegen_cranelift/src/toolchain.rs
+++ b/compiler/rustc_codegen_cranelift/src/toolchain.rs
@@ -8,10 +8,8 @@ use rustc_session::Session;
 /// Tries to infer the path of a binary for the target toolchain from the linker name.
 pub(crate) fn get_toolchain_binary(sess: &Session, tool: &str) -> PathBuf {
     let (mut linker, _linker_flavor) = linker_and_flavor(sess);
-    let linker_file_name = linker
-        .file_name()
-        .and_then(|name| name.to_str())
-        .unwrap_or_else(|| sess.fatal("couldn't extract file name from specified linker"));
+    let linker_file_name =
+        linker.file_name().unwrap().to_str().expect("linker filename should be valid UTF-8");
 
     if linker_file_name == "ld.lld" {
         if tool != "ld" {
diff --git a/compiler/rustc_codegen_cranelift/src/trap.rs b/compiler/rustc_codegen_cranelift/src/trap.rs
index 923269c4d..82a2ec579 100644
--- a/compiler/rustc_codegen_cranelift/src/trap.rs
+++ b/compiler/rustc_codegen_cranelift/src/trap.rs
@@ -25,33 +25,10 @@ fn codegen_print(fx: &mut FunctionCx<'_, '_, '_>, msg: &str) {
     fx.bcx.ins().call(puts, &[msg_ptr]);
 }
 
-/// Use this for example when a function call should never return. This will fill the current block,
-/// so you can **not** add instructions to it afterwards.
-///
-/// Trap code: user65535
-pub(crate) fn trap_unreachable(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
-    codegen_print(fx, msg.as_ref());
-    fx.bcx.ins().trap(TrapCode::UnreachableCodeReached);
-}
 /// Use this when something is unimplemented, but `libcore` or `libstd` requires it to codegen.
-/// Unlike `trap_unreachable` this will not fill the current block, so you **must** add instructions
-/// to it afterwards.
 ///
 /// Trap code: user65535
 pub(crate) fn trap_unimplemented(fx: &mut FunctionCx<'_, '_, '_>, msg: impl AsRef<str>) {
     codegen_print(fx, msg.as_ref());
-    let true_ = fx.bcx.ins().iconst(types::I32, 1);
-    fx.bcx.ins().trapnz(true_, TrapCode::User(!0));
-}
-
-/// Like `trap_unimplemented` but returns a fake value of the specified type.
-///
-/// Trap code: user65535
-pub(crate) fn trap_unimplemented_ret_value<'tcx>(
-    fx: &mut FunctionCx<'_, '_, 'tcx>,
-    dest_layout: TyAndLayout<'tcx>,
-    msg: impl AsRef<str>,
-) -> CValue<'tcx> {
-    trap_unimplemented(fx, msg);
-    CValue::by_ref(Pointer::const_addr(fx, 0), dest_layout)
+    fx.bcx.ins().trap(TrapCode::User(!0));
 }
diff --git a/compiler/rustc_codegen_cranelift/src/unsize.rs b/compiler/rustc_codegen_cranelift/src/unsize.rs
index 052ca0a08..dd9d891dd 100644
--- a/compiler/rustc_codegen_cranelift/src/unsize.rs
+++ b/compiler/rustc_codegen_cranelift/src/unsize.rs
@@ -29,6 +29,7 @@ pub(crate) fn unsized_info<'tcx>(
             let old_info =
                 old_info.expect("unsized_info: missing old info for trait upcasting coercion");
             if data_a.principal_def_id() == data_b.principal_def_id() {
+                // A NOP cast that doesn't actually change anything, should be allowed even with invalid vtables.
                 return old_info;
             }
 
diff --git a/compiler/rustc_codegen_cranelift/src/value_and_place.rs b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
index 45ae2bd8f..cfaadca94 100644
--- a/compiler/rustc_codegen_cranelift/src/value_and_place.rs
+++ b/compiler/rustc_codegen_cranelift/src/value_and_place.rs
@@ -122,7 +122,7 @@ impl<'tcx> CValue<'tcx> {
                 let clif_ty = match layout.abi {
                     Abi::Scalar(scalar) => scalar_to_clif_type(fx.tcx, scalar),
                     Abi::Vector { element, count } => scalar_to_clif_type(fx.tcx, element)
-                        .by(u16::try_from(count).unwrap())
+                        .by(u32::try_from(count).unwrap())
                         .unwrap(),
                     _ => unreachable!("{:?}", layout.ty),
                 };
@@ -330,7 +330,7 @@ impl<'tcx> CPlace<'tcx> {
                 .fatal(&format!("values of type {} are too big to store on the stack", layout.ty));
         }
 
-        let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
+        let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
             kind: StackSlotKind::ExplicitSlot,
             // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
             // specify stack slot alignment.
@@ -472,7 +472,7 @@ impl<'tcx> CPlace<'tcx> {
                 }
                 _ if src_ty.is_vector() || dst_ty.is_vector() => {
                     // FIXME do something more efficient for transmutes between vectors and integers.
-                    let stack_slot = fx.bcx.create_stack_slot(StackSlotData {
+                    let stack_slot = fx.bcx.create_sized_stack_slot(StackSlotData {
                         kind: StackSlotKind::ExplicitSlot,
                         // FIXME Don't force the size to a multiple of 16 bytes once Cranelift gets a way to
                         // specify stack slot alignment.
@@ -519,7 +519,7 @@ impl<'tcx> CPlace<'tcx> {
                 if let ty::Array(element, len) = dst_layout.ty.kind() {
                     // Can only happen for vector types
                     let len =
-                        u16::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
+                        u32::try_from(len.eval_usize(fx.tcx, ParamEnv::reveal_all())).unwrap();
                     let vector_ty = fx.clif_type(*element).unwrap().by(len).unwrap();
 
                     let data = match from.0 {
@@ -614,7 +614,7 @@ impl<'tcx> CPlace<'tcx> {
                     dst_align,
                     src_align,
                     true,
-                    MemFlags::trusted(),
+                    flags,
                 );
             }
             CValueInner::ByRef(_, Some(_)) => todo!(),
@@ -815,7 +815,8 @@ pub(crate) fn assert_assignable<'tcx>(
             );
             // fn(&T) -> for<'l> fn(&'l T) is allowed
         }
-        (&ty::Dynamic(from_traits, _), &ty::Dynamic(to_traits, _)) => {
+        (&ty::Dynamic(from_traits, _, _from_kind), &ty::Dynamic(to_traits, _, _to_kind)) => {
+            // FIXME(dyn-star): Do the right thing with DynKinds
             for (from, to) in from_traits.iter().zip(to_traits) {
                 let from =
                     fx.tcx.normalize_erasing_late_bound_regions(ParamEnv::reveal_all(), from);