Adding upstream version 86.0.1.upstream/86.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

8 files changed, 2983 insertions, 0 deletions

diff --git a/third_party/rust/cranelift-codegen/src/legalizer/boundary.rs b/third_party/rust/cranelift-codegen/src/legalizer/boundary.rs
new file mode 100644
index 0000000000..6e14c884b8
--- /dev/null
+++ b/third_party/rust/cranelift-codegen/src/legalizer/boundary.rs
@@ -0,0 +1,1175 @@
+//! Legalize ABI boundaries.
+//!
+//! This legalizer sub-module contains code for dealing with ABI boundaries:
+//!
+//! - Function arguments passed to the entry block.
+//! - Function arguments passed to call instructions.
+//! - Return values from call instructions.
+//! - Return values passed to return instructions.
+//!
+//! The ABI boundary legalization happens in two phases:
+//!
+//! 1. The `legalize_signatures` function rewrites all the preamble signatures with ABI information
+//!    and possibly new argument types. It also rewrites the entry block arguments to match.
+//! 2. The `handle_call_abi` and `handle_return_abi` functions rewrite call and return instructions
+//!    to match the new ABI signatures.
+//!
+//! Between the two phases, preamble signatures and call/return arguments don't match. This
+//! intermediate state doesn't type check.
+
+use crate::abi::{legalize_abi_value, ValueConversion};
+use crate::cursor::{Cursor, FuncCursor};
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::instructions::CallInfo;
+use crate::ir::{
+    AbiParam, ArgumentLoc, ArgumentPurpose, Block, DataFlowGraph, ExtFuncData, ExternalName,
+    Function, Inst, InstBuilder, LibCall, MemFlags, SigRef, Signature, StackSlotData,
+    StackSlotKind, Type, Value, ValueLoc,
+};
+use crate::isa::TargetIsa;
+use crate::legalizer::split::{isplit, vsplit};
+use alloc::borrow::Cow;
+use alloc::vec::Vec;
+use core::mem;
+use cranelift_entity::EntityList;
+use log::debug;
+
+/// Legalize all the function signatures in `func`.
+///
+/// This changes all signatures to be ABI-compliant with full `ArgumentLoc` annotations. It doesn't
+/// change the entry block arguments, calls, or return instructions, so this can leave the function
+/// in a state with type discrepancies.
+pub fn legalize_signatures(func: &mut Function, isa: &dyn TargetIsa) {
+    if let Some(new) = legalize_signature(&func.signature, true, isa) {
+        let old = mem::replace(&mut func.signature, new);
+        func.old_signature = Some(old);
+    }
+
+    for (sig_ref, sig_data) in func.dfg.signatures.iter_mut() {
+        if let Some(new) = legalize_signature(sig_data, false, isa) {
+            let old = mem::replace(sig_data, new);
+            func.dfg.old_signatures[sig_ref] = Some(old);
+        }
+    }
+
+    if let Some(entry) = func.layout.entry_block() {
+        legalize_entry_params(func, entry);
+        spill_entry_params(func, entry);
+    }
+}
+
+/// Legalize the libcall signature, which we may generate on the fly after
+/// `legalize_signatures` has been called.
+pub fn legalize_libcall_signature(signature: &mut Signature, isa: &dyn TargetIsa) {
+    if let Some(s) = legalize_signature(signature, false, isa) {
+        *signature = s;
+    }
+}
+
+/// Legalize the given signature.
+///
+/// `current` is true if this is the signature for the current function.
+fn legalize_signature(
+    signature: &Signature,
+    current: bool,
+    isa: &dyn TargetIsa,
+) -> Option<Signature> {
+    let mut cow = Cow::Borrowed(signature);
+    isa.legalize_signature(&mut cow, current);
+    match cow {
+        Cow::Borrowed(_) => None,
+        Cow::Owned(s) => Some(s),
+    }
+}
+
+/// Legalize the entry block parameters after `func`'s signature has been legalized.
+///
+/// The legalized signature may contain more parameters than the original signature, and the
+/// parameter types have been changed. This function goes through the parameters of the entry block
+/// and replaces them with parameters of the right type for the ABI.
+///
+/// The original entry block parameters are computed from the new ABI parameters by code inserted at
+/// the top of the entry block.
+fn legalize_entry_params(func: &mut Function, entry: Block) {
+    let mut has_sret = false;
+    let mut has_link = false;
+    let mut has_vmctx = false;
+    let mut has_sigid = false;
+    let mut has_stack_limit = false;
+
+    // Insert position for argument conversion code.
+    // We want to insert instructions before the first instruction in the entry block.
+    // If the entry block is empty, append instructions to it instead.
+    let mut pos = FuncCursor::new(func).at_first_inst(entry);
+
+    // Keep track of the argument types in the ABI-legalized signature.
+    let mut abi_arg = 0;
+
+    // Process the block parameters one at a time, possibly replacing one argument with multiple new
+    // ones. We do this by detaching the entry block parameters first.
+    let block_params = pos.func.dfg.detach_block_params(entry);
+    let mut old_arg = 0;
+    while let Some(arg) = block_params.get(old_arg, &pos.func.dfg.value_lists) {
+        old_arg += 1;
+
+        let abi_type = pos.func.signature.params[abi_arg];
+        let arg_type = pos.func.dfg.value_type(arg);
+        if let ArgumentPurpose::StructArgument(size) = abi_type.purpose {
+            let offset = if let ArgumentLoc::Stack(offset) = abi_type.location {
+                offset
+            } else {
+                unreachable!("StructArgument must already have a Stack ArgumentLoc assigned");
+            };
+            let ss = pos.func.stack_slots.make_incoming_arg(size, offset);
+            let struct_arg = pos.ins().stack_addr(arg_type, ss, 0);
+            pos.func.dfg.change_to_alias(arg, struct_arg);
+            let dummy = pos
+                .func
+                .dfg
+                .append_block_param(entry, crate::ir::types::SARG_T);
+            pos.func.locations[dummy] = ValueLoc::Stack(ss);
+            abi_arg += 1;
+            continue;
+        }
+
+        if arg_type == abi_type.value_type {
+            // No value translation is necessary, this argument matches the ABI type.
+            // Just use the original block argument value. This is the most common case.
+            pos.func.dfg.attach_block_param(entry, arg);
+            match abi_type.purpose {
+                ArgumentPurpose::Normal => {}
+                ArgumentPurpose::StructArgument(_) => unreachable!("Handled above"),
+                ArgumentPurpose::FramePointer => {}
+                ArgumentPurpose::CalleeSaved => {}
+                ArgumentPurpose::StructReturn => {
+                    debug_assert!(!has_sret, "Multiple sret arguments found");
+                    has_sret = true;
+                }
+                ArgumentPurpose::VMContext => {
+                    debug_assert!(!has_vmctx, "Multiple vmctx arguments found");
+                    has_vmctx = true;
+                }
+                ArgumentPurpose::SignatureId => {
+                    debug_assert!(!has_sigid, "Multiple sigid arguments found");
+                    has_sigid = true;
+                }
+                ArgumentPurpose::StackLimit => {
+                    debug_assert!(!has_stack_limit, "Multiple stack_limit arguments found");
+                    has_stack_limit = true;
+                }
+                ArgumentPurpose::Link => panic!("Unexpected link arg {}", abi_type),
+                ArgumentPurpose::CallerTLS | ArgumentPurpose::CalleeTLS => {}
+            }
+            abi_arg += 1;
+        } else {
+            // Compute the value we want for `arg` from the legalized ABI parameters.
+            let mut get_arg = |func: &mut Function, ty| {
+                let abi_type = func.signature.params[abi_arg];
+                debug_assert_eq!(
+                    abi_type.purpose,
+                    ArgumentPurpose::Normal,
+                    "Can't legalize special-purpose argument"
+                );
+                if ty == abi_type.value_type {
+                    abi_arg += 1;
+                    Ok(func.dfg.append_block_param(entry, ty))
+                } else {
+                    Err(abi_type)
+                }
+            };
+            let converted = convert_from_abi(&mut pos, arg_type, Some(arg), &mut get_arg);
+            // The old `arg` is no longer an attached block argument, but there are probably still
+            // uses of the value.
+            debug_assert_eq!(pos.func.dfg.resolve_aliases(arg), converted);
+        }
+    }
+
+    // The legalized signature may contain additional parameters representing special-purpose
+    // registers.
+    for &arg in &pos.func.signature.params[abi_arg..] {
+        match arg.purpose {
+            // Any normal parameters should have been processed above.
+            ArgumentPurpose::Normal | ArgumentPurpose::StructArgument(_) => {
+                panic!("Leftover arg: {}", arg);
+            }
+            // The callee-save parameters should not appear until after register allocation is
+            // done.
+            ArgumentPurpose::FramePointer | ArgumentPurpose::CalleeSaved => {
+                panic!("Premature callee-saved arg {}", arg);
+            }
+            // These can be meaningfully added by `legalize_signature()`.
+            ArgumentPurpose::Link => {
+                debug_assert!(!has_link, "Multiple link parameters found");
+                has_link = true;
+            }
+            ArgumentPurpose::StructReturn => {
+                debug_assert!(!has_sret, "Multiple sret parameters found");
+                has_sret = true;
+            }
+            ArgumentPurpose::VMContext => {
+                debug_assert!(!has_vmctx, "Multiple vmctx parameters found");
+                has_vmctx = true;
+            }
+            ArgumentPurpose::SignatureId => {
+                debug_assert!(!has_sigid, "Multiple sigid parameters found");
+                has_sigid = true;
+            }
+            ArgumentPurpose::StackLimit => {
+                debug_assert!(!has_stack_limit, "Multiple stack_limit parameters found");
+                has_stack_limit = true;
+            }
+            ArgumentPurpose::CallerTLS | ArgumentPurpose::CalleeTLS => {}
+        }
+
+        // Just create entry block values to match here. We will use them in `handle_return_abi()`
+        // below.
+        pos.func.dfg.append_block_param(entry, arg.value_type);
+    }
+}
+
+/// Legalize the results returned from a call instruction to match the ABI signature.
+///
+/// The cursor `pos` points to a call instruction with at least one return value. The cursor will
+/// be left pointing after the instructions inserted to convert the return values.
+///
+/// This function is very similar to the `legalize_entry_params` function above.
+///
+/// Returns the possibly new instruction representing the call.
+fn legalize_inst_results<ResType>(pos: &mut FuncCursor, mut get_abi_type: ResType) -> Inst
+where
+    ResType: FnMut(&Function, usize) -> AbiParam,
+{
+    let call = pos
+        .current_inst()
+        .expect("Cursor must point to a call instruction");
+
+    // We theoretically allow for call instructions that return a number of fixed results before
+    // the call return values. In practice, it doesn't happen.
+    debug_assert_eq!(
+        pos.func.dfg[call]
+            .opcode()
+            .constraints()
+            .num_fixed_results(),
+        0,
+        "Fixed results on calls not supported"
+    );
+
+    let results = pos.func.dfg.detach_results(call);
+    let mut next_res = 0;
+    let mut abi_res = 0;
+
+    // Point immediately after the call.
+    pos.next_inst();
+
+    while let Some(res) = results.get(next_res, &pos.func.dfg.value_lists) {
+        next_res += 1;
+
+        let res_type = pos.func.dfg.value_type(res);
+        if res_type == get_abi_type(pos.func, abi_res).value_type {
+            // No value translation is necessary, this result matches the ABI type.
+            pos.func.dfg.attach_result(call, res);
+            abi_res += 1;
+        } else {
+            let mut get_res = |func: &mut Function, ty| {
+                let abi_type = get_abi_type(func, abi_res);
+                if ty == abi_type.value_type {
+                    let last_res = func.dfg.append_result(call, ty);
+                    abi_res += 1;
+                    Ok(last_res)
+                } else {
+                    Err(abi_type)
+                }
+            };
+            let v = convert_from_abi(pos, res_type, Some(res), &mut get_res);
+            debug_assert_eq!(pos.func.dfg.resolve_aliases(res), v);
+        }
+    }
+
+    call
+}
+
+fn assert_is_valid_sret_legalization(
+    old_ret_list: &EntityList<Value>,
+    old_sig: &Signature,
+    new_sig: &Signature,
+    pos: &FuncCursor,
+) {
+    debug_assert_eq!(
+        old_sig.returns.len(),
+        old_ret_list.len(&pos.func.dfg.value_lists)
+    );
+
+    // Assert that the only difference in special parameters is that there
+    // is an appended struct return pointer parameter.
+    let old_special_params: Vec<_> = old_sig
+        .params
+        .iter()
+        .filter(|r| r.purpose != ArgumentPurpose::Normal)
+        .collect();
+    let new_special_params: Vec<_> = new_sig
+        .params
+        .iter()
+        .filter(|r| r.purpose != ArgumentPurpose::Normal)
+        .collect();
+    debug_assert_eq!(old_special_params.len() + 1, new_special_params.len());
+    debug_assert!(old_special_params
+        .iter()
+        .zip(&new_special_params)
+        .all(|(old, new)| old.purpose == new.purpose));
+    debug_assert_eq!(
+        new_special_params.last().unwrap().purpose,
+        ArgumentPurpose::StructReturn
+    );
+
+    // If the special returns have changed at all, then the only change
+    // should be that the struct return pointer is returned back out of the
+    // function, so that callers don't have to load its stack address again.
+    let old_special_returns: Vec<_> = old_sig
+        .returns
+        .iter()
+        .filter(|r| r.purpose != ArgumentPurpose::Normal)
+        .collect();
+    let new_special_returns: Vec<_> = new_sig
+        .returns
+        .iter()
+        .filter(|r| r.purpose != ArgumentPurpose::Normal)
+        .collect();
+    debug_assert!(old_special_returns
+        .iter()
+        .zip(&new_special_returns)
+        .all(|(old, new)| old.purpose == new.purpose));
+    debug_assert!(
+        old_special_returns.len() == new_special_returns.len()
+            || (old_special_returns.len() + 1 == new_special_returns.len()
+                && new_special_returns.last().unwrap().purpose == ArgumentPurpose::StructReturn)
+    );
+}
+
+fn legalize_sret_call(isa: &dyn TargetIsa, pos: &mut FuncCursor, sig_ref: SigRef, call: Inst) {
+    let old_ret_list = pos.func.dfg.detach_results(call);
+    let old_sig = pos.func.dfg.old_signatures[sig_ref]
+        .take()
+        .expect("must have an old signature when using an `sret` parameter");
+
+    // We make a bunch of assumptions about the shape of the old, multi-return
+    // signature and the new, sret-using signature in this legalization
+    // function. Assert that these assumptions hold true in debug mode.
+    if cfg!(debug_assertions) {
+        assert_is_valid_sret_legalization(
+            &old_ret_list,
+            &old_sig,
+            &pos.func.dfg.signatures[sig_ref],
+            &pos,
+        );
+    }
+
+    // Go through and remove all normal return values from the `call`
+    // instruction's returns list. These will be stored into the stack slot that
+    // the sret points to. At the same time, calculate the size of the sret
+    // stack slot.
+    let mut sret_slot_size = 0;
+    for (i, ret) in old_sig.returns.iter().enumerate() {
+        let v = old_ret_list.get(i, &pos.func.dfg.value_lists).unwrap();
+        let ty = pos.func.dfg.value_type(v);
+        if ret.purpose == ArgumentPurpose::Normal {
+            debug_assert_eq!(ret.location, ArgumentLoc::Unassigned);
+            let ty = legalized_type_for_sret(ty);
+            let size = ty.bytes();
+            sret_slot_size = round_up_to_multiple_of_type_align(sret_slot_size, ty) + size;
+        } else {
+            let new_v = pos.func.dfg.append_result(call, ty);
+            pos.func.dfg.change_to_alias(v, new_v);
+        }
+    }
+
+    let stack_slot = pos.func.stack_slots.push(StackSlotData {
+        kind: StackSlotKind::StructReturnSlot,
+        size: sret_slot_size,
+        offset: None,
+    });
+
+    // Append the sret pointer to the `call` instruction's arguments.
+    let ptr_type = Type::triple_pointer_type(isa.triple());
+    let sret_arg = pos.ins().stack_addr(ptr_type, stack_slot, 0);
+    pos.func.dfg.append_inst_arg(call, sret_arg);
+
+    // The sret pointer might be returned by the signature as well. If so, we
+    // need to add it to the `call` instruction's results list.
+    //
+    // Additionally, when the sret is explicitly returned in this calling
+    // convention, then use it when loading the sret returns back into ssa
+    // values to avoid keeping the original `sret_arg` live and potentially
+    // having to do spills and fills.
+    let sret =
+        if pos.func.dfg.signatures[sig_ref].uses_special_return(ArgumentPurpose::StructReturn) {
+            pos.func.dfg.append_result(call, ptr_type)
+        } else {
+            sret_arg
+        };
+
+    // Finally, load each of the call's return values out of the sret stack
+    // slot.
+    pos.goto_after_inst(call);
+    let mut offset = 0;
+    for i in 0..old_ret_list.len(&pos.func.dfg.value_lists) {
+        if old_sig.returns[i].purpose != ArgumentPurpose::Normal {
+            continue;
+        }
+
+        let old_v = old_ret_list.get(i, &pos.func.dfg.value_lists).unwrap();
+        let ty = pos.func.dfg.value_type(old_v);
+        let mut legalized_ty = legalized_type_for_sret(ty);
+
+        offset = round_up_to_multiple_of_type_align(offset, legalized_ty);
+
+        let new_legalized_v =
+            pos.ins()
+                .load(legalized_ty, MemFlags::trusted(), sret, offset as i32);
+
+        // "Illegalize" the loaded value from the legalized type back to its
+        // original `ty`. This is basically the opposite of
+        // `legalize_type_for_sret_store`.
+        let mut new_v = new_legalized_v;
+        if ty.is_bool() {
+            legalized_ty = legalized_ty.as_bool_pedantic();
+            new_v = pos.ins().raw_bitcast(legalized_ty, new_v);
+
+            if ty.bits() < legalized_ty.bits() {
+                legalized_ty = ty;
+                new_v = pos.ins().breduce(legalized_ty, new_v);
+            }
+        }
+
+        pos.func.dfg.change_to_alias(old_v, new_v);
+
+        offset += legalized_ty.bytes();
+    }
+
+    pos.func.dfg.old_signatures[sig_ref] = Some(old_sig);
+}
+
+/// Compute original value of type `ty` from the legalized ABI arguments.
+///
+/// The conversion is recursive, controlled by the `get_arg` closure which is called to retrieve an
+/// ABI argument. It returns:
+///
+/// - `Ok(arg)` if the requested type matches the next ABI argument.
+/// - `Err(arg_type)` if further conversions are needed from the ABI argument `arg_type`.
+///
+/// If the `into_result` value is provided, the converted result will be written into that value.
+fn convert_from_abi<GetArg>(
+    pos: &mut FuncCursor,
+    ty: Type,
+    into_result: Option<Value>,
+    get_arg: &mut GetArg,
+) -> Value
+where
+    GetArg: FnMut(&mut Function, Type) -> Result<Value, AbiParam>,
+{
+    // Terminate the recursion when we get the desired type.
+    let arg_type = match get_arg(pos.func, ty) {
+        Ok(v) => {
+            debug_assert_eq!(pos.func.dfg.value_type(v), ty);
+            debug_assert_eq!(into_result, None);
+            return v;
+        }
+        Err(t) => t,
+    };
+
+    // Reconstruct how `ty` was legalized into the `arg_type` argument.
+    let conversion = legalize_abi_value(ty, &arg_type);
+
+    debug!("convert_from_abi({}): {:?}", ty, conversion);
+
+    // The conversion describes value to ABI argument. We implement the reverse conversion here.
+    match conversion {
+        // Construct a `ty` by concatenating two ABI integers.
+        ValueConversion::IntSplit => {
+            let abi_ty = ty.half_width().expect("Invalid type for conversion");
+            let lo = convert_from_abi(pos, abi_ty, None, get_arg);
+            let hi = convert_from_abi(pos, abi_ty, None, get_arg);
+            debug!(
+                "intsplit {}: {}, {}: {}",
+                lo,
+                pos.func.dfg.value_type(lo),
+                hi,
+                pos.func.dfg.value_type(hi)
+            );
+            pos.ins().with_results([into_result]).iconcat(lo, hi)
+        }
+        // Construct a `ty` by concatenating two halves of a vector.
+        ValueConversion::VectorSplit => {
+            let abi_ty = ty.half_vector().expect("Invalid type for conversion");
+            let lo = convert_from_abi(pos, abi_ty, None, get_arg);
+            let hi = convert_from_abi(pos, abi_ty, None, get_arg);
+            pos.ins().with_results([into_result]).vconcat(lo, hi)
+        }
+        // Construct a `ty` by bit-casting from an integer type.
+        ValueConversion::IntBits => {
+            debug_assert!(!ty.is_int());
+            let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
+            let arg = convert_from_abi(pos, abi_ty, None, get_arg);
+            pos.ins().with_results([into_result]).bitcast(ty, arg)
+        }
+        // ABI argument is a sign-extended version of the value we want.
+        ValueConversion::Sext(abi_ty) => {
+            let arg = convert_from_abi(pos, abi_ty, None, get_arg);
+            // TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
+            // We could insert an `assert_sreduce` which would fold with a following `sextend` of
+            // this value.
+            pos.ins().with_results([into_result]).ireduce(ty, arg)
+        }
+        ValueConversion::Uext(abi_ty) => {
+            let arg = convert_from_abi(pos, abi_ty, None, get_arg);
+            // TODO: Currently, we don't take advantage of the ABI argument being sign-extended.
+            // We could insert an `assert_ureduce` which would fold with a following `uextend` of
+            // this value.
+            pos.ins().with_results([into_result]).ireduce(ty, arg)
+        }
+        // ABI argument is a pointer to the value we want.
+        ValueConversion::Pointer(abi_ty) => {
+            let arg = convert_from_abi(pos, abi_ty, None, get_arg);
+            pos.ins()
+                .with_results([into_result])
+                .load(ty, MemFlags::new(), arg, 0)
+        }
+    }
+}
+
+/// Convert `value` to match an ABI signature by inserting instructions at `pos`.
+///
+/// This may require expanding the value to multiple ABI arguments. The conversion process is
+/// recursive and controlled by the `put_arg` closure. When a candidate argument value is presented
+/// to the closure, it will perform one of two actions:
+///
+/// 1. If the suggested argument has an acceptable value type, consume it by adding it to the list
+///    of arguments and return `Ok(())`.
+/// 2. If the suggested argument doesn't have the right value type, don't change anything, but
+///    return the `Err(AbiParam)` that is needed.
+///
+fn convert_to_abi<PutArg>(
+    pos: &mut FuncCursor,
+    cfg: &ControlFlowGraph,
+    value: Value,
+    put_arg: &mut PutArg,
+) where
+    PutArg: FnMut(&mut Function, Value) -> Result<(), AbiParam>,
+{
+    // Start by invoking the closure to either terminate the recursion or get the argument type
+    // we're trying to match.
+    let arg_type = match put_arg(pos.func, value) {
+        Ok(_) => return,
+        Err(t) => t,
+    };
+
+    let ty = pos.func.dfg.value_type(value);
+    match legalize_abi_value(ty, &arg_type) {
+        ValueConversion::IntSplit => {
+            let curpos = pos.position();
+            let srcloc = pos.srcloc();
+            let (lo, hi) = isplit(&mut pos.func, cfg, curpos, srcloc, value);
+            convert_to_abi(pos, cfg, lo, put_arg);
+            convert_to_abi(pos, cfg, hi, put_arg);
+        }
+        ValueConversion::VectorSplit => {
+            let curpos = pos.position();
+            let srcloc = pos.srcloc();
+            let (lo, hi) = vsplit(&mut pos.func, cfg, curpos, srcloc, value);
+            convert_to_abi(pos, cfg, lo, put_arg);
+            convert_to_abi(pos, cfg, hi, put_arg);
+        }
+        ValueConversion::IntBits => {
+            debug_assert!(!ty.is_int());
+            let abi_ty = Type::int(ty.bits()).expect("Invalid type for conversion");
+            let arg = pos.ins().bitcast(abi_ty, value);
+            convert_to_abi(pos, cfg, arg, put_arg);
+        }
+        ValueConversion::Sext(abi_ty) => {
+            let arg = pos.ins().sextend(abi_ty, value);
+            convert_to_abi(pos, cfg, arg, put_arg);
+        }
+        ValueConversion::Uext(abi_ty) => {
+            let arg = pos.ins().uextend(abi_ty, value);
+            convert_to_abi(pos, cfg, arg, put_arg);
+        }
+        ValueConversion::Pointer(abi_ty) => {
+            // Note: This conversion can only happen for call arguments,
+            // so we can allocate the value on stack safely.
+            let stack_slot = pos.func.create_stack_slot(StackSlotData {
+                kind: StackSlotKind::ExplicitSlot,
+                size: ty.bytes(),
+                offset: None,
+            });
+            let arg = pos.ins().stack_addr(abi_ty, stack_slot, 0);
+            pos.ins().store(MemFlags::new(), value, arg, 0);
+            convert_to_abi(pos, cfg, arg, put_arg);
+        }
+    }
+}
+
+/// Check if a sequence of arguments match a desired sequence of argument types.
+fn check_arg_types(dfg: &DataFlowGraph, args: &[Value], types: &[AbiParam]) -> bool {
+    args.len() == types.len()
+        && args.iter().zip(types.iter()).all(|(v, at)| {
+            if let ArgumentPurpose::StructArgument(_) = at.purpose {
+                true
+            } else {
+                dfg.value_type(*v) == at.value_type
+            }
+        })
+}
+
+/// Check if the arguments of the call `inst` match the signature.
+///
+/// Returns `Ok(())` if the signature matches and no changes are needed, or `Err(sig_ref)` if the
+/// signature doesn't match.
+fn check_call_signature(dfg: &DataFlowGraph, inst: Inst) -> Result<(), SigRef> {
+    // Extract the signature and argument values.
+    let (sig_ref, args) = match dfg[inst].analyze_call(&dfg.value_lists) {
+        CallInfo::Direct(func, args) => (dfg.ext_funcs[func].signature, args),
+        CallInfo::Indirect(sig_ref, args) => (sig_ref, args),
+        CallInfo::NotACall => panic!("Expected call, got {:?}", dfg[inst]),
+    };
+    let sig = &dfg.signatures[sig_ref];
+
+    if check_arg_types(dfg, args, &sig.params[..])
+        && check_arg_types(dfg, dfg.inst_results(inst), &sig.returns[..])
+    {
+        // All types check out.
+        Ok(())
+    } else {
+        // Call types need fixing.
+        Err(sig_ref)
+    }
+}
+
+/// Check if the arguments of the return `inst` match the signature.
+fn check_return_signature(dfg: &DataFlowGraph, inst: Inst, sig: &Signature) -> bool {
+    check_arg_types(dfg, dfg.inst_variable_args(inst), &sig.returns)
+}
+
+/// Insert ABI conversion code for the arguments to the call or return instruction at `pos`.
+///
+/// - `abi_args` is the number of arguments that the ABI signature requires.
+/// - `get_abi_type` is a closure that can provide the desired `AbiParam` for a given ABI
+///   argument number in `0..abi_args`.
+///
+fn legalize_inst_arguments<ArgType>(
+    pos: &mut FuncCursor,
+    cfg: &ControlFlowGraph,
+    abi_args: usize,
+    mut get_abi_type: ArgType,
+) where
+    ArgType: FnMut(&Function, usize) -> AbiParam,
+{
+    let inst = pos
+        .current_inst()
+        .expect("Cursor must point to a call instruction");
+
+    // Lift the value list out of the call instruction so we modify it.
+    let mut vlist = pos.func.dfg[inst]
+        .take_value_list()
+        .expect("Call must have a value list");
+
+    // The value list contains all arguments to the instruction, including the callee on an
+    // indirect call which isn't part of the call arguments that must match the ABI signature.
+    // Figure out how many fixed values are at the front of the list. We won't touch those.
+    let num_fixed_values = pos.func.dfg[inst]
+        .opcode()
+        .constraints()
+        .num_fixed_value_arguments();
+    let have_args = vlist.len(&pos.func.dfg.value_lists) - num_fixed_values;
+    if abi_args < have_args {
+        // This happens with multiple return values after we've legalized the
+        // signature but haven't legalized the return instruction yet. This
+        // legalization is handled in `handle_return_abi`.
+        pos.func.dfg[inst].put_value_list(vlist);
+        return;
+    }
+
+    // Grow the value list to the right size and shift all the existing arguments to the right.
+    // This lets us write the new argument values into the list without overwriting the old
+    // arguments.
+    //
+    // Before:
+    //
+    //    <-->              fixed_values
+    //        <-----------> have_args
+    //   [FFFFOOOOOOOOOOOOO]
+    //
+    // After grow_at():
+    //
+    //    <-->                     fixed_values
+    //               <-----------> have_args
+    //        <------------------> abi_args
+    //   [FFFF-------OOOOOOOOOOOOO]
+    //               ^
+    //               old_arg_offset
+    //
+    // After writing the new arguments:
+    //
+    //    <-->                     fixed_values
+    //        <------------------> abi_args
+    //   [FFFFNNNNNNNNNNNNNNNNNNNN]
+    //
+    vlist.grow_at(
+        num_fixed_values,
+        abi_args - have_args,
+        &mut pos.func.dfg.value_lists,
+    );
+    let old_arg_offset = num_fixed_values + abi_args - have_args;
+
+    let mut abi_arg = 0;
+    for old_arg in 0..have_args {
+        let old_value = vlist
+            .get(old_arg_offset + old_arg, &pos.func.dfg.value_lists)
+            .unwrap();
+        let mut put_arg = |func: &mut Function, arg| {
+            let abi_type = get_abi_type(func, abi_arg);
+            let struct_argument = if let ArgumentPurpose::StructArgument(_) = abi_type.purpose {
+                true
+            } else {
+                false
+            };
+            if func.dfg.value_type(arg) == abi_type.value_type || struct_argument {
+                // This is the argument type we need.
+                vlist.as_mut_slice(&mut func.dfg.value_lists)[num_fixed_values + abi_arg] = arg;
+                abi_arg += 1;
+                Ok(())
+            } else {
+                Err(abi_type)
+            }
+        };
+        convert_to_abi(pos, cfg, old_value, &mut put_arg);
+    }
+
+    // Put the modified value list back.
+    pos.func.dfg[inst].put_value_list(vlist);
+}
+
+/// Ensure that the `ty` being returned is a type that can be loaded and stored
+/// (potentially after another narrowing legalization) from memory, since it
+/// will go into the `sret` space.
+fn legalized_type_for_sret(ty: Type) -> Type {
+    if ty.is_bool() {
+        let bits = std::cmp::max(8, ty.bits());
+        Type::int(bits).unwrap()
+    } else {
+        ty
+    }
+}
+
+/// Insert any legalization code required to ensure that `val` can be stored
+/// into the `sret` memory. Returns the (potentially new, potentially
+/// unmodified) legalized value and its type.
+fn legalize_type_for_sret_store(pos: &mut FuncCursor, val: Value, ty: Type) -> (Value, Type) {
+    if ty.is_bool() {
+        let bits = std::cmp::max(8, ty.bits());
+        let ty = Type::int(bits).unwrap();
+        let val = pos.ins().bint(ty, val);
+        (val, ty)
+    } else {
+        (val, ty)
+    }
+}
+
+/// Insert ABI conversion code before and after the call instruction at `pos`.
+///
+/// Instructions inserted before the call will compute the appropriate ABI values for the
+/// callee's new ABI-legalized signature. The function call arguments are rewritten in place to
+/// match the new signature.
+///
+/// Instructions will be inserted after the call to convert returned ABI values back to the
+/// original return values. The call's result values will be adapted to match the new signature.
+///
+/// Returns `true` if any instructions were inserted.
+pub fn handle_call_abi(
+    isa: &dyn TargetIsa,
+    mut inst: Inst,
+    func: &mut Function,
+    cfg: &ControlFlowGraph,
+) -> bool {
+    let pos = &mut FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    // Start by checking if the argument types already match the signature.
+    let sig_ref = match check_call_signature(&pos.func.dfg, inst) {
+        Ok(_) => return spill_call_arguments(pos, isa),
+        Err(s) => s,
+    };
+
+    let sig = &pos.func.dfg.signatures[sig_ref];
+    let old_sig = &pos.func.dfg.old_signatures[sig_ref];
+
+    if sig.uses_struct_return_param()
+        && old_sig
+            .as_ref()
+            .map_or(false, |s| !s.uses_struct_return_param())
+    {
+        legalize_sret_call(isa, pos, sig_ref, inst);
+    } else {
+        if !pos.func.dfg.signatures[sig_ref].returns.is_empty() {
+            inst = legalize_inst_results(pos, |func, abi_res| {
+                func.dfg.signatures[sig_ref].returns[abi_res]
+            });
+        }
+    }
+
+    // Go back and fix the call arguments to match the ABI signature.
+    pos.goto_inst(inst);
+    let abi_args = pos.func.dfg.signatures[sig_ref].params.len();
+    legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
+        func.dfg.signatures[sig_ref].params[abi_arg]
+    });
+
+    debug_assert!(
+        check_call_signature(&pos.func.dfg, inst).is_ok(),
+        "Signature still wrong: {}, {}{}",
+        pos.func.dfg.display_inst(inst, None),
+        sig_ref,
+        pos.func.dfg.signatures[sig_ref]
+    );
+
+    // Go back and insert spills for any stack arguments.
+    pos.goto_inst(inst);
+    spill_call_arguments(pos, isa);
+
+    // Yes, we changed stuff.
+    true
+}
+
+/// Insert ABI conversion code before and after the return instruction at `inst`.
+///
+/// Return `true` if any instructions were inserted.
+pub fn handle_return_abi(inst: Inst, func: &mut Function, cfg: &ControlFlowGraph) -> bool {
+    // Check if the returned types already match the signature.
+    if check_return_signature(&func.dfg, inst, &func.signature) {
+        return false;
+    }
+
+    // Count the special-purpose return values (`link`, `sret`, and `vmctx`) that were appended to
+    // the legalized signature.
+    let special_args = func
+        .signature
+        .returns
+        .iter()
+        .rev()
+        .take_while(|&rt| {
+            rt.purpose == ArgumentPurpose::Link
+                || rt.purpose == ArgumentPurpose::StructReturn
+                || rt.purpose == ArgumentPurpose::VMContext
+        })
+        .count();
+    let abi_args = func.signature.returns.len() - special_args;
+
+    let pos = &mut FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    legalize_inst_arguments(pos, cfg, abi_args, |func, abi_arg| {
+        let arg = func.signature.returns[abi_arg];
+        debug_assert!(
+            !arg.legalized_to_pointer,
+            "Return value cannot be legalized to pointer"
+        );
+        arg
+    });
+    // Append special return arguments for any `sret`, `link`, and `vmctx` return values added to
+    // the legalized signature. These values should simply be propagated from the entry block
+    // arguments.
+    if special_args > 0 {
+        debug!(
+            "Adding {} special-purpose arguments to {}",
+            special_args,
+            pos.func.dfg.display_inst(inst, None)
+        );
+        let mut vlist = pos.func.dfg[inst].take_value_list().unwrap();
+        let mut sret = None;
+
+        for arg in &pos.func.signature.returns[abi_args..] {
+            match arg.purpose {
+                ArgumentPurpose::Link
+                | ArgumentPurpose::StructReturn
+                | ArgumentPurpose::VMContext => {}
+                ArgumentPurpose::Normal => panic!("unexpected return value {}", arg),
+                _ => panic!("Unsupported special purpose return value {}", arg),
+            }
+            // A `link`/`sret`/`vmctx` return value can only appear in a signature that has a
+            // unique matching argument. They are appended at the end, so search the signature from
+            // the end.
+            let idx = pos
+                .func
+                .signature
+                .params
+                .iter()
+                .rposition(|t| t.purpose == arg.purpose)
+                .expect("No matching special purpose argument.");
+            // Get the corresponding entry block value and add it to the return instruction's
+            // arguments.
+            let val = pos
+                .func
+                .dfg
+                .block_params(pos.func.layout.entry_block().unwrap())[idx];
+            debug_assert_eq!(pos.func.dfg.value_type(val), arg.value_type);
+            vlist.push(val, &mut pos.func.dfg.value_lists);
+
+            if let ArgumentPurpose::StructReturn = arg.purpose {
+                sret = Some(val);
+            }
+        }
+
+        // Store all the regular returns into the retptr space and remove them
+        // from the `return` instruction's value list.
+        if let Some(sret) = sret {
+            let mut offset = 0;
+            let num_regular_rets = vlist.len(&pos.func.dfg.value_lists) - special_args;
+            for i in 0..num_regular_rets {
+                debug_assert_eq!(
+                    pos.func.old_signature.as_ref().unwrap().returns[i].purpose,
+                    ArgumentPurpose::Normal,
+                );
+
+                // The next return value to process is always at `0`, since the
+                // list is emptied as we iterate.
+                let v = vlist.get(0, &pos.func.dfg.value_lists).unwrap();
+                let ty = pos.func.dfg.value_type(v);
+                let (v, ty) = legalize_type_for_sret_store(pos, v, ty);
+
+                let size = ty.bytes();
+                offset = round_up_to_multiple_of_type_align(offset, ty);
+
+                pos.ins().store(MemFlags::trusted(), v, sret, offset as i32);
+                vlist.remove(0, &mut pos.func.dfg.value_lists);
+
+                offset += size;
+            }
+        }
+        pos.func.dfg[inst].put_value_list(vlist);
+    }
+
+    debug_assert_eq!(
+        pos.func.dfg.inst_variable_args(inst).len(),
+        abi_args + special_args
+    );
+    debug_assert!(
+        check_return_signature(&pos.func.dfg, inst, &pos.func.signature),
+        "Signature still wrong: {} / signature {}",
+        pos.func.dfg.display_inst(inst, None),
+        pos.func.signature
+    );
+
+    // Yes, we changed stuff.
+    true
+}
+
+fn round_up_to_multiple_of_type_align(bytes: u32, ty: Type) -> u32 {
+    // We don't have a dedicated alignment for types, so assume they are
+    // size-aligned.
+    let align = ty.bytes();
+    round_up_to_multiple_of_pow2(bytes, align)
+}
+
+/// Round `n` up to the next multiple of `to` that is greater than or equal to
+/// `n`.
+///
+/// `to` must be a power of two and greater than zero.
+///
+/// This is useful for rounding an offset or pointer up to some type's required
+/// alignment.
+fn round_up_to_multiple_of_pow2(n: u32, to: u32) -> u32 {
+    debug_assert!(to > 0);
+    debug_assert!(to.is_power_of_two());
+
+    // The simple version of this function is
+    //
+    //     (n + to - 1) / to * to
+    //
+    // Consider the numerator: `n + to - 1`. This is ensuring that if there is
+    // any remainder for `n / to`, then the result of the division is one
+    // greater than `n / to`, and that otherwise we get exactly the same result
+    // as `n / to` due to integer division rounding off the remainder. In other
+    // words, we only round up if `n` is not aligned to `to`.
+    //
+    // However, we know `to` is a power of two, and therefore `anything / to` is
+    // equivalent to `anything >> log2(to)` and `anything * to` is equivalent to
+    // `anything << log2(to)`. We can therefore rewrite our simplified function
+    // into the following:
+    //
+    //     (n + to - 1) >> log2(to) << log2(to)
+    //
+    // But shifting a value right by some number of bits `b` and then shifting
+    // it left by that same number of bits `b` is equivalent to clearing the
+    // bottom `b` bits of the number. We can clear the bottom `b` bits of a
+    // number by bit-wise and'ing the number with the bit-wise not of `2^b - 1`.
+    // Plugging this into our function and simplifying, we get:
+    //
+    //       (n + to - 1) >> log2(to) << log2(to)
+    //     = (n + to - 1) & !(2^log2(to) - 1)
+    //     = (n + to - 1) & !(to - 1)
+    //
+    // And now we have the final version of this function!
+
+    (n + to - 1) & !(to - 1)
+}
+
+/// Assign stack slots to incoming function parameters on the stack.
+///
+/// Values that are passed into the function on the stack must be assigned to an `IncomingArg`
+/// stack slot already during legalization.
+fn spill_entry_params(func: &mut Function, entry: Block) {
+    for (abi, &arg) in func
+        .signature
+        .params
+        .iter()
+        .zip(func.dfg.block_params(entry))
+    {
+        if let ArgumentPurpose::StructArgument(_) = abi.purpose {
+            // Location has already been assigned during legalization.
+        } else if let ArgumentLoc::Stack(offset) = abi.location {
+            let ss = func
+                .stack_slots
+                .make_incoming_arg(abi.value_type.bytes(), offset);
+            func.locations[arg] = ValueLoc::Stack(ss);
+        }
+    }
+}
+
+/// Assign stack slots to outgoing function arguments on the stack.
+///
+/// Values that are passed to a called function on the stack must be assigned to a matching
+/// `OutgoingArg` stack slot. The assignment must happen immediately before the call.
+///
+/// TODO: The outgoing stack slots can be written a bit earlier, as long as there are no branches
+/// or calls between writing the stack slots and the call instruction. Writing the slots earlier
+/// could help reduce register pressure before the call.
+fn spill_call_arguments(pos: &mut FuncCursor, isa: &dyn TargetIsa) -> bool {
+    let inst = pos
+        .current_inst()
+        .expect("Cursor must point to a call instruction");
+    let sig_ref = pos
+        .func
+        .dfg
+        .call_signature(inst)
+        .expect("Call instruction expected.");
+
+    // Start by building a list of stack slots and arguments to be replaced.
+    // This requires borrowing `pos.func.dfg`, so we can't change anything.
+    let arglist = {
+        let locations = &pos.func.locations;
+        let stack_slots = &mut pos.func.stack_slots;
+        pos.func
+            .dfg
+            .inst_variable_args(inst)
+            .iter()
+            .zip(&pos.func.dfg.signatures[sig_ref].params)
+            .enumerate()
+            .filter_map(|(idx, (&arg, abi))| {
+                match abi.location {
+                    ArgumentLoc::Stack(offset) => {
+                        // Assign `arg` to a new stack slot, unless it's already in the correct
+                        // slot. The legalization needs to be idempotent, so we should see a
+                        // correct outgoing slot on the second pass.
+                        let (ss, size) = match abi.purpose {
+                            ArgumentPurpose::StructArgument(size) => {
+                                (stack_slots.get_outgoing_arg(size, offset), Some(size))
+                            }
+                            _ => (
+                                stack_slots.get_outgoing_arg(abi.value_type.bytes(), offset),
+                                None,
+                            ),
+                        };
+                        if locations[arg] != ValueLoc::Stack(ss) {
+                            Some((idx, arg, ss, size))
+                        } else {
+                            None
+                        }
+                    }
+                    _ => None,
+                }
+            })
+            .collect::<Vec<_>>()
+    };
+
+    if arglist.is_empty() {
+        return false;
+    }
+
+    let mut libc_memcpy = None;
+    let mut import_memcpy = |func: &mut Function, pointer_type| {
+        if let Some(libc_memcpy) = libc_memcpy {
+            return libc_memcpy;
+        }
+
+        let signature = {
+            let mut s = Signature::new(isa.default_call_conv());
+            s.params.push(AbiParam::new(pointer_type));
+            s.params.push(AbiParam::new(pointer_type));
+            // The last argument of `memcpy` is a `size_t`. This is the same size as a pointer on
+            // all architectures we are interested in.
+            s.params.push(AbiParam::new(pointer_type));
+            legalize_libcall_signature(&mut s, isa);
+            func.import_signature(s)
+        };
+
+        let func = func.import_function(ExtFuncData {
+            name: ExternalName::LibCall(LibCall::Memcpy),
+            signature,
+            colocated: false,
+        });
+        libc_memcpy = Some(func);
+        func
+    };
+
+    // Insert the spill instructions and rewrite call arguments.
+    for (idx, arg, ss, size) in arglist {
+        let stack_val = if let Some(size) = size {
+            // Struct argument
+            let pointer_type = pos.func.dfg.value_type(arg);
+            let src = arg;
+            let dest = pos.ins().stack_addr(pointer_type, ss, 0);
+            let size = pos.ins().iconst(pointer_type, i64::from(size));
+
+            let libc_memcpy = import_memcpy(pos.func, pointer_type);
+            pos.ins().call(libc_memcpy, &[dest, src, size]);
+            pos.ins().dummy_sarg_t()
+        } else {
+            // Non struct argument
+            pos.ins().spill(arg)
+        };
+        pos.func.locations[stack_val] = ValueLoc::Stack(ss);
+        pos.func.dfg.inst_variable_args_mut(inst)[idx] = stack_val;
+    }
+
+    // We changed stuff.
+    true
+}
+
+#[cfg(test)]
+mod tests {
+    use super::round_up_to_multiple_of_pow2;
+
+    #[test]
+    fn round_up_to_multiple_of_pow2_works() {
+        for (n, to, expected) in vec![
+            (0, 1, 0),
+            (1, 1, 1),
+            (2, 1, 2),
+            (0, 2, 0),
+            (1, 2, 2),
+            (2, 2, 2),
+            (3, 2, 4),
+            (0, 4, 0),
+            (1, 4, 4),
+            (2, 4, 4),
+            (3, 4, 4),
+            (4, 4, 4),
+            (5, 4, 8),
+        ] {
+            let actual = round_up_to_multiple_of_pow2(n, to);
+            assert_eq!(
+                actual, expected,
+                "round_up_to_multiple_of_pow2(n = {}, to = {}) = {} (expected {})",
+                n, to, actual, expected
+            );
+        }
+    }
+}
diff --git a/third_party/rust/cranelift-codegen/src/legalizer/call.rs b/third_party/rust/cranelift-codegen/src/legalizer/call.rs
new file mode 100644
index 0000000000..4321dbb90b
--- /dev/null
+++ b/third_party/rust/cranelift-codegen/src/legalizer/call.rs
@@ -0,0 +1,54 @@
+//! Legalization of calls.
+//!
+//! This module exports the `expand_call` function which transforms a `call`
+//! instruction into `func_addr` and `call_indirect` instructions.
+
+use crate::cursor::{Cursor, FuncCursor};
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::{self, InstBuilder};
+use crate::isa::TargetIsa;
+
+/// Expand a `call` instruction. This lowers it to a `call_indirect`, which
+/// is only done if the ABI doesn't support direct calls.
+pub fn expand_call(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) {
+    // Unpack the instruction.
+    let (func_ref, old_args) = match func.dfg[inst] {
+        ir::InstructionData::Call {
+            opcode,
+            ref args,
+            func_ref,
+        } => {
+            debug_assert_eq!(opcode, ir::Opcode::Call);
+            (func_ref, args.clone())
+        }
+        _ => panic!("Wanted call: {}", func.dfg.display_inst(inst, None)),
+    };
+
+    let ptr_ty = isa.pointer_type();
+
+    let sig = func.dfg.ext_funcs[func_ref].signature;
+
+    let callee = {
+        let mut pos = FuncCursor::new(func).at_inst(inst);
+        pos.use_srcloc(inst);
+        pos.ins().func_addr(ptr_ty, func_ref)
+    };
+
+    let mut new_args = ir::ValueList::default();
+    new_args.push(callee, &mut func.dfg.value_lists);
+    for i in 0..old_args.len(&func.dfg.value_lists) {
+        new_args.push(
+            old_args.as_slice(&func.dfg.value_lists)[i],
+            &mut func.dfg.value_lists,
+        );
+    }
+
+    func.dfg
+        .replace(inst)
+        .CallIndirect(ir::Opcode::CallIndirect, ptr_ty, sig, new_args);
+}
diff --git a/third_party/rust/cranelift-codegen/src/legalizer/globalvalue.rs b/third_party/rust/cranelift-codegen/src/legalizer/globalvalue.rs
new file mode 100644
index 0000000000..5c7a72b45c
--- /dev/null
+++ b/third_party/rust/cranelift-codegen/src/legalizer/globalvalue.rs
@@ -0,0 +1,140 @@
+//! Legalization of global values.
+//!
+//! This module exports the `expand_global_value` function which transforms a `global_value`
+//! instruction into code that depends on the kind of global value referenced.
+
+use crate::cursor::{Cursor, FuncCursor};
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::{self, InstBuilder};
+use crate::isa::TargetIsa;
+
+/// Expand a `global_value` instruction according to the definition of the global value.
+pub fn expand_global_value(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) {
+    // Unpack the instruction.
+    let gv = match func.dfg[inst] {
+        ir::InstructionData::UnaryGlobalValue {
+            opcode,
+            global_value,
+        } => {
+            debug_assert_eq!(opcode, ir::Opcode::GlobalValue);
+            global_value
+        }
+        _ => panic!("Wanted global_value: {}", func.dfg.display_inst(inst, None)),
+    };
+
+    match func.global_values[gv] {
+        ir::GlobalValueData::VMContext => vmctx_addr(inst, func),
+        ir::GlobalValueData::IAddImm {
+            base,
+            offset,
+            global_type,
+        } => iadd_imm_addr(inst, func, base, offset.into(), global_type),
+        ir::GlobalValueData::Load {
+            base,
+            offset,
+            global_type,
+            readonly,
+        } => load_addr(inst, func, base, offset, global_type, readonly, isa),
+        ir::GlobalValueData::Symbol { tls, .. } => symbol(inst, func, gv, isa, tls),
+    }
+}
+
+/// Expand a `global_value` instruction for a vmctx global.
+fn vmctx_addr(inst: ir::Inst, func: &mut ir::Function) {
+    // Get the value representing the `vmctx` argument.
+    let vmctx = func
+        .special_param(ir::ArgumentPurpose::VMContext)
+        .expect("Missing vmctx parameter");
+
+    // Replace the `global_value` instruction's value with an alias to the vmctx arg.
+    let result = func.dfg.first_result(inst);
+    func.dfg.clear_results(inst);
+    func.dfg.change_to_alias(result, vmctx);
+    func.layout.remove_inst(inst);
+}
+
+/// Expand a `global_value` instruction for an iadd_imm global.
+fn iadd_imm_addr(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    base: ir::GlobalValue,
+    offset: i64,
+    global_type: ir::Type,
+) {
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+
+    // Get the value for the lhs. For tidiness, expand VMContext here so that we avoid
+    // `vmctx_addr` which creates an otherwise unneeded value alias.
+    let lhs = if let ir::GlobalValueData::VMContext = pos.func.global_values[base] {
+        pos.func
+            .special_param(ir::ArgumentPurpose::VMContext)
+            .expect("Missing vmctx parameter")
+    } else {
+        pos.ins().global_value(global_type, base)
+    };
+
+    // Simply replace the `global_value` instruction with an `iadd_imm`, reusing the result value.
+    pos.func.dfg.replace(inst).iadd_imm(lhs, offset);
+}
+
+/// Expand a `global_value` instruction for a load global.
+fn load_addr(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    base: ir::GlobalValue,
+    offset: ir::immediates::Offset32,
+    global_type: ir::Type,
+    readonly: bool,
+    isa: &dyn TargetIsa,
+) {
+    // We need to load a pointer from the `base` global value, so insert a new `global_value`
+    // instruction. This depends on the iterative legalization loop. Note that the IR verifier
+    // detects any cycles in the `load` globals.
+    let ptr_ty = isa.pointer_type();
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    // Get the value for the base. For tidiness, expand VMContext here so that we avoid
+    // `vmctx_addr` which creates an otherwise unneeded value alias.
+    let base_addr = if let ir::GlobalValueData::VMContext = pos.func.global_values[base] {
+        pos.func
+            .special_param(ir::ArgumentPurpose::VMContext)
+            .expect("Missing vmctx parameter")
+    } else {
+        pos.ins().global_value(ptr_ty, base)
+    };
+
+    // Global-value loads are always notrap and aligned. They may be readonly.
+    let mut mflags = ir::MemFlags::trusted();
+    if readonly {
+        mflags.set_readonly();
+    }
+
+    // Perform the load.
+    pos.func
+        .dfg
+        .replace(inst)
+        .load(global_type, mflags, base_addr, offset);
+}
+
+/// Expand a `global_value` instruction for a symbolic name global.
+fn symbol(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    gv: ir::GlobalValue,
+    isa: &dyn TargetIsa,
+    tls: bool,
+) {
+    let ptr_ty = isa.pointer_type();
+
+    if tls {
+        func.dfg.replace(inst).tls_value(ptr_ty, gv);
+    } else {
+        func.dfg.replace(inst).symbol_value(ptr_ty, gv);
+    }
+}
diff --git a/third_party/rust/cranelift-codegen/src/legalizer/heap.rs b/third_party/rust/cranelift-codegen/src/legalizer/heap.rs
new file mode 100644
index 0000000000..26a0640f08
--- /dev/null
+++ b/third_party/rust/cranelift-codegen/src/legalizer/heap.rs
@@ -0,0 +1,241 @@
+//! Legalization of heaps.
+//!
+//! This module exports the `expand_heap_addr` function which transforms a `heap_addr`
+//! instruction into code that depends on the kind of heap referenced.
+
+use crate::cursor::{Cursor, FuncCursor};
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::condcodes::IntCC;
+use crate::ir::{self, InstBuilder};
+use crate::isa::TargetIsa;
+
+/// Expand a `heap_addr` instruction according to the definition of the heap.
+pub fn expand_heap_addr(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) {
+    // Unpack the instruction.
+    let (heap, offset, access_size) = match func.dfg[inst] {
+        ir::InstructionData::HeapAddr {
+            opcode,
+            heap,
+            arg,
+            imm,
+        } => {
+            debug_assert_eq!(opcode, ir::Opcode::HeapAddr);
+            (heap, arg, imm.into())
+        }
+        _ => panic!("Wanted heap_addr: {}", func.dfg.display_inst(inst, None)),
+    };
+
+    match func.heaps[heap].style {
+        ir::HeapStyle::Dynamic { bound_gv } => {
+            dynamic_addr(isa, inst, heap, offset, access_size, bound_gv, func)
+        }
+        ir::HeapStyle::Static { bound } => static_addr(
+            isa,
+            inst,
+            heap,
+            offset,
+            access_size,
+            bound.into(),
+            func,
+            cfg,
+        ),
+    }
+}
+
+/// Expand a `heap_addr` for a dynamic heap.
+fn dynamic_addr(
+    isa: &dyn TargetIsa,
+    inst: ir::Inst,
+    heap: ir::Heap,
+    offset: ir::Value,
+    access_size: u32,
+    bound_gv: ir::GlobalValue,
+    func: &mut ir::Function,
+) {
+    let access_size = u64::from(access_size);
+    let offset_ty = func.dfg.value_type(offset);
+    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
+    let min_size = func.heaps[heap].min_size.into();
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    // Start with the bounds check. Trap if `offset + access_size > bound`.
+    let bound = pos.ins().global_value(offset_ty, bound_gv);
+    let (cc, lhs, bound) = if access_size == 1 {
+        // `offset > bound - 1` is the same as `offset >= bound`.
+        (IntCC::UnsignedGreaterThanOrEqual, offset, bound)
+    } else if access_size <= min_size {
+        // We know that bound >= min_size, so here we can compare `offset > bound - access_size`
+        // without wrapping.
+        let adj_bound = pos.ins().iadd_imm(bound, -(access_size as i64));
+        (IntCC::UnsignedGreaterThan, offset, adj_bound)
+    } else {
+        // We need an overflow check for the adjusted offset.
+        let access_size_val = pos.ins().iconst(offset_ty, access_size as i64);
+        let (adj_offset, overflow) = pos.ins().iadd_ifcout(offset, access_size_val);
+        pos.ins().trapif(
+            isa.unsigned_add_overflow_condition(),
+            overflow,
+            ir::TrapCode::HeapOutOfBounds,
+        );
+        (IntCC::UnsignedGreaterThan, adj_offset, bound)
+    };
+    let oob = pos.ins().icmp(cc, lhs, bound);
+    pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
+
+    let spectre_oob_comparison = if isa.flags().enable_heap_access_spectre_mitigation() {
+        Some((cc, lhs, bound))
+    } else {
+        None
+    };
+
+    compute_addr(
+        isa,
+        inst,
+        heap,
+        addr_ty,
+        offset,
+        offset_ty,
+        pos.func,
+        spectre_oob_comparison,
+    );
+}
+
+/// Expand a `heap_addr` for a static heap.
+fn static_addr(
+    isa: &dyn TargetIsa,
+    inst: ir::Inst,
+    heap: ir::Heap,
+    offset: ir::Value,
+    access_size: u32,
+    bound: u64,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+) {
+    let access_size = u64::from(access_size);
+    let offset_ty = func.dfg.value_type(offset);
+    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    // The goal here is to trap if `offset + access_size > bound`.
+    //
+    // This first case is a trivial case where we can easily trap.
+    if access_size > bound {
+        // This will simply always trap since `offset >= 0`.
+        pos.ins().trap(ir::TrapCode::HeapOutOfBounds);
+        pos.func.dfg.replace(inst).iconst(addr_ty, 0);
+
+        // Split Block, as the trap is a terminator instruction.
+        let curr_block = pos.current_block().expect("Cursor is not in a block");
+        let new_block = pos.func.dfg.make_block();
+        pos.insert_block(new_block);
+        cfg.recompute_block(pos.func, curr_block);
+        cfg.recompute_block(pos.func, new_block);
+        return;
+    }
+
+    // After the trivial case is done we're now mostly interested in trapping
+    // if `offset > bound - access_size`. We know `bound - access_size` here is
+    // non-negative from the above comparison.
+    //
+    // If we can know `bound - access_size >= 4GB` then with a 32-bit offset
+    // we're guaranteed:
+    //
+    //      bound - access_size >= 4GB > offset
+    //
+    // or, in other words, `offset < bound - access_size`, meaning we can't trap
+    // for any value of `offset`.
+    //
+    // With that we have an optimization here where with 32-bit offsets and
+    // `bound - access_size >= 4GB` we can omit a bounds check.
+    let limit = bound - access_size;
+    let mut spectre_oob_comparison = None;
+    if offset_ty != ir::types::I32 || limit < 0xffff_ffff {
+        let (cc, lhs, limit_imm) = if limit & 1 == 1 {
+            // Prefer testing `offset >= limit - 1` when limit is odd because an even number is
+            // likely to be a convenient constant on ARM and other RISC architectures.
+            let limit = limit as i64 - 1;
+            (IntCC::UnsignedGreaterThanOrEqual, offset, limit)
+        } else {
+            let limit = limit as i64;
+            (IntCC::UnsignedGreaterThan, offset, limit)
+        };
+        let oob = pos.ins().icmp_imm(cc, lhs, limit_imm);
+        pos.ins().trapnz(oob, ir::TrapCode::HeapOutOfBounds);
+        if isa.flags().enable_heap_access_spectre_mitigation() {
+            let limit = pos.ins().iconst(offset_ty, limit_imm);
+            spectre_oob_comparison = Some((cc, lhs, limit));
+        }
+    }
+
+    compute_addr(
+        isa,
+        inst,
+        heap,
+        addr_ty,
+        offset,
+        offset_ty,
+        pos.func,
+        spectre_oob_comparison,
+    );
+}
+
+/// Emit code for the base address computation of a `heap_addr` instruction.
+fn compute_addr(
+    isa: &dyn TargetIsa,
+    inst: ir::Inst,
+    heap: ir::Heap,
+    addr_ty: ir::Type,
+    mut offset: ir::Value,
+    offset_ty: ir::Type,
+    func: &mut ir::Function,
+    // If we are performing Spectre mitigation with conditional selects, the
+    // values to compare and the condition code that indicates an out-of bounds
+    // condition; on this condition, the conditional move will choose a
+    // speculatively safe address (a zero / null pointer) instead.
+    spectre_oob_comparison: Option<(IntCC, ir::Value, ir::Value)>,
+) {
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    // Convert `offset` to `addr_ty`.
+    if offset_ty != addr_ty {
+        let labels_value = offset;
+        offset = pos.ins().uextend(addr_ty, offset);
+        if let Some(values_labels) = pos.func.dfg.values_labels.as_mut() {
+            values_labels.insert(
+                offset,
+                ir::ValueLabelAssignments::Alias {
+                    from: pos.func.srclocs[inst],
+                    value: labels_value,
+                },
+            );
+        }
+    }
+
+    // Add the heap base address base
+    let base = if isa.flags().enable_pinned_reg() && isa.flags().use_pinned_reg_as_heap_base() {
+        pos.ins().get_pinned_reg(isa.pointer_type())
+    } else {
+        let base_gv = pos.func.heaps[heap].base;
+        pos.ins().global_value(addr_ty, base_gv)
+    };
+
+    if let Some((cc, a, b)) = spectre_oob_comparison {
+        let final_addr = pos.ins().iadd(base, offset);
+        let zero = pos.ins().iconst(addr_ty, 0);
+        let flags = pos.ins().ifcmp(a, b);
+        pos.func
+            .dfg
+            .replace(inst)
+            .selectif_spectre_guard(addr_ty, cc, flags, zero, final_addr);
+    } else {
+        pos.func.dfg.replace(inst).iadd(base, offset);
+    }
+}
diff --git a/third_party/rust/cranelift-codegen/src/legalizer/libcall.rs b/third_party/rust/cranelift-codegen/src/legalizer/libcall.rs
new file mode 100644
index 0000000000..0973422a24
--- /dev/null
+++ b/third_party/rust/cranelift-codegen/src/legalizer/libcall.rs
@@ -0,0 +1,40 @@
+//! Expanding instructions as runtime library calls.
+
+use crate::ir;
+use crate::ir::{libcall::get_libcall_funcref, InstBuilder};
+use crate::isa::{CallConv, TargetIsa};
+use crate::legalizer::boundary::legalize_libcall_signature;
+use alloc::vec::Vec;
+
+/// Try to expand `inst` as a library call, returning true is successful.
+pub fn expand_as_libcall(inst: ir::Inst, func: &mut ir::Function, isa: &dyn TargetIsa) -> bool {
+    // Does the opcode/ctrl_type combo even have a well-known runtime library name.
+    let libcall = match ir::LibCall::for_inst(func.dfg[inst].opcode(), func.dfg.ctrl_typevar(inst))
+    {
+        Some(lc) => lc,
+        None => return false,
+    };
+
+    // Now we convert `inst` to a call. First save the arguments.
+    let mut args = Vec::new();
+    args.extend_from_slice(func.dfg.inst_args(inst));
+
+    let call_conv = CallConv::for_libcall(isa.flags(), isa.default_call_conv());
+    if call_conv.extends_baldrdash() {
+        let vmctx = func
+            .special_param(ir::ArgumentPurpose::VMContext)
+            .expect("Missing vmctx parameter for baldrdash libcall");
+        args.push(vmctx);
+    }
+
+    // The replace builder will preserve the instruction result values.
+    let funcref = get_libcall_funcref(libcall, call_conv, func, inst, isa);
+    func.dfg.replace(inst).call(funcref, &args);
+
+    // Ask the ISA to legalize the signature.
+    let fn_data = &func.dfg.ext_funcs[funcref];
+    let sig_data = &mut func.dfg.signatures[fn_data.signature];
+    legalize_libcall_signature(sig_data, isa);
+
+    true
+}
diff --git a/third_party/rust/cranelift-codegen/src/legalizer/mod.rs b/third_party/rust/cranelift-codegen/src/legalizer/mod.rs
new file mode 100644
index 0000000000..1900b144ce
--- /dev/null
+++ b/third_party/rust/cranelift-codegen/src/legalizer/mod.rs
@@ -0,0 +1,815 @@
+//! Legalize instructions.
+//!
+//! A legal instruction is one that can be mapped directly to a machine code instruction for the
+//! target ISA. The `legalize_function()` function takes as input any function and transforms it
+//! into an equivalent function using only legal instructions.
+//!
+//! The characteristics of legal instructions depend on the target ISA, so any given instruction
+//! can be legal for one ISA and illegal for another.
+//!
+//! Besides transforming instructions, the legalizer also fills out the `function.encodings` map
+//! which provides a legal encoding recipe for every instruction.
+//!
+//! The legalizer does not deal with register allocation constraints. These constraints are derived
+//! from the encoding recipes, and solved later by the register allocator.
+
+use crate::bitset::BitSet;
+use crate::cursor::{Cursor, FuncCursor};
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::types::{I32, I64};
+use crate::ir::{self, InstBuilder, MemFlags};
+use crate::isa::TargetIsa;
+
+#[cfg(any(
+    feature = "x86",
+    feature = "arm32",
+    feature = "arm64",
+    feature = "riscv"
+))]
+use crate::predicates;
+#[cfg(any(
+    feature = "x86",
+    feature = "arm32",
+    feature = "arm64",
+    feature = "riscv"
+))]
+use alloc::vec::Vec;
+
+use crate::timing;
+use alloc::collections::BTreeSet;
+
+mod boundary;
+mod call;
+mod globalvalue;
+mod heap;
+mod libcall;
+mod split;
+mod table;
+
+use self::call::expand_call;
+use self::globalvalue::expand_global_value;
+use self::heap::expand_heap_addr;
+pub(crate) use self::libcall::expand_as_libcall;
+use self::table::expand_table_addr;
+
+enum LegalizeInstResult {
+    Done,
+    Legalized,
+    SplitLegalizePending,
+}
+
+/// Legalize `inst` for `isa`.
+fn legalize_inst(
+    inst: ir::Inst,
+    pos: &mut FuncCursor,
+    cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) -> LegalizeInstResult {
+    let opcode = pos.func.dfg[inst].opcode();
+
+    // Check for ABI boundaries that need to be converted to the legalized signature.
+    if opcode.is_call() {
+        if boundary::handle_call_abi(isa, inst, pos.func, cfg) {
+            return LegalizeInstResult::Legalized;
+        }
+    } else if opcode.is_return() {
+        if boundary::handle_return_abi(inst, pos.func, cfg) {
+            return LegalizeInstResult::Legalized;
+        }
+    } else if opcode.is_branch() {
+        split::simplify_branch_arguments(&mut pos.func.dfg, inst);
+    } else if opcode == ir::Opcode::Isplit {
+        pos.use_srcloc(inst);
+
+        let arg = match pos.func.dfg[inst] {
+            ir::InstructionData::Unary { arg, .. } => pos.func.dfg.resolve_aliases(arg),
+            _ => panic!("Expected isplit: {}", pos.func.dfg.display_inst(inst, None)),
+        };
+
+        match pos.func.dfg.value_def(arg) {
+            ir::ValueDef::Result(inst, _num) => {
+                if let ir::InstructionData::Binary {
+                    opcode: ir::Opcode::Iconcat,
+                    ..
+                } = pos.func.dfg[inst]
+                {
+                    // `arg` was created by an `iconcat` instruction.
+                } else {
+                    // `arg` was not created by an `iconcat` instruction. Don't try to resolve it,
+                    // as otherwise `split::isplit` will re-insert the original `isplit`, causing
+                    // an endless loop.
+                    return LegalizeInstResult::SplitLegalizePending;
+                }
+            }
+            ir::ValueDef::Param(_block, _num) => {}
+        }
+
+        let res = pos.func.dfg.inst_results(inst).to_vec();
+        assert_eq!(res.len(), 2);
+        let (resl, resh) = (res[0], res[1]); // Prevent borrowck error
+
+        // Remove old isplit
+        pos.func.dfg.clear_results(inst);
+        pos.remove_inst();
+
+        let curpos = pos.position();
+        let srcloc = pos.srcloc();
+        let (xl, xh) = split::isplit(pos.func, cfg, curpos, srcloc, arg);
+
+        pos.func.dfg.change_to_alias(resl, xl);
+        pos.func.dfg.change_to_alias(resh, xh);
+
+        return LegalizeInstResult::Legalized;
+    }
+
+    match pos.func.update_encoding(inst, isa) {
+        Ok(()) => LegalizeInstResult::Done,
+        Err(action) => {
+            // We should transform the instruction into legal equivalents.
+            // If the current instruction was replaced, we need to double back and revisit
+            // the expanded sequence. This is both to assign encodings and possible to
+            // expand further.
+            // There's a risk of infinite looping here if the legalization patterns are
+            // unsound. Should we attempt to detect that?
+            if action(inst, pos.func, cfg, isa) {
+                return LegalizeInstResult::Legalized;
+            }
+
+            // We don't have any pattern expansion for this instruction either.
+            // Try converting it to a library call as a last resort.
+            if expand_as_libcall(inst, pos.func, isa) {
+                LegalizeInstResult::Legalized
+            } else {
+                LegalizeInstResult::Done
+            }
+        }
+    }
+}
+
+/// Legalize `func` for `isa`.
+///
+/// - Transform any instructions that don't have a legal representation in `isa`.
+/// - Fill out `func.encodings`.
+///
+pub fn legalize_function(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa: &dyn TargetIsa) {
+    let _tt = timing::legalize();
+    debug_assert!(cfg.is_valid());
+
+    boundary::legalize_signatures(func, isa);
+
+    func.encodings.resize(func.dfg.num_insts());
+
+    let mut pos = FuncCursor::new(func);
+    let func_begin = pos.position();
+
+    // Split block params before trying to legalize instructions, so that the newly introduced
+    // isplit instructions get legalized.
+    while let Some(block) = pos.next_block() {
+        split::split_block_params(pos.func, cfg, block);
+    }
+
+    pos.set_position(func_begin);
+
+    // This must be a set to prevent trying to legalize `isplit` and `vsplit` twice in certain cases.
+    let mut pending_splits = BTreeSet::new();
+
+    // Process blocks in layout order. Some legalization actions may split the current block or append
+    // new ones to the end. We need to make sure we visit those new blocks too.
+    while let Some(_block) = pos.next_block() {
+        // Keep track of the cursor position before the instruction being processed, so we can
+        // double back when replacing instructions.
+        let mut prev_pos = pos.position();
+
+        while let Some(inst) = pos.next_inst() {
+            match legalize_inst(inst, &mut pos, cfg, isa) {
+                // Remember this position in case we need to double back.
+                LegalizeInstResult::Done => prev_pos = pos.position(),
+
+                // Go back and legalize the inserted return value conversion instructions.
+                LegalizeInstResult::Legalized => pos.set_position(prev_pos),
+
+                // The argument of a `isplit` or `vsplit` instruction didn't resolve to a
+                // `iconcat` or `vconcat` instruction. Try again after legalizing the rest of
+                // the instructions.
+                LegalizeInstResult::SplitLegalizePending => {
+                    pending_splits.insert(inst);
+                }
+            }
+        }
+    }
+
+    // Try legalizing `isplit` and `vsplit` instructions, which could not previously be legalized.
+    for inst in pending_splits {
+        pos.goto_inst(inst);
+        legalize_inst(inst, &mut pos, cfg, isa);
+    }
+
+    // Now that we've lowered all br_tables, we don't need the jump tables anymore.
+    if !isa.flags().enable_jump_tables() {
+        pos.func.jump_tables.clear();
+    }
+}
+
+/// Perform a simple legalization by expansion of the function, without
+/// platform-specific transforms.
+pub fn simple_legalize(func: &mut ir::Function, cfg: &mut ControlFlowGraph, isa: &dyn TargetIsa) {
+    let mut pos = FuncCursor::new(func);
+    let func_begin = pos.position();
+    pos.set_position(func_begin);
+    while let Some(_block) = pos.next_block() {
+        let mut prev_pos = pos.position();
+        while let Some(inst) = pos.next_inst() {
+            let expanded = match pos.func.dfg[inst].opcode() {
+                ir::Opcode::BrIcmp
+                | ir::Opcode::GlobalValue
+                | ir::Opcode::HeapAddr
+                | ir::Opcode::StackLoad
+                | ir::Opcode::StackStore
+                | ir::Opcode::TableAddr
+                | ir::Opcode::Trapnz
+                | ir::Opcode::Trapz
+                | ir::Opcode::ResumableTrapnz
+                | ir::Opcode::BandImm
+                | ir::Opcode::BorImm
+                | ir::Opcode::BxorImm
+                | ir::Opcode::IaddImm
+                | ir::Opcode::IfcmpImm
+                | ir::Opcode::ImulImm
+                | ir::Opcode::IrsubImm
+                | ir::Opcode::IshlImm
+                | ir::Opcode::RotlImm
+                | ir::Opcode::RotrImm
+                | ir::Opcode::SdivImm
+                | ir::Opcode::SremImm
+                | ir::Opcode::SshrImm
+                | ir::Opcode::UdivImm
+                | ir::Opcode::UremImm
+                | ir::Opcode::UshrImm
+                | ir::Opcode::IcmpImm => expand(inst, &mut pos.func, cfg, isa),
+                _ => false,
+            };
+
+            if expanded {
+                // Legalization implementations require fixpoint loop
+                // here. TODO: fix this.
+                pos.set_position(prev_pos);
+            } else {
+                prev_pos = pos.position();
+            }
+        }
+    }
+}
+
+// Include legalization patterns that were generated by `gen_legalizer.rs` from the
+// `TransformGroup` in `cranelift-codegen/meta/shared/legalize.rs`.
+//
+// Concretely, this defines private functions `narrow()`, and `expand()`.
+include!(concat!(env!("OUT_DIR"), "/legalizer.rs"));
+
+/// Custom expansion for conditional trap instructions.
+/// TODO: Add CFG support to the Rust DSL patterns so we won't have to do this.
+fn expand_cond_trap(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    // Parse the instruction.
+    let trapz;
+    let (arg, code, opcode) = match func.dfg[inst] {
+        ir::InstructionData::CondTrap { opcode, arg, code } => {
+            // We want to branch *over* an unconditional trap.
+            trapz = match opcode {
+                ir::Opcode::Trapz => true,
+                ir::Opcode::Trapnz | ir::Opcode::ResumableTrapnz => false,
+                _ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst, None)),
+            };
+            (arg, code, opcode)
+        }
+        _ => panic!("Expected cond trap: {}", func.dfg.display_inst(inst, None)),
+    };
+
+    // Split the block after `inst`:
+    //
+    //     trapnz arg
+    //     ..
+    //
+    // Becomes:
+    //
+    //     brz arg, new_block_resume
+    //     jump new_block_trap
+    //
+    //   new_block_trap:
+    //     trap
+    //
+    //   new_block_resume:
+    //     ..
+    let old_block = func.layout.pp_block(inst);
+    let new_block_trap = func.dfg.make_block();
+    let new_block_resume = func.dfg.make_block();
+
+    // Replace trap instruction by the inverted condition.
+    if trapz {
+        func.dfg.replace(inst).brnz(arg, new_block_resume, &[]);
+    } else {
+        func.dfg.replace(inst).brz(arg, new_block_resume, &[]);
+    }
+
+    // Add jump instruction after the inverted branch.
+    let mut pos = FuncCursor::new(func).after_inst(inst);
+    pos.use_srcloc(inst);
+    pos.ins().jump(new_block_trap, &[]);
+
+    // Insert the new label and the unconditional trap terminator.
+    pos.insert_block(new_block_trap);
+
+    match opcode {
+        ir::Opcode::Trapz | ir::Opcode::Trapnz => {
+            pos.ins().trap(code);
+        }
+        ir::Opcode::ResumableTrapnz => {
+            pos.ins().resumable_trap(code);
+            pos.ins().jump(new_block_resume, &[]);
+        }
+        _ => unreachable!(),
+    }
+
+    // Insert the new label and resume the execution when the trap fails.
+    pos.insert_block(new_block_resume);
+
+    // Finally update the CFG.
+    cfg.recompute_block(pos.func, old_block);
+    cfg.recompute_block(pos.func, new_block_resume);
+    cfg.recompute_block(pos.func, new_block_trap);
+}
+
+/// Jump tables.
+fn expand_br_table(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) {
+    if isa.flags().enable_jump_tables() {
+        expand_br_table_jt(inst, func, cfg, isa);
+    } else {
+        expand_br_table_conds(inst, func, cfg, isa);
+    }
+}
+
+/// Expand br_table to jump table.
+fn expand_br_table_jt(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) {
+    use crate::ir::condcodes::IntCC;
+
+    let (arg, default_block, table) = match func.dfg[inst] {
+        ir::InstructionData::BranchTable {
+            opcode: ir::Opcode::BrTable,
+            arg,
+            destination,
+            table,
+        } => (arg, destination, table),
+        _ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)),
+    };
+
+    // Rewrite:
+    //
+    //     br_table $idx, default_block, $jt
+    //
+    // To:
+    //
+    //     $oob = ifcmp_imm $idx, len($jt)
+    //     brif uge $oob, default_block
+    //     jump fallthrough_block
+    //
+    //   fallthrough_block:
+    //     $base = jump_table_base.i64 $jt
+    //     $rel_addr = jump_table_entry.i64 $idx, $base, 4, $jt
+    //     $addr = iadd $base, $rel_addr
+    //     indirect_jump_table_br $addr, $jt
+
+    let block = func.layout.pp_block(inst);
+    let jump_table_block = func.dfg.make_block();
+
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    // Bounds check.
+    let table_size = pos.func.jump_tables[table].len() as i64;
+    let oob = pos
+        .ins()
+        .icmp_imm(IntCC::UnsignedGreaterThanOrEqual, arg, table_size);
+
+    pos.ins().brnz(oob, default_block, &[]);
+    pos.ins().jump(jump_table_block, &[]);
+    pos.insert_block(jump_table_block);
+
+    let addr_ty = isa.pointer_type();
+
+    let arg = if pos.func.dfg.value_type(arg) == addr_ty {
+        arg
+    } else {
+        pos.ins().uextend(addr_ty, arg)
+    };
+
+    let base_addr = pos.ins().jump_table_base(addr_ty, table);
+    let entry = pos
+        .ins()
+        .jump_table_entry(arg, base_addr, I32.bytes() as u8, table);
+
+    let addr = pos.ins().iadd(base_addr, entry);
+    pos.ins().indirect_jump_table_br(addr, table);
+
+    pos.remove_inst();
+    cfg.recompute_block(pos.func, block);
+    cfg.recompute_block(pos.func, jump_table_block);
+}
+
+/// Expand br_table to series of conditionals.
+fn expand_br_table_conds(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    use crate::ir::condcodes::IntCC;
+
+    let (arg, default_block, table) = match func.dfg[inst] {
+        ir::InstructionData::BranchTable {
+            opcode: ir::Opcode::BrTable,
+            arg,
+            destination,
+            table,
+        } => (arg, destination, table),
+        _ => panic!("Expected br_table: {}", func.dfg.display_inst(inst, None)),
+    };
+
+    let block = func.layout.pp_block(inst);
+
+    // This is a poor man's jump table using just a sequence of conditional branches.
+    let table_size = func.jump_tables[table].len();
+    let mut cond_failed_block = vec![];
+    if table_size >= 1 {
+        cond_failed_block = alloc::vec::Vec::with_capacity(table_size - 1);
+        for _ in 0..table_size - 1 {
+            cond_failed_block.push(func.dfg.make_block());
+        }
+    }
+
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    // Ignore the lint for this loop as the range needs to be 0 to table_size
+    #[allow(clippy::needless_range_loop)]
+    for i in 0..table_size {
+        let dest = pos.func.jump_tables[table].as_slice()[i];
+        let t = pos.ins().icmp_imm(IntCC::Equal, arg, i as i64);
+        pos.ins().brnz(t, dest, &[]);
+        // Jump to the next case.
+        if i < table_size - 1 {
+            let block = cond_failed_block[i];
+            pos.ins().jump(block, &[]);
+            pos.insert_block(block);
+        }
+    }
+
+    // `br_table` jumps to the default destination if nothing matches
+    pos.ins().jump(default_block, &[]);
+
+    pos.remove_inst();
+    cfg.recompute_block(pos.func, block);
+    for failed_block in cond_failed_block.into_iter() {
+        cfg.recompute_block(pos.func, failed_block);
+    }
+}
+
+/// Expand the select instruction.
+///
+/// Conditional moves are available in some ISAs for some register classes. The remaining selects
+/// are handled by a branch.
+fn expand_select(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    let (ctrl, tval, fval) = match func.dfg[inst] {
+        ir::InstructionData::Ternary {
+            opcode: ir::Opcode::Select,
+            args,
+        } => (args[0], args[1], args[2]),
+        _ => panic!("Expected select: {}", func.dfg.display_inst(inst, None)),
+    };
+
+    // Replace `result = select ctrl, tval, fval` with:
+    //
+    //   brnz ctrl, new_block(tval)
+    //   jump new_block(fval)
+    // new_block(result):
+    let old_block = func.layout.pp_block(inst);
+    let result = func.dfg.first_result(inst);
+    func.dfg.clear_results(inst);
+    let new_block = func.dfg.make_block();
+    func.dfg.attach_block_param(new_block, result);
+
+    func.dfg.replace(inst).brnz(ctrl, new_block, &[tval]);
+    let mut pos = FuncCursor::new(func).after_inst(inst);
+    pos.use_srcloc(inst);
+    pos.ins().jump(new_block, &[fval]);
+    pos.insert_block(new_block);
+
+    cfg.recompute_block(pos.func, new_block);
+    cfg.recompute_block(pos.func, old_block);
+}
+
+fn expand_br_icmp(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    let (cond, a, b, destination, block_args) = match func.dfg[inst] {
+        ir::InstructionData::BranchIcmp {
+            cond,
+            destination,
+            ref args,
+            ..
+        } => (
+            cond,
+            args.get(0, &func.dfg.value_lists).unwrap(),
+            args.get(1, &func.dfg.value_lists).unwrap(),
+            destination,
+            args.as_slice(&func.dfg.value_lists)[2..].to_vec(),
+        ),
+        _ => panic!("Expected br_icmp {}", func.dfg.display_inst(inst, None)),
+    };
+
+    let old_block = func.layout.pp_block(inst);
+    func.dfg.clear_results(inst);
+
+    let icmp_res = func.dfg.replace(inst).icmp(cond, a, b);
+    let mut pos = FuncCursor::new(func).after_inst(inst);
+    pos.use_srcloc(inst);
+    pos.ins().brnz(icmp_res, destination, &block_args);
+
+    cfg.recompute_block(pos.func, destination);
+    cfg.recompute_block(pos.func, old_block);
+}
+
+/// Expand illegal `f32const` and `f64const` instructions.
+fn expand_fconst(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    let ty = func.dfg.value_type(func.dfg.first_result(inst));
+    debug_assert!(!ty.is_vector(), "Only scalar fconst supported: {}", ty);
+
+    // In the future, we may want to generate constant pool entries for these constants, but for
+    // now use an `iconst` and a bit cast.
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+    let ival = match pos.func.dfg[inst] {
+        ir::InstructionData::UnaryIeee32 {
+            opcode: ir::Opcode::F32const,
+            imm,
+        } => pos.ins().iconst(ir::types::I32, i64::from(imm.bits())),
+        ir::InstructionData::UnaryIeee64 {
+            opcode: ir::Opcode::F64const,
+            imm,
+        } => pos.ins().iconst(ir::types::I64, imm.bits() as i64),
+        _ => panic!("Expected fconst: {}", pos.func.dfg.display_inst(inst, None)),
+    };
+    pos.func.dfg.replace(inst).bitcast(ty, ival);
+}
+
+/// Expand illegal `stack_load` instructions.
+fn expand_stack_load(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) {
+    let ty = func.dfg.value_type(func.dfg.first_result(inst));
+    let addr_ty = isa.pointer_type();
+
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    let (stack_slot, offset) = match pos.func.dfg[inst] {
+        ir::InstructionData::StackLoad {
+            opcode: _opcode,
+            stack_slot,
+            offset,
+        } => (stack_slot, offset),
+        _ => panic!(
+            "Expected stack_load: {}",
+            pos.func.dfg.display_inst(inst, None)
+        ),
+    };
+
+    let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset);
+
+    // Stack slots are required to be accessible and aligned.
+    let mflags = MemFlags::trusted();
+    pos.func.dfg.replace(inst).load(ty, mflags, addr, 0);
+}
+
+/// Expand illegal `stack_store` instructions.
+fn expand_stack_store(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) {
+    let addr_ty = isa.pointer_type();
+
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    let (val, stack_slot, offset) = match pos.func.dfg[inst] {
+        ir::InstructionData::StackStore {
+            opcode: _opcode,
+            arg,
+            stack_slot,
+            offset,
+        } => (arg, stack_slot, offset),
+        _ => panic!(
+            "Expected stack_store: {}",
+            pos.func.dfg.display_inst(inst, None)
+        ),
+    };
+
+    let addr = pos.ins().stack_addr(addr_ty, stack_slot, offset);
+
+    let mut mflags = MemFlags::new();
+    // Stack slots are required to be accessible and aligned.
+    mflags.set_notrap();
+    mflags.set_aligned();
+    pos.func.dfg.replace(inst).store(mflags, val, addr, 0);
+}
+
+/// Split a load into two parts before `iconcat`ing the result together.
+fn narrow_load(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    let (ptr, offset, flags) = match pos.func.dfg[inst] {
+        ir::InstructionData::Load {
+            opcode: ir::Opcode::Load,
+            arg,
+            offset,
+            flags,
+        } => (arg, offset, flags),
+        _ => panic!("Expected load: {}", pos.func.dfg.display_inst(inst, None)),
+    };
+
+    let res_ty = pos.func.dfg.ctrl_typevar(inst);
+    let small_ty = res_ty.half_width().expect("Can't narrow load");
+
+    let al = pos.ins().load(small_ty, flags, ptr, offset);
+    let ah = pos.ins().load(
+        small_ty,
+        flags,
+        ptr,
+        offset.try_add_i64(8).expect("load offset overflow"),
+    );
+    pos.func.dfg.replace(inst).iconcat(al, ah);
+}
+
+/// Split a store into two parts after `isplit`ing the value.
+fn narrow_store(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    let (val, ptr, offset, flags) = match pos.func.dfg[inst] {
+        ir::InstructionData::Store {
+            opcode: ir::Opcode::Store,
+            args,
+            offset,
+            flags,
+        } => (args[0], args[1], offset, flags),
+        _ => panic!("Expected store: {}", pos.func.dfg.display_inst(inst, None)),
+    };
+
+    let (al, ah) = pos.ins().isplit(val);
+    pos.ins().store(flags, al, ptr, offset);
+    pos.ins().store(
+        flags,
+        ah,
+        ptr,
+        offset.try_add_i64(8).expect("store offset overflow"),
+    );
+    pos.remove_inst();
+}
+
+/// Expands an illegal iconst value by splitting it into two.
+fn narrow_iconst(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    isa: &dyn TargetIsa,
+) {
+    let imm: i64 = if let ir::InstructionData::UnaryImm {
+        opcode: ir::Opcode::Iconst,
+        imm,
+    } = &func.dfg[inst]
+    {
+        (*imm).into()
+    } else {
+        panic!("unexpected instruction in narrow_iconst");
+    };
+
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    let ty = pos.func.dfg.ctrl_typevar(inst);
+    if isa.pointer_bits() == 32 && ty == I64 {
+        let low = pos.ins().iconst(I32, imm & 0xffffffff);
+        let high = pos.ins().iconst(I32, imm >> 32);
+        // The instruction has as many results as iconcat, so no need to replace them.
+        pos.func.dfg.replace(inst).iconcat(low, high);
+        return;
+    }
+
+    unimplemented!("missing encoding or legalization for iconst.{:?}", ty);
+}
+
+fn narrow_icmp_imm(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    use crate::ir::condcodes::{CondCode, IntCC};
+
+    let (arg, cond, imm): (ir::Value, IntCC, i64) = match func.dfg[inst] {
+        ir::InstructionData::IntCompareImm {
+            opcode: ir::Opcode::IcmpImm,
+            arg,
+            cond,
+            imm,
+        } => (arg, cond, imm.into()),
+        _ => panic!("unexpected instruction in narrow_icmp_imm"),
+    };
+
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    let ty = pos.func.dfg.ctrl_typevar(inst);
+    let ty_half = ty.half_width().unwrap();
+
+    let mask = ((1u128 << ty_half.bits()) - 1) as i64;
+    let imm_low = pos.ins().iconst(ty_half, imm & mask);
+    let imm_high = pos.ins().iconst(
+        ty_half,
+        imm.checked_shr(ty_half.bits().into()).unwrap_or(0) & mask,
+    );
+    let (arg_low, arg_high) = pos.ins().isplit(arg);
+
+    match cond {
+        IntCC::Equal => {
+            let res_low = pos.ins().icmp(cond, arg_low, imm_low);
+            let res_high = pos.ins().icmp(cond, arg_high, imm_high);
+            pos.func.dfg.replace(inst).band(res_low, res_high);
+        }
+        IntCC::NotEqual => {
+            let res_low = pos.ins().icmp(cond, arg_low, imm_low);
+            let res_high = pos.ins().icmp(cond, arg_high, imm_high);
+            pos.func.dfg.replace(inst).bor(res_low, res_high);
+        }
+        IntCC::SignedGreaterThan
+        | IntCC::SignedGreaterThanOrEqual
+        | IntCC::SignedLessThan
+        | IntCC::SignedLessThanOrEqual
+        | IntCC::UnsignedGreaterThan
+        | IntCC::UnsignedGreaterThanOrEqual
+        | IntCC::UnsignedLessThan
+        | IntCC::UnsignedLessThanOrEqual => {
+            let b1 = pos.ins().icmp(cond.without_equal(), arg_high, imm_high);
+            let b2 = pos
+                .ins()
+                .icmp(cond.inverse().without_equal(), arg_high, imm_high);
+            let b3 = pos.ins().icmp(cond.unsigned(), arg_low, imm_low);
+            let c1 = pos.ins().bnot(b2);
+            let c2 = pos.ins().band(c1, b3);
+            pos.func.dfg.replace(inst).bor(b1, c2);
+        }
+        _ => unimplemented!("missing legalization for condition {:?}", cond),
+    }
+}
diff --git a/third_party/rust/cranelift-codegen/src/legalizer/split.rs b/third_party/rust/cranelift-codegen/src/legalizer/split.rs
new file mode 100644
index 0000000000..7576926142
--- /dev/null
+++ b/third_party/rust/cranelift-codegen/src/legalizer/split.rs
@@ -0,0 +1,405 @@
+//! Value splitting.
+//!
+//! Some value types are too large to fit in registers, so they need to be split into smaller parts
+//! that the ISA can operate on. There's two dimensions of splitting, represented by two
+//! complementary instruction pairs:
+//!
+//! - `isplit` and `iconcat` for splitting integer types into smaller integers.
+//! - `vsplit` and `vconcat` for splitting vector types into smaller vector types with the same
+//!   lane types.
+//!
+//! There is no floating point splitting. If an ISA doesn't support `f64` values, they probably
+//! have to be bit-cast to `i64` and possibly split into two `i32` values that fit in registers.
+//! This breakdown is handled by the ABI lowering.
+//!
+//! When legalizing a single instruction, it is wrapped in splits and concatenations:
+//!
+//! ```clif
+//!     v1 = bxor.i64 v2, v3
+//! ```
+//!
+//! becomes:
+//!
+//! ```clif
+//!     v20, v21 = isplit v2
+//!     v30, v31 = isplit v3
+//!     v10 = bxor.i32 v20, v30
+//!     v11 = bxor.i32 v21, v31
+//!     v1 = iconcat v10, v11
+//! ```
+//!
+//! This local expansion approach still leaves the original `i64` values in the code as operands on
+//! the `split` and `concat` instructions. It also creates a lot of redundant code to clean up as
+//! values are constantly split and concatenated.
+//!
+//! # Optimized splitting
+//!
+//! We can eliminate a lot of the splitting code quite easily. Whenever we need to split a value,
+//! first check if the value is defined by the corresponding concatenation. If so, then just use
+//! the two concatenation inputs directly:
+//!
+//! ```clif
+//!     v4 = iadd_imm.i64 v1, 1
+//! ```
+//!
+//! becomes, using the expanded code from above:
+//!
+//! ```clif
+//!     v40, v5 = iadd_imm_cout.i32 v10, 1
+//!     v6 = bint.i32
+//!     v41 = iadd.i32 v11, v6
+//!     v4 = iconcat v40, v41
+//! ```
+//!
+//! This means that the `iconcat` instructions defining `v1` and `v4` end up with no uses, so they
+//! can be trivially deleted by a dead code elimination pass.
+//!
+//! # block arguments
+//!
+//! If all instructions that produce an `i64` value are legalized as above, we will eventually end
+//! up with no `i64` values anywhere, except for block arguments. We can work around this by
+//! iteratively splitting block arguments too. That should leave us with no illegal value types
+//! anywhere.
+//!
+//! It is possible to have circular dependencies of block arguments that are never used by any real
+//! instructions. These loops will remain in the program.
+
+use crate::cursor::{Cursor, CursorPosition, FuncCursor};
+use crate::flowgraph::{BlockPredecessor, ControlFlowGraph};
+use crate::ir::{self, Block, Inst, InstBuilder, InstructionData, Opcode, Type, Value, ValueDef};
+use alloc::vec::Vec;
+use core::iter;
+use smallvec::SmallVec;
+
+/// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values
+/// if possible.
+pub fn isplit(
+    func: &mut ir::Function,
+    cfg: &ControlFlowGraph,
+    pos: CursorPosition,
+    srcloc: ir::SourceLoc,
+    value: Value,
+) -> (Value, Value) {
+    split_any(func, cfg, pos, srcloc, value, Opcode::Iconcat)
+}
+
+/// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if
+/// possible.
+pub fn vsplit(
+    func: &mut ir::Function,
+    cfg: &ControlFlowGraph,
+    pos: CursorPosition,
+    srcloc: ir::SourceLoc,
+    value: Value,
+) -> (Value, Value) {
+    split_any(func, cfg, pos, srcloc, value, Opcode::Vconcat)
+}
+
+/// After splitting a block argument, we need to go back and fix up all of the predecessor
+/// instructions. This is potentially a recursive operation, but we don't implement it recursively
+/// since that could use up too muck stack.
+///
+/// Instead, the repairs are deferred and placed on a work list in stack form.
+struct Repair {
+    concat: Opcode,
+    // The argument type after splitting.
+    split_type: Type,
+    // The destination block whose arguments have been split.
+    block: Block,
+    // Number of the original block argument which has been replaced by the low part.
+    num: usize,
+    // Number of the new block argument which represents the high part after the split.
+    hi_num: usize,
+}
+
+/// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode.
+fn split_any(
+    func: &mut ir::Function,
+    cfg: &ControlFlowGraph,
+    pos: CursorPosition,
+    srcloc: ir::SourceLoc,
+    value: Value,
+    concat: Opcode,
+) -> (Value, Value) {
+    let mut repairs = Vec::new();
+    let pos = &mut FuncCursor::new(func).at_position(pos).with_srcloc(srcloc);
+    let result = split_value(pos, value, concat, &mut repairs);
+
+    perform_repairs(pos, cfg, repairs);
+
+    result
+}
+
+pub fn split_block_params(func: &mut ir::Function, cfg: &ControlFlowGraph, block: Block) {
+    let pos = &mut FuncCursor::new(func).at_top(block);
+    let block_params = pos.func.dfg.block_params(block);
+
+    // Add further splittable types here.
+    fn type_requires_splitting(ty: Type) -> bool {
+        ty == ir::types::I128
+    }
+
+    // A shortcut.  If none of the param types require splitting, exit now.  This helps because
+    // the loop below necessarily has to copy the block params into a new vector, so it's better to
+    // avoid doing so when possible.
+    if !block_params
+        .iter()
+        .any(|block_param| type_requires_splitting(pos.func.dfg.value_type(*block_param)))
+    {
+        return;
+    }
+
+    let mut repairs = Vec::new();
+    for (num, block_param) in block_params.to_vec().into_iter().enumerate() {
+        if !type_requires_splitting(pos.func.dfg.value_type(block_param)) {
+            continue;
+        }
+
+        split_block_param(pos, block, num, block_param, Opcode::Iconcat, &mut repairs);
+    }
+
+    perform_repairs(pos, cfg, repairs);
+}
+
+fn perform_repairs(pos: &mut FuncCursor, cfg: &ControlFlowGraph, mut repairs: Vec<Repair>) {
+    // We have split the value requested, and now we may need to fix some block predecessors.
+    while let Some(repair) = repairs.pop() {
+        for BlockPredecessor { inst, .. } in cfg.pred_iter(repair.block) {
+            let branch_opc = pos.func.dfg[inst].opcode();
+            debug_assert!(
+                branch_opc.is_branch(),
+                "Predecessor not a branch: {}",
+                pos.func.dfg.display_inst(inst, None)
+            );
+            let num_fixed_args = branch_opc.constraints().num_fixed_value_arguments();
+            let mut args = pos.func.dfg[inst]
+                .take_value_list()
+                .expect("Branches must have value lists.");
+            let num_args = args.len(&pos.func.dfg.value_lists);
+            // Get the old value passed to the block argument we're repairing.
+            let old_arg = args
+                .get(num_fixed_args + repair.num, &pos.func.dfg.value_lists)
+                .expect("Too few branch arguments");
+
+            // It's possible that the CFG's predecessor list has duplicates. Detect them here.
+            if pos.func.dfg.value_type(old_arg) == repair.split_type {
+                pos.func.dfg[inst].put_value_list(args);
+                continue;
+            }
+
+            // Split the old argument, possibly causing more repairs to be scheduled.
+            pos.goto_inst(inst);
+
+            let inst_block = pos.func.layout.inst_block(inst).expect("inst in block");
+
+            // Insert split values prior to the terminal branch group.
+            let canonical = pos
+                .func
+                .layout
+                .canonical_branch_inst(&pos.func.dfg, inst_block);
+            if let Some(first_branch) = canonical {
+                pos.goto_inst(first_branch);
+            }
+
+            let (lo, hi) = split_value(pos, old_arg, repair.concat, &mut repairs);
+
+            // The `lo` part replaces the original argument.
+            *args
+                .get_mut(num_fixed_args + repair.num, &mut pos.func.dfg.value_lists)
+                .unwrap() = lo;
+
+            // The `hi` part goes at the end. Since multiple repairs may have been scheduled to the
+            // same block, there could be multiple arguments missing.
+            if num_args > num_fixed_args + repair.hi_num {
+                *args
+                    .get_mut(
+                        num_fixed_args + repair.hi_num,
+                        &mut pos.func.dfg.value_lists,
+                    )
+                    .unwrap() = hi;
+            } else {
+                // We need to append one or more arguments. If we're adding more than one argument,
+                // there must be pending repairs on the stack that will fill in the correct values
+                // instead of `hi`.
+                args.extend(
+                    iter::repeat(hi).take(1 + num_fixed_args + repair.hi_num - num_args),
+                    &mut pos.func.dfg.value_lists,
+                );
+            }
+
+            // Put the value list back after manipulating it.
+            pos.func.dfg[inst].put_value_list(args);
+        }
+    }
+}
+
+/// Split a single value using the integer or vector semantics given by the `concat` opcode.
+///
+/// If the value is defined by a `concat` instruction, just reuse the operand values of that
+/// instruction.
+///
+/// Return the two new values representing the parts of `value`.
+fn split_value(
+    pos: &mut FuncCursor,
+    value: Value,
+    concat: Opcode,
+    repairs: &mut Vec<Repair>,
+) -> (Value, Value) {
+    let value = pos.func.dfg.resolve_aliases(value);
+    let mut reuse = None;
+
+    match pos.func.dfg.value_def(value) {
+        ValueDef::Result(inst, num) => {
+            // This is an instruction result. See if the value was created by a `concat`
+            // instruction.
+            if let InstructionData::Binary { opcode, args, .. } = pos.func.dfg[inst] {
+                debug_assert_eq!(num, 0);
+                if opcode == concat {
+                    reuse = Some((args[0], args[1]));
+                }
+            }
+        }
+        ValueDef::Param(block, num) => {
+            // This is a block parameter.
+            // We can split the parameter value unless this is the entry block.
+            if pos.func.layout.entry_block() != Some(block) {
+                reuse = Some(split_block_param(pos, block, num, value, concat, repairs));
+            }
+        }
+    }
+
+    // Did the code above succeed in finding values we can reuse?
+    if let Some(pair) = reuse {
+        pair
+    } else {
+        // No, we'll just have to insert the requested split instruction at `pos`. Note that `pos`
+        // has not been moved by the block argument code above when `reuse` is `None`.
+        match concat {
+            Opcode::Iconcat => pos.ins().isplit(value),
+            Opcode::Vconcat => pos.ins().vsplit(value),
+            _ => panic!("Unhandled concat opcode: {}", concat),
+        }
+    }
+}
+
+fn split_block_param(
+    pos: &mut FuncCursor,
+    block: Block,
+    param_num: usize,
+    value: Value,
+    concat: Opcode,
+    repairs: &mut Vec<Repair>,
+) -> (Value, Value) {
+    // We are going to replace the parameter at `num` with two new arguments.
+    // Determine the new value types.
+    let ty = pos.func.dfg.value_type(value);
+    let split_type = match concat {
+        Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"),
+        Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"),
+        _ => panic!("Unhandled concat opcode: {}", concat),
+    };
+
+    // Since the `repairs` stack potentially contains other parameter numbers for
+    // `block`, avoid shifting and renumbering block parameters. It could invalidate other
+    // `repairs` entries.
+    //
+    // Replace the original `value` with the low part, and append the high part at the
+    // end of the argument list.
+    let lo = pos.func.dfg.replace_block_param(value, split_type);
+    let hi_num = pos.func.dfg.num_block_params(block);
+    let hi = pos.func.dfg.append_block_param(block, split_type);
+
+    // Now the original value is dangling. Insert a concatenation instruction that can
+    // compute it from the two new parameters. This also serves as a record of what we
+    // did so a future call to this function doesn't have to redo the work.
+    //
+    // Note that it is safe to move `pos` here since `reuse` was set above, so we don't
+    // need to insert a split instruction before returning.
+    pos.goto_first_inst(block);
+    pos.ins()
+        .with_result(value)
+        .Binary(concat, split_type, lo, hi);
+
+    // Finally, splitting the block parameter is not enough. We also have to repair all
+    // of the predecessor instructions that branch here.
+    add_repair(concat, split_type, block, param_num, hi_num, repairs);
+
+    (lo, hi)
+}
+
+// Add a repair entry to the work list.
+fn add_repair(
+    concat: Opcode,
+    split_type: Type,
+    block: Block,
+    num: usize,
+    hi_num: usize,
+    repairs: &mut Vec<Repair>,
+) {
+    repairs.push(Repair {
+        concat,
+        split_type,
+        block,
+        num,
+        hi_num,
+    });
+}
+
+/// Strip concat-split chains. Return a simpler way of computing the same value.
+///
+/// Given this input:
+///
+/// ```clif
+///     v10 = iconcat v1, v2
+///     v11, v12 = isplit v10
+/// ```
+///
+/// This function resolves `v11` to `v1` and `v12` to `v2`.
+fn resolve_splits(dfg: &ir::DataFlowGraph, value: Value) -> Value {
+    let value = dfg.resolve_aliases(value);
+
+    // Deconstruct a split instruction.
+    let split_res;
+    let concat_opc;
+    let split_arg;
+    if let ValueDef::Result(inst, num) = dfg.value_def(value) {
+        split_res = num;
+        concat_opc = match dfg[inst].opcode() {
+            Opcode::Isplit => Opcode::Iconcat,
+            Opcode::Vsplit => Opcode::Vconcat,
+            _ => return value,
+        };
+        split_arg = dfg.inst_args(inst)[0];
+    } else {
+        return value;
+    }
+
+    // See if split_arg is defined by a concatenation instruction.
+    if let ValueDef::Result(inst, _) = dfg.value_def(split_arg) {
+        if dfg[inst].opcode() == concat_opc {
+            return dfg.inst_args(inst)[split_res];
+        }
+    }
+
+    value
+}
+
+/// Simplify the arguments to a branch *after* the instructions leading up to the branch have been
+/// legalized.
+///
+/// The branch argument repairs performed by `split_any()` above may be performed on branches that
+/// have not yet been legalized. The repaired arguments can be defined by actual split
+/// instructions in that case.
+///
+/// After legalizing the instructions computing the value that was split, it is likely that we can
+/// avoid depending on the split instruction. Its input probably comes from a concatenation.
+pub fn simplify_branch_arguments(dfg: &mut ir::DataFlowGraph, branch: Inst) {
+    let mut new_args = SmallVec::<[Value; 32]>::new();
+
+    for &arg in dfg.inst_args(branch) {
+        let new_arg = resolve_splits(dfg, arg);
+        new_args.push(new_arg);
+    }
+
+    dfg.inst_args_mut(branch).copy_from_slice(&new_args);
+}
diff --git a/third_party/rust/cranelift-codegen/src/legalizer/table.rs b/third_party/rust/cranelift-codegen/src/legalizer/table.rs
new file mode 100644
index 0000000000..0c4385e96b
--- /dev/null
+++ b/third_party/rust/cranelift-codegen/src/legalizer/table.rs
@@ -0,0 +1,113 @@
+//! Legalization of tables.
+//!
+//! This module exports the `expand_table_addr` function which transforms a `table_addr`
+//! instruction into code that depends on the kind of table referenced.
+
+use crate::cursor::{Cursor, FuncCursor};
+use crate::flowgraph::ControlFlowGraph;
+use crate::ir::condcodes::IntCC;
+use crate::ir::immediates::Offset32;
+use crate::ir::{self, InstBuilder};
+use crate::isa::TargetIsa;
+
+/// Expand a `table_addr` instruction according to the definition of the table.
+pub fn expand_table_addr(
+    inst: ir::Inst,
+    func: &mut ir::Function,
+    _cfg: &mut ControlFlowGraph,
+    _isa: &dyn TargetIsa,
+) {
+    // Unpack the instruction.
+    let (table, index, element_offset) = match func.dfg[inst] {
+        ir::InstructionData::TableAddr {
+            opcode,
+            table,
+            arg,
+            offset,
+        } => {
+            debug_assert_eq!(opcode, ir::Opcode::TableAddr);
+            (table, arg, offset)
+        }
+        _ => panic!("Wanted table_addr: {}", func.dfg.display_inst(inst, None)),
+    };
+
+    dynamic_addr(inst, table, index, element_offset, func);
+}
+
+/// Expand a `table_addr` for a dynamic table.
+fn dynamic_addr(
+    inst: ir::Inst,
+    table: ir::Table,
+    index: ir::Value,
+    element_offset: Offset32,
+    func: &mut ir::Function,
+) {
+    let bound_gv = func.tables[table].bound_gv;
+    let index_ty = func.dfg.value_type(index);
+    let addr_ty = func.dfg.value_type(func.dfg.first_result(inst));
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    // Start with the bounds check. Trap if `index + 1 > bound`.
+    let bound = pos.ins().global_value(index_ty, bound_gv);
+
+    // `index > bound - 1` is the same as `index >= bound`.
+    let oob = pos
+        .ins()
+        .icmp(IntCC::UnsignedGreaterThanOrEqual, index, bound);
+    pos.ins().trapnz(oob, ir::TrapCode::TableOutOfBounds);
+
+    compute_addr(
+        inst,
+        table,
+        addr_ty,
+        index,
+        index_ty,
+        element_offset,
+        pos.func,
+    );
+}
+
+/// Emit code for the base address computation of a `table_addr` instruction.
+fn compute_addr(
+    inst: ir::Inst,
+    table: ir::Table,
+    addr_ty: ir::Type,
+    mut index: ir::Value,
+    index_ty: ir::Type,
+    element_offset: Offset32,
+    func: &mut ir::Function,
+) {
+    let mut pos = FuncCursor::new(func).at_inst(inst);
+    pos.use_srcloc(inst);
+
+    // Convert `index` to `addr_ty`.
+    if index_ty != addr_ty {
+        index = pos.ins().uextend(addr_ty, index);
+    }
+
+    // Add the table base address base
+    let base_gv = pos.func.tables[table].base_gv;
+    let base = pos.ins().global_value(addr_ty, base_gv);
+
+    let element_size = pos.func.tables[table].element_size;
+    let mut offset;
+    let element_size: u64 = element_size.into();
+    if element_size == 1 {
+        offset = index;
+    } else if element_size.is_power_of_two() {
+        offset = pos
+            .ins()
+            .ishl_imm(index, i64::from(element_size.trailing_zeros()));
+    } else {
+        offset = pos.ins().imul_imm(index, element_size as i64);
+    }
+
+    if element_offset == Offset32::new(0) {
+        pos.func.dfg.replace(inst).iadd(base, offset);
+    } else {
+        let imm: i64 = element_offset.into();
+        offset = pos.ins().iadd(base, offset);
+        pos.func.dfg.replace(inst).iadd_imm(offset, imm);
+    }
+}