//! Value splitting. //! //! Some value types are too large to fit in registers, so they need to be split into smaller parts //! that the ISA can operate on. There's two dimensions of splitting, represented by two //! complementary instruction pairs: //! //! - `isplit` and `iconcat` for splitting integer types into smaller integers. //! - `vsplit` and `vconcat` for splitting vector types into smaller vector types with the same //! lane types. //! //! There is no floating point splitting. If an ISA doesn't support `f64` values, they probably //! have to be bit-cast to `i64` and possibly split into two `i32` values that fit in registers. //! This breakdown is handled by the ABI lowering. //! //! When legalizing a single instruction, it is wrapped in splits and concatenations: //! //! ```clif //! v1 = bxor.i64 v2, v3 //! ``` //! //! becomes: //! //! ```clif //! v20, v21 = isplit v2 //! v30, v31 = isplit v3 //! v10 = bxor.i32 v20, v30 //! v11 = bxor.i32 v21, v31 //! v1 = iconcat v10, v11 //! ``` //! //! This local expansion approach still leaves the original `i64` values in the code as operands on //! the `split` and `concat` instructions. It also creates a lot of redundant code to clean up as //! values are constantly split and concatenated. //! //! # Optimized splitting //! //! We can eliminate a lot of the splitting code quite easily. Whenever we need to split a value, //! first check if the value is defined by the corresponding concatenation. If so, then just use //! the two concatenation inputs directly: //! //! ```clif //! v4 = iadd_imm.i64 v1, 1 //! ``` //! //! becomes, using the expanded code from above: //! //! ```clif //! v40, v5 = iadd_imm_cout.i32 v10, 1 //! v6 = bint.i32 //! v41 = iadd.i32 v11, v6 //! v4 = iconcat v40, v41 //! ``` //! //! This means that the `iconcat` instructions defining `v1` and `v4` end up with no uses, so they //! can be trivially deleted by a dead code elimination pass. //! //! # block arguments //! //! If all instructions that produce an `i64` value are legalized as above, we will eventually end //! up with no `i64` values anywhere, except for block arguments. We can work around this by //! iteratively splitting block arguments too. That should leave us with no illegal value types //! anywhere. //! //! It is possible to have circular dependencies of block arguments that are never used by any real //! instructions. These loops will remain in the program. use crate::cursor::{Cursor, CursorPosition, FuncCursor}; use crate::flowgraph::{BlockPredecessor, ControlFlowGraph}; use crate::ir::{self, Block, Inst, InstBuilder, InstructionData, Opcode, Type, Value, ValueDef}; use alloc::vec::Vec; use core::iter; use smallvec::SmallVec; /// Split `value` into two values using the `isplit` semantics. Do this by reusing existing values /// if possible. pub fn isplit( func: &mut ir::Function, cfg: &ControlFlowGraph, pos: CursorPosition, srcloc: ir::SourceLoc, value: Value, ) -> (Value, Value) { split_any(func, cfg, pos, srcloc, value, Opcode::Iconcat) } /// Split `value` into halves using the `vsplit` semantics. Do this by reusing existing values if /// possible. pub fn vsplit( func: &mut ir::Function, cfg: &ControlFlowGraph, pos: CursorPosition, srcloc: ir::SourceLoc, value: Value, ) -> (Value, Value) { split_any(func, cfg, pos, srcloc, value, Opcode::Vconcat) } /// After splitting a block argument, we need to go back and fix up all of the predecessor /// instructions. This is potentially a recursive operation, but we don't implement it recursively /// since that could use up too muck stack. /// /// Instead, the repairs are deferred and placed on a work list in stack form. struct Repair { concat: Opcode, // The argument type after splitting. split_type: Type, // The destination block whose arguments have been split. block: Block, // Number of the original block argument which has been replaced by the low part. num: usize, // Number of the new block argument which represents the high part after the split. hi_num: usize, } /// Generic version of `isplit` and `vsplit` controlled by the `concat` opcode. fn split_any( func: &mut ir::Function, cfg: &ControlFlowGraph, pos: CursorPosition, srcloc: ir::SourceLoc, value: Value, concat: Opcode, ) -> (Value, Value) { let mut repairs = Vec::new(); let pos = &mut FuncCursor::new(func).at_position(pos).with_srcloc(srcloc); let result = split_value(pos, value, concat, &mut repairs); perform_repairs(pos, cfg, repairs); result } pub fn split_block_params(func: &mut ir::Function, cfg: &ControlFlowGraph, block: Block) { let pos = &mut FuncCursor::new(func).at_top(block); let block_params = pos.func.dfg.block_params(block); // Add further splittable types here. fn type_requires_splitting(ty: Type) -> bool { ty == ir::types::I128 } // A shortcut. If none of the param types require splitting, exit now. This helps because // the loop below necessarily has to copy the block params into a new vector, so it's better to // avoid doing so when possible. if !block_params .iter() .any(|block_param| type_requires_splitting(pos.func.dfg.value_type(*block_param))) { return; } let mut repairs = Vec::new(); for (num, block_param) in block_params.to_vec().into_iter().enumerate() { if !type_requires_splitting(pos.func.dfg.value_type(block_param)) { continue; } split_block_param(pos, block, num, block_param, Opcode::Iconcat, &mut repairs); } perform_repairs(pos, cfg, repairs); } fn perform_repairs(pos: &mut FuncCursor, cfg: &ControlFlowGraph, mut repairs: Vec) { // We have split the value requested, and now we may need to fix some block predecessors. while let Some(repair) = repairs.pop() { for BlockPredecessor { inst, .. } in cfg.pred_iter(repair.block) { let branch_opc = pos.func.dfg[inst].opcode(); debug_assert!( branch_opc.is_branch(), "Predecessor not a branch: {}", pos.func.dfg.display_inst(inst, None) ); let num_fixed_args = branch_opc.constraints().num_fixed_value_arguments(); let mut args = pos.func.dfg[inst] .take_value_list() .expect("Branches must have value lists."); let num_args = args.len(&pos.func.dfg.value_lists); // Get the old value passed to the block argument we're repairing. let old_arg = args .get(num_fixed_args + repair.num, &pos.func.dfg.value_lists) .expect("Too few branch arguments"); // It's possible that the CFG's predecessor list has duplicates. Detect them here. if pos.func.dfg.value_type(old_arg) == repair.split_type { pos.func.dfg[inst].put_value_list(args); continue; } // Split the old argument, possibly causing more repairs to be scheduled. pos.goto_inst(inst); let inst_block = pos.func.layout.inst_block(inst).expect("inst in block"); // Insert split values prior to the terminal branch group. let canonical = pos .func .layout .canonical_branch_inst(&pos.func.dfg, inst_block); if let Some(first_branch) = canonical { pos.goto_inst(first_branch); } let (lo, hi) = split_value(pos, old_arg, repair.concat, &mut repairs); // The `lo` part replaces the original argument. *args .get_mut(num_fixed_args + repair.num, &mut pos.func.dfg.value_lists) .unwrap() = lo; // The `hi` part goes at the end. Since multiple repairs may have been scheduled to the // same block, there could be multiple arguments missing. if num_args > num_fixed_args + repair.hi_num { *args .get_mut( num_fixed_args + repair.hi_num, &mut pos.func.dfg.value_lists, ) .unwrap() = hi; } else { // We need to append one or more arguments. If we're adding more than one argument, // there must be pending repairs on the stack that will fill in the correct values // instead of `hi`. args.extend( iter::repeat(hi).take(1 + num_fixed_args + repair.hi_num - num_args), &mut pos.func.dfg.value_lists, ); } // Put the value list back after manipulating it. pos.func.dfg[inst].put_value_list(args); } } } /// Split a single value using the integer or vector semantics given by the `concat` opcode. /// /// If the value is defined by a `concat` instruction, just reuse the operand values of that /// instruction. /// /// Return the two new values representing the parts of `value`. fn split_value( pos: &mut FuncCursor, value: Value, concat: Opcode, repairs: &mut Vec, ) -> (Value, Value) { let value = pos.func.dfg.resolve_aliases(value); let mut reuse = None; match pos.func.dfg.value_def(value) { ValueDef::Result(inst, num) => { // This is an instruction result. See if the value was created by a `concat` // instruction. if let InstructionData::Binary { opcode, args, .. } = pos.func.dfg[inst] { debug_assert_eq!(num, 0); if opcode == concat { reuse = Some((args[0], args[1])); } } } ValueDef::Param(block, num) => { // This is a block parameter. // We can split the parameter value unless this is the entry block. if pos.func.layout.entry_block() != Some(block) { reuse = Some(split_block_param(pos, block, num, value, concat, repairs)); } } } // Did the code above succeed in finding values we can reuse? if let Some(pair) = reuse { pair } else { // No, we'll just have to insert the requested split instruction at `pos`. Note that `pos` // has not been moved by the block argument code above when `reuse` is `None`. match concat { Opcode::Iconcat => pos.ins().isplit(value), Opcode::Vconcat => pos.ins().vsplit(value), _ => panic!("Unhandled concat opcode: {}", concat), } } } fn split_block_param( pos: &mut FuncCursor, block: Block, param_num: usize, value: Value, concat: Opcode, repairs: &mut Vec, ) -> (Value, Value) { // We are going to replace the parameter at `num` with two new arguments. // Determine the new value types. let ty = pos.func.dfg.value_type(value); let split_type = match concat { Opcode::Iconcat => ty.half_width().expect("Invalid type for isplit"), Opcode::Vconcat => ty.half_vector().expect("Invalid type for vsplit"), _ => panic!("Unhandled concat opcode: {}", concat), }; // Since the `repairs` stack potentially contains other parameter numbers for // `block`, avoid shifting and renumbering block parameters. It could invalidate other // `repairs` entries. // // Replace the original `value` with the low part, and append the high part at the // end of the argument list. let lo = pos.func.dfg.replace_block_param(value, split_type); let hi_num = pos.func.dfg.num_block_params(block); let hi = pos.func.dfg.append_block_param(block, split_type); // Now the original value is dangling. Insert a concatenation instruction that can // compute it from the two new parameters. This also serves as a record of what we // did so a future call to this function doesn't have to redo the work. // // Note that it is safe to move `pos` here since `reuse` was set above, so we don't // need to insert a split instruction before returning. pos.goto_first_inst(block); pos.ins() .with_result(value) .Binary(concat, split_type, lo, hi); // Finally, splitting the block parameter is not enough. We also have to repair all // of the predecessor instructions that branch here. add_repair(concat, split_type, block, param_num, hi_num, repairs); (lo, hi) } // Add a repair entry to the work list. fn add_repair( concat: Opcode, split_type: Type, block: Block, num: usize, hi_num: usize, repairs: &mut Vec, ) { repairs.push(Repair { concat, split_type, block, num, hi_num, }); } /// Strip concat-split chains. Return a simpler way of computing the same value. /// /// Given this input: /// /// ```clif /// v10 = iconcat v1, v2 /// v11, v12 = isplit v10 /// ``` /// /// This function resolves `v11` to `v1` and `v12` to `v2`. fn resolve_splits(dfg: &ir::DataFlowGraph, value: Value) -> Value { let value = dfg.resolve_aliases(value); // Deconstruct a split instruction. let split_res; let concat_opc; let split_arg; if let ValueDef::Result(inst, num) = dfg.value_def(value) { split_res = num; concat_opc = match dfg[inst].opcode() { Opcode::Isplit => Opcode::Iconcat, Opcode::Vsplit => Opcode::Vconcat, _ => return value, }; split_arg = dfg.inst_args(inst)[0]; } else { return value; } // See if split_arg is defined by a concatenation instruction. if let ValueDef::Result(inst, _) = dfg.value_def(split_arg) { if dfg[inst].opcode() == concat_opc { return dfg.inst_args(inst)[split_res]; } } value } /// Simplify the arguments to a branch *after* the instructions leading up to the branch have been /// legalized. /// /// The branch argument repairs performed by `split_any()` above may be performed on branches that /// have not yet been legalized. The repaired arguments can be defined by actual split /// instructions in that case. /// /// After legalizing the instructions computing the value that was split, it is likely that we can /// avoid depending on the split instruction. Its input probably comes from a concatenation. pub fn simplify_branch_arguments(dfg: &mut ir::DataFlowGraph, branch: Inst) { let mut new_args = SmallVec::<[Value; 32]>::new(); for &arg in dfg.inst_args(branch) { let new_arg = resolve_splits(dfg, arg); new_args.push(new_arg); } dfg.inst_args_mut(branch).copy_from_slice(&new_args); }