diff options
Diffstat (limited to 'third_party/rust/cranelift-codegen-meta/src/shared')
8 files changed, 6803 insertions, 0 deletions
diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/entities.rs b/third_party/rust/cranelift-codegen-meta/src/shared/entities.rs new file mode 100644 index 0000000000..c3f2bc0387 --- /dev/null +++ b/third_party/rust/cranelift-codegen-meta/src/shared/entities.rs @@ -0,0 +1,73 @@ +use crate::cdsl::operands::{OperandKind, OperandKindFields}; + +/// Small helper to initialize an OperandBuilder with the right kind, for a given name and doc. +fn new(format_field_name: &'static str, rust_type: &'static str, doc: &'static str) -> OperandKind { + OperandKind::new(format_field_name, rust_type, OperandKindFields::EntityRef).with_doc(doc) +} + +pub(crate) struct EntityRefs { + /// A reference to a basic block in the same function. + /// This is primarliy used in control flow instructions. + pub(crate) block: OperandKind, + + /// A reference to a stack slot declared in the function preamble. + pub(crate) stack_slot: OperandKind, + + /// A reference to a global value. + pub(crate) global_value: OperandKind, + + /// A reference to a function signature declared in the function preamble. + /// This is used to provide the call signature in a call_indirect instruction. + pub(crate) sig_ref: OperandKind, + + /// A reference to an external function declared in the function preamble. + /// This is used to provide the callee and signature in a call instruction. + pub(crate) func_ref: OperandKind, + + /// A reference to a jump table declared in the function preamble. + pub(crate) jump_table: OperandKind, + + /// A reference to a heap declared in the function preamble. + pub(crate) heap: OperandKind, + + /// A reference to a table declared in the function preamble. + pub(crate) table: OperandKind, + + /// A variable-sized list of value operands. Use for Block and function call arguments. + pub(crate) varargs: OperandKind, +} + +impl EntityRefs { + pub fn new() -> Self { + Self { + block: new( + "destination", + "ir::Block", + "a basic block in the same function.", + ), + stack_slot: new("stack_slot", "ir::StackSlot", "A stack slot"), + + global_value: new("global_value", "ir::GlobalValue", "A global value."), + + sig_ref: new("sig_ref", "ir::SigRef", "A function signature."), + + func_ref: new("func_ref", "ir::FuncRef", "An external function."), + + jump_table: new("table", "ir::JumpTable", "A jump table."), + + heap: new("heap", "ir::Heap", "A heap."), + + table: new("table", "ir::Table", "A table."), + + varargs: OperandKind::new("", "&[Value]", OperandKindFields::VariableArgs).with_doc( + r#" + A variable size list of `value` operands. + + Use this to represent arguments passed to a function call, arguments + passed to a basic block, or a variable number of results + returned from an instruction. + "#, + ), + } + } +} diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/formats.rs b/third_party/rust/cranelift-codegen-meta/src/shared/formats.rs new file mode 100644 index 0000000000..3d081951a5 --- /dev/null +++ b/third_party/rust/cranelift-codegen-meta/src/shared/formats.rs @@ -0,0 +1,330 @@ +use crate::cdsl::formats::{InstructionFormat, InstructionFormatBuilder as Builder}; +use crate::shared::{entities::EntityRefs, immediates::Immediates}; +use std::rc::Rc; + +pub(crate) struct Formats { + pub(crate) atomic_cas: Rc<InstructionFormat>, + pub(crate) atomic_rmw: Rc<InstructionFormat>, + pub(crate) binary: Rc<InstructionFormat>, + pub(crate) binary_imm8: Rc<InstructionFormat>, + pub(crate) binary_imm64: Rc<InstructionFormat>, + pub(crate) branch: Rc<InstructionFormat>, + pub(crate) branch_float: Rc<InstructionFormat>, + pub(crate) branch_icmp: Rc<InstructionFormat>, + pub(crate) branch_int: Rc<InstructionFormat>, + pub(crate) branch_table: Rc<InstructionFormat>, + pub(crate) branch_table_base: Rc<InstructionFormat>, + pub(crate) branch_table_entry: Rc<InstructionFormat>, + pub(crate) call: Rc<InstructionFormat>, + pub(crate) call_indirect: Rc<InstructionFormat>, + pub(crate) cond_trap: Rc<InstructionFormat>, + pub(crate) copy_special: Rc<InstructionFormat>, + pub(crate) copy_to_ssa: Rc<InstructionFormat>, + pub(crate) float_compare: Rc<InstructionFormat>, + pub(crate) float_cond: Rc<InstructionFormat>, + pub(crate) float_cond_trap: Rc<InstructionFormat>, + pub(crate) func_addr: Rc<InstructionFormat>, + pub(crate) heap_addr: Rc<InstructionFormat>, + pub(crate) indirect_jump: Rc<InstructionFormat>, + pub(crate) int_compare: Rc<InstructionFormat>, + pub(crate) int_compare_imm: Rc<InstructionFormat>, + pub(crate) int_cond: Rc<InstructionFormat>, + pub(crate) int_cond_trap: Rc<InstructionFormat>, + pub(crate) int_select: Rc<InstructionFormat>, + pub(crate) jump: Rc<InstructionFormat>, + pub(crate) load: Rc<InstructionFormat>, + pub(crate) load_complex: Rc<InstructionFormat>, + pub(crate) load_no_offset: Rc<InstructionFormat>, + pub(crate) multiary: Rc<InstructionFormat>, + pub(crate) nullary: Rc<InstructionFormat>, + pub(crate) reg_fill: Rc<InstructionFormat>, + pub(crate) reg_move: Rc<InstructionFormat>, + pub(crate) reg_spill: Rc<InstructionFormat>, + pub(crate) shuffle: Rc<InstructionFormat>, + pub(crate) stack_load: Rc<InstructionFormat>, + pub(crate) stack_store: Rc<InstructionFormat>, + pub(crate) store: Rc<InstructionFormat>, + pub(crate) store_complex: Rc<InstructionFormat>, + pub(crate) store_no_offset: Rc<InstructionFormat>, + pub(crate) table_addr: Rc<InstructionFormat>, + pub(crate) ternary: Rc<InstructionFormat>, + pub(crate) ternary_imm8: Rc<InstructionFormat>, + pub(crate) trap: Rc<InstructionFormat>, + pub(crate) unary: Rc<InstructionFormat>, + pub(crate) unary_bool: Rc<InstructionFormat>, + pub(crate) unary_const: Rc<InstructionFormat>, + pub(crate) unary_global_value: Rc<InstructionFormat>, + pub(crate) unary_ieee32: Rc<InstructionFormat>, + pub(crate) unary_ieee64: Rc<InstructionFormat>, + pub(crate) unary_imm: Rc<InstructionFormat>, +} + +impl Formats { + pub fn new(imm: &Immediates, entities: &EntityRefs) -> Self { + Self { + unary: Builder::new("Unary").value().build(), + + unary_imm: Builder::new("UnaryImm").imm(&imm.imm64).build(), + + unary_ieee32: Builder::new("UnaryIeee32").imm(&imm.ieee32).build(), + + unary_ieee64: Builder::new("UnaryIeee64").imm(&imm.ieee64).build(), + + unary_bool: Builder::new("UnaryBool").imm(&imm.boolean).build(), + + unary_const: Builder::new("UnaryConst").imm(&imm.pool_constant).build(), + + unary_global_value: Builder::new("UnaryGlobalValue") + .imm(&entities.global_value) + .build(), + + binary: Builder::new("Binary").value().value().build(), + + binary_imm8: Builder::new("BinaryImm8").value().imm(&imm.uimm8).build(), + + binary_imm64: Builder::new("BinaryImm64").value().imm(&imm.imm64).build(), + + // The select instructions are controlled by the second VALUE operand. + // The first VALUE operand is the controlling flag which has a derived type. + // The fma instruction has the same constraint on all inputs. + ternary: Builder::new("Ternary") + .value() + .value() + .value() + .typevar_operand(1) + .build(), + + ternary_imm8: Builder::new("TernaryImm8") + .value() + .imm(&imm.uimm8) + .value() + .build(), + + // Catch-all for instructions with many outputs and inputs and no immediate + // operands. + multiary: Builder::new("MultiAry").varargs().build(), + + nullary: Builder::new("NullAry").build(), + + shuffle: Builder::new("Shuffle") + .value() + .value() + .imm_with_name("mask", &imm.uimm128) + .build(), + + int_compare: Builder::new("IntCompare") + .imm(&imm.intcc) + .value() + .value() + .build(), + + int_compare_imm: Builder::new("IntCompareImm") + .imm(&imm.intcc) + .value() + .imm(&imm.imm64) + .build(), + + int_cond: Builder::new("IntCond").imm(&imm.intcc).value().build(), + + float_compare: Builder::new("FloatCompare") + .imm(&imm.floatcc) + .value() + .value() + .build(), + + float_cond: Builder::new("FloatCond").imm(&imm.floatcc).value().build(), + + int_select: Builder::new("IntSelect") + .imm(&imm.intcc) + .value() + .value() + .value() + .build(), + + jump: Builder::new("Jump").imm(&entities.block).varargs().build(), + + branch: Builder::new("Branch") + .value() + .imm(&entities.block) + .varargs() + .build(), + + branch_int: Builder::new("BranchInt") + .imm(&imm.intcc) + .value() + .imm(&entities.block) + .varargs() + .build(), + + branch_float: Builder::new("BranchFloat") + .imm(&imm.floatcc) + .value() + .imm(&entities.block) + .varargs() + .build(), + + branch_icmp: Builder::new("BranchIcmp") + .imm(&imm.intcc) + .value() + .value() + .imm(&entities.block) + .varargs() + .build(), + + branch_table: Builder::new("BranchTable") + .value() + .imm(&entities.block) + .imm(&entities.jump_table) + .build(), + + branch_table_entry: Builder::new("BranchTableEntry") + .value() + .value() + .imm(&imm.uimm8) + .imm(&entities.jump_table) + .build(), + + branch_table_base: Builder::new("BranchTableBase") + .imm(&entities.jump_table) + .build(), + + indirect_jump: Builder::new("IndirectJump") + .value() + .imm(&entities.jump_table) + .build(), + + call: Builder::new("Call") + .imm(&entities.func_ref) + .varargs() + .build(), + + call_indirect: Builder::new("CallIndirect") + .imm(&entities.sig_ref) + .value() + .varargs() + .build(), + + func_addr: Builder::new("FuncAddr").imm(&entities.func_ref).build(), + + atomic_rmw: Builder::new("AtomicRmw") + .imm(&imm.memflags) + .imm(&imm.atomic_rmw_op) + .value() + .value() + .build(), + + atomic_cas: Builder::new("AtomicCas") + .imm(&imm.memflags) + .value() + .value() + .value() + .typevar_operand(2) + .build(), + + load: Builder::new("Load") + .imm(&imm.memflags) + .value() + .imm(&imm.offset32) + .build(), + + load_complex: Builder::new("LoadComplex") + .imm(&imm.memflags) + .varargs() + .imm(&imm.offset32) + .build(), + + load_no_offset: Builder::new("LoadNoOffset") + .imm(&imm.memflags) + .value() + .build(), + + store: Builder::new("Store") + .imm(&imm.memflags) + .value() + .value() + .imm(&imm.offset32) + .build(), + + store_complex: Builder::new("StoreComplex") + .imm(&imm.memflags) + .value() + .varargs() + .imm(&imm.offset32) + .build(), + + store_no_offset: Builder::new("StoreNoOffset") + .imm(&imm.memflags) + .value() + .value() + .build(), + + stack_load: Builder::new("StackLoad") + .imm(&entities.stack_slot) + .imm(&imm.offset32) + .build(), + + stack_store: Builder::new("StackStore") + .value() + .imm(&entities.stack_slot) + .imm(&imm.offset32) + .build(), + + // Accessing a WebAssembly heap. + heap_addr: Builder::new("HeapAddr") + .imm(&entities.heap) + .value() + .imm(&imm.uimm32) + .build(), + + // Accessing a WebAssembly table. + table_addr: Builder::new("TableAddr") + .imm(&entities.table) + .value() + .imm(&imm.offset32) + .build(), + + reg_move: Builder::new("RegMove") + .value() + .imm_with_name("src", &imm.regunit) + .imm_with_name("dst", &imm.regunit) + .build(), + + copy_special: Builder::new("CopySpecial") + .imm_with_name("src", &imm.regunit) + .imm_with_name("dst", &imm.regunit) + .build(), + + copy_to_ssa: Builder::new("CopyToSsa") + .imm_with_name("src", &imm.regunit) + .build(), + + reg_spill: Builder::new("RegSpill") + .value() + .imm_with_name("src", &imm.regunit) + .imm_with_name("dst", &entities.stack_slot) + .build(), + + reg_fill: Builder::new("RegFill") + .value() + .imm_with_name("src", &entities.stack_slot) + .imm_with_name("dst", &imm.regunit) + .build(), + + trap: Builder::new("Trap").imm(&imm.trapcode).build(), + + cond_trap: Builder::new("CondTrap").value().imm(&imm.trapcode).build(), + + int_cond_trap: Builder::new("IntCondTrap") + .imm(&imm.intcc) + .value() + .imm(&imm.trapcode) + .build(), + + float_cond_trap: Builder::new("FloatCondTrap") + .imm(&imm.floatcc) + .value() + .imm(&imm.trapcode) + .build(), + } + } +} diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/immediates.rs b/third_party/rust/cranelift-codegen-meta/src/shared/immediates.rs new file mode 100644 index 0000000000..0aa4129daf --- /dev/null +++ b/third_party/rust/cranelift-codegen-meta/src/shared/immediates.rs @@ -0,0 +1,175 @@ +use crate::cdsl::operands::{EnumValues, OperandKind, OperandKindFields}; + +use std::collections::HashMap; + +pub(crate) struct Immediates { + /// A 64-bit immediate integer operand. + /// + /// This type of immediate integer can interact with SSA values with any IntType type. + pub imm64: OperandKind, + + /// An unsigned 8-bit immediate integer operand. + /// + /// This small operand is used to indicate lane indexes in SIMD vectors and immediate bit + /// counts on shift instructions. + pub uimm8: OperandKind, + + /// An unsigned 32-bit immediate integer operand. + pub uimm32: OperandKind, + + /// An unsigned 128-bit immediate integer operand. + /// + /// This operand is used to pass entire 128-bit vectors as immediates to instructions like + /// const. + pub uimm128: OperandKind, + + /// A constant stored in the constant pool. + /// + /// This operand is used to pass constants to instructions like vconst while storing the + /// actual bytes in the constant pool. + pub pool_constant: OperandKind, + + /// A 32-bit immediate signed offset. + /// + /// This is used to represent an immediate address offset in load/store instructions. + pub offset32: OperandKind, + + /// A 32-bit immediate floating point operand. + /// + /// IEEE 754-2008 binary32 interchange format. + pub ieee32: OperandKind, + + /// A 64-bit immediate floating point operand. + /// + /// IEEE 754-2008 binary64 interchange format. + pub ieee64: OperandKind, + + /// An immediate boolean operand. + /// + /// This type of immediate boolean can interact with SSA values with any BoolType type. + pub boolean: OperandKind, + + /// A condition code for comparing integer values. + /// + /// This enumerated operand kind is used for the `icmp` instruction and corresponds to the + /// condcodes::IntCC` Rust type. + pub intcc: OperandKind, + + /// A condition code for comparing floating point values. + /// + /// This enumerated operand kind is used for the `fcmp` instruction and corresponds to the + /// `condcodes::FloatCC` Rust type. + pub floatcc: OperandKind, + + /// Flags for memory operations like `load` and `store`. + pub memflags: OperandKind, + + /// A register unit in the current target ISA. + pub regunit: OperandKind, + + /// A trap code indicating the reason for trapping. + /// + /// The Rust enum type also has a `User(u16)` variant for user-provided trap codes. + pub trapcode: OperandKind, + + /// A code indicating the arithmetic operation to perform in an atomic_rmw memory access. + pub atomic_rmw_op: OperandKind, +} + +fn new_imm(format_field_name: &'static str, rust_type: &'static str) -> OperandKind { + OperandKind::new(format_field_name, rust_type, OperandKindFields::ImmValue) +} +fn new_enum( + format_field_name: &'static str, + rust_type: &'static str, + values: EnumValues, +) -> OperandKind { + OperandKind::new( + format_field_name, + rust_type, + OperandKindFields::ImmEnum(values), + ) +} + +impl Immediates { + pub fn new() -> Self { + Self { + imm64: new_imm("imm", "ir::immediates::Imm64").with_doc("A 64-bit immediate integer."), + uimm8: new_imm("imm", "ir::immediates::Uimm8") + .with_doc("An 8-bit immediate unsigned integer."), + uimm32: new_imm("imm", "ir::immediates::Uimm32") + .with_doc("A 32-bit immediate unsigned integer."), + uimm128: new_imm("imm", "ir::Immediate") + .with_doc("A 128-bit immediate unsigned integer."), + pool_constant: new_imm("constant_handle", "ir::Constant") + .with_doc("A constant stored in the constant pool."), + offset32: new_imm("offset", "ir::immediates::Offset32") + .with_doc("A 32-bit immediate signed offset."), + ieee32: new_imm("imm", "ir::immediates::Ieee32") + .with_doc("A 32-bit immediate floating point number."), + ieee64: new_imm("imm", "ir::immediates::Ieee64") + .with_doc("A 64-bit immediate floating point number."), + boolean: new_imm("imm", "bool").with_doc("An immediate boolean."), + intcc: { + let mut intcc_values = HashMap::new(); + intcc_values.insert("eq", "Equal"); + intcc_values.insert("ne", "NotEqual"); + intcc_values.insert("sge", "SignedGreaterThanOrEqual"); + intcc_values.insert("sgt", "SignedGreaterThan"); + intcc_values.insert("sle", "SignedLessThanOrEqual"); + intcc_values.insert("slt", "SignedLessThan"); + intcc_values.insert("uge", "UnsignedGreaterThanOrEqual"); + intcc_values.insert("ugt", "UnsignedGreaterThan"); + intcc_values.insert("ule", "UnsignedLessThanOrEqual"); + intcc_values.insert("ult", "UnsignedLessThan"); + intcc_values.insert("of", "Overflow"); + intcc_values.insert("nof", "NotOverflow"); + new_enum("cond", "ir::condcodes::IntCC", intcc_values) + .with_doc("An integer comparison condition code.") + }, + + floatcc: { + let mut floatcc_values = HashMap::new(); + floatcc_values.insert("ord", "Ordered"); + floatcc_values.insert("uno", "Unordered"); + floatcc_values.insert("eq", "Equal"); + floatcc_values.insert("ne", "NotEqual"); + floatcc_values.insert("one", "OrderedNotEqual"); + floatcc_values.insert("ueq", "UnorderedOrEqual"); + floatcc_values.insert("lt", "LessThan"); + floatcc_values.insert("le", "LessThanOrEqual"); + floatcc_values.insert("gt", "GreaterThan"); + floatcc_values.insert("ge", "GreaterThanOrEqual"); + floatcc_values.insert("ult", "UnorderedOrLessThan"); + floatcc_values.insert("ule", "UnorderedOrLessThanOrEqual"); + floatcc_values.insert("ugt", "UnorderedOrGreaterThan"); + floatcc_values.insert("uge", "UnorderedOrGreaterThanOrEqual"); + new_enum("cond", "ir::condcodes::FloatCC", floatcc_values) + .with_doc("A floating point comparison condition code") + }, + + memflags: new_imm("flags", "ir::MemFlags").with_doc("Memory operation flags"), + regunit: new_imm("regunit", "isa::RegUnit") + .with_doc("A register unit in the target ISA"), + trapcode: { + let mut trapcode_values = HashMap::new(); + trapcode_values.insert("stk_ovf", "StackOverflow"); + trapcode_values.insert("heap_oob", "HeapOutOfBounds"); + trapcode_values.insert("int_ovf", "IntegerOverflow"); + trapcode_values.insert("int_divz", "IntegerDivisionByZero"); + new_enum("code", "ir::TrapCode", trapcode_values).with_doc("A trap reason code.") + }, + atomic_rmw_op: { + let mut atomic_rmw_op_values = HashMap::new(); + atomic_rmw_op_values.insert("add", "Add"); + atomic_rmw_op_values.insert("sub", "Sub"); + atomic_rmw_op_values.insert("and", "And"); + atomic_rmw_op_values.insert("or", "Or"); + atomic_rmw_op_values.insert("xor", "Xor"); + atomic_rmw_op_values.insert("xchg", "Xchg"); + new_enum("op", "ir::AtomicRmwOp", atomic_rmw_op_values) + .with_doc("Atomic Read-Modify-Write Ops") + }, + } + } +} diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs b/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs new file mode 100644 index 0000000000..bd1444d79c --- /dev/null +++ b/third_party/rust/cranelift-codegen-meta/src/shared/instructions.rs @@ -0,0 +1,4514 @@ +#![allow(non_snake_case)] + +use crate::cdsl::instructions::{ + AllInstructions, InstructionBuilder as Inst, InstructionGroup, InstructionGroupBuilder, +}; +use crate::cdsl::operands::Operand; +use crate::cdsl::type_inference::Constraint::WiderOrEq; +use crate::cdsl::types::{LaneType, ValueType}; +use crate::cdsl::typevar::{Interval, TypeSetBuilder, TypeVar}; +use crate::shared::formats::Formats; +use crate::shared::types; +use crate::shared::{entities::EntityRefs, immediates::Immediates}; + +#[inline(never)] +fn define_control_flow( + ig: &mut InstructionGroupBuilder, + formats: &Formats, + imm: &Immediates, + entities: &EntityRefs, +) { + let block = &Operand::new("block", &entities.block).with_doc("Destination basic block"); + let args = &Operand::new("args", &entities.varargs).with_doc("block arguments"); + + ig.push( + Inst::new( + "jump", + r#" + Jump. + + Unconditionally jump to a basic block, passing the specified + block arguments. The number and types of arguments must match the + destination block. + "#, + &formats.jump, + ) + .operands_in(vec![block, args]) + .is_terminator(true) + .is_branch(true), + ); + + ig.push( + Inst::new( + "fallthrough", + r#" + Fall through to the next block. + + This is the same as `jump`, except the destination block must be + the next one in the layout. + + Jumps are turned into fall-through instructions by the branch + relaxation pass. There is no reason to use this instruction outside + that pass. + "#, + &formats.jump, + ) + .operands_in(vec![block, args]) + .is_terminator(true) + .is_branch(true), + ); + + let Testable = &TypeVar::new( + "Testable", + "A scalar boolean or integer type", + TypeSetBuilder::new() + .ints(Interval::All) + .bools(Interval::All) + .build(), + ); + + { + let c = &Operand::new("c", Testable).with_doc("Controlling value to test"); + + ig.push( + Inst::new( + "brz", + r#" + Branch when zero. + + If ``c`` is a `b1` value, take the branch when ``c`` is false. If + ``c`` is an integer value, take the branch when ``c = 0``. + "#, + &formats.branch, + ) + .operands_in(vec![c, block, args]) + .is_branch(true), + ); + + ig.push( + Inst::new( + "brnz", + r#" + Branch when non-zero. + + If ``c`` is a `b1` value, take the branch when ``c`` is true. If + ``c`` is an integer value, take the branch when ``c != 0``. + "#, + &formats.branch, + ) + .operands_in(vec![c, block, args]) + .is_branch(true), + ); + } + + let iB = &TypeVar::new( + "iB", + "A scalar integer type", + TypeSetBuilder::new().ints(Interval::All).build(), + ); + let iflags: &TypeVar = &ValueType::Special(types::Flag::IFlags.into()).into(); + let fflags: &TypeVar = &ValueType::Special(types::Flag::FFlags.into()).into(); + + { + let Cond = &Operand::new("Cond", &imm.intcc); + let x = &Operand::new("x", iB); + let y = &Operand::new("y", iB); + + ig.push( + Inst::new( + "br_icmp", + r#" + Compare scalar integers and branch. + + Compare ``x`` and ``y`` in the same way as the `icmp` instruction + and take the branch if the condition is true: + + ```text + br_icmp ugt v1, v2, block4(v5, v6) + ``` + + is semantically equivalent to: + + ```text + v10 = icmp ugt, v1, v2 + brnz v10, block4(v5, v6) + ``` + + Some RISC architectures like MIPS and RISC-V provide instructions that + implement all or some of the condition codes. The instruction can also + be used to represent *macro-op fusion* on architectures like Intel's. + "#, + &formats.branch_icmp, + ) + .operands_in(vec![Cond, x, y, block, args]) + .is_branch(true), + ); + + let f = &Operand::new("f", iflags); + + ig.push( + Inst::new( + "brif", + r#" + Branch when condition is true in integer CPU flags. + "#, + &formats.branch_int, + ) + .operands_in(vec![Cond, f, block, args]) + .is_branch(true), + ); + } + + { + let Cond = &Operand::new("Cond", &imm.floatcc); + + let f = &Operand::new("f", fflags); + + ig.push( + Inst::new( + "brff", + r#" + Branch when condition is true in floating point CPU flags. + "#, + &formats.branch_float, + ) + .operands_in(vec![Cond, f, block, args]) + .is_branch(true), + ); + } + + { + let x = &Operand::new("x", iB).with_doc("index into jump table"); + let JT = &Operand::new("JT", &entities.jump_table); + + ig.push( + Inst::new( + "br_table", + r#" + Indirect branch via jump table. + + Use ``x`` as an unsigned index into the jump table ``JT``. If a jump + table entry is found, branch to the corresponding block. If no entry was + found or the index is out-of-bounds, branch to the given default block. + + Note that this branch instruction can't pass arguments to the targeted + blocks. Split critical edges as needed to work around this. + + Do not confuse this with "tables" in WebAssembly. ``br_table`` is for + jump tables with destinations within the current function only -- think + of a ``match`` in Rust or a ``switch`` in C. If you want to call a + function in a dynamic library, that will typically use + ``call_indirect``. + "#, + &formats.branch_table, + ) + .operands_in(vec![x, block, JT]) + .is_terminator(true) + .is_branch(true), + ); + } + + let iAddr = &TypeVar::new( + "iAddr", + "An integer address type", + TypeSetBuilder::new().ints(32..64).refs(32..64).build(), + ); + + { + let x = &Operand::new("x", iAddr).with_doc("index into jump table"); + let addr = &Operand::new("addr", iAddr); + let Size = &Operand::new("Size", &imm.uimm8).with_doc("Size in bytes"); + let JT = &Operand::new("JT", &entities.jump_table); + let entry = &Operand::new("entry", iAddr).with_doc("entry of jump table"); + + ig.push( + Inst::new( + "jump_table_entry", + r#" + Get an entry from a jump table. + + Load a serialized ``entry`` from a jump table ``JT`` at a given index + ``addr`` with a specific ``Size``. The retrieved entry may need to be + decoded after loading, depending upon the jump table type used. + + Currently, the only type supported is entries which are relative to the + base of the jump table. + "#, + &formats.branch_table_entry, + ) + .operands_in(vec![x, addr, Size, JT]) + .operands_out(vec![entry]) + .can_load(true), + ); + + ig.push( + Inst::new( + "jump_table_base", + r#" + Get the absolute base address of a jump table. + + This is used for jump tables wherein the entries are stored relative to + the base of jump table. In order to use these, generated code should first + load an entry using ``jump_table_entry``, then use this instruction to add + the relative base back to it. + "#, + &formats.branch_table_base, + ) + .operands_in(vec![JT]) + .operands_out(vec![addr]), + ); + + ig.push( + Inst::new( + "indirect_jump_table_br", + r#" + Branch indirectly via a jump table entry. + + Unconditionally jump via a jump table entry that was previously loaded + with the ``jump_table_entry`` instruction. + "#, + &formats.indirect_jump, + ) + .operands_in(vec![addr, JT]) + .is_indirect_branch(true) + .is_terminator(true) + .is_branch(true), + ); + } + + ig.push( + Inst::new( + "debugtrap", + r#" + Encodes an assembly debug trap. + "#, + &formats.nullary, + ) + .other_side_effects(true) + .can_load(true) + .can_store(true), + ); + + { + let code = &Operand::new("code", &imm.trapcode); + ig.push( + Inst::new( + "trap", + r#" + Terminate execution unconditionally. + "#, + &formats.trap, + ) + .operands_in(vec![code]) + .can_trap(true) + .is_terminator(true), + ); + + let c = &Operand::new("c", Testable).with_doc("Controlling value to test"); + ig.push( + Inst::new( + "trapz", + r#" + Trap when zero. + + if ``c`` is non-zero, execution continues at the following instruction. + "#, + &formats.cond_trap, + ) + .operands_in(vec![c, code]) + .can_trap(true), + ); + + ig.push( + Inst::new( + "resumable_trap", + r#" + A resumable trap. + + This instruction allows non-conditional traps to be used as non-terminal instructions. + "#, + &formats.trap, + ) + .operands_in(vec![code]) + .can_trap(true), + ); + + let c = &Operand::new("c", Testable).with_doc("Controlling value to test"); + ig.push( + Inst::new( + "trapnz", + r#" + Trap when non-zero. + + If ``c`` is zero, execution continues at the following instruction. + "#, + &formats.cond_trap, + ) + .operands_in(vec![c, code]) + .can_trap(true), + ); + + ig.push( + Inst::new( + "resumable_trapnz", + r#" + A resumable trap to be called when the passed condition is non-zero. + + If ``c`` is zero, execution continues at the following instruction. + "#, + &formats.cond_trap, + ) + .operands_in(vec![c, code]) + .can_trap(true), + ); + + let Cond = &Operand::new("Cond", &imm.intcc); + let f = &Operand::new("f", iflags); + ig.push( + Inst::new( + "trapif", + r#" + Trap when condition is true in integer CPU flags. + "#, + &formats.int_cond_trap, + ) + .operands_in(vec![Cond, f, code]) + .can_trap(true), + ); + + let Cond = &Operand::new("Cond", &imm.floatcc); + let f = &Operand::new("f", fflags); + let code = &Operand::new("code", &imm.trapcode); + ig.push( + Inst::new( + "trapff", + r#" + Trap when condition is true in floating point CPU flags. + "#, + &formats.float_cond_trap, + ) + .operands_in(vec![Cond, f, code]) + .can_trap(true), + ); + } + + let rvals = &Operand::new("rvals", &entities.varargs).with_doc("return values"); + ig.push( + Inst::new( + "return", + r#" + Return from the function. + + Unconditionally transfer control to the calling function, passing the + provided return values. The list of return values must match the + function signature's return types. + "#, + &formats.multiary, + ) + .operands_in(vec![rvals]) + .is_return(true) + .is_terminator(true), + ); + + let rvals = &Operand::new("rvals", &entities.varargs).with_doc("return values"); + ig.push( + Inst::new( + "fallthrough_return", + r#" + Return from the function by fallthrough. + + This is a specialized instruction for use where one wants to append + a custom epilogue, which will then perform the real return. This + instruction has no encoding. + "#, + &formats.multiary, + ) + .operands_in(vec![rvals]) + .is_return(true) + .is_terminator(true), + ); + + let FN = &Operand::new("FN", &entities.func_ref) + .with_doc("function to call, declared by `function`"); + let args = &Operand::new("args", &entities.varargs).with_doc("call arguments"); + let rvals = &Operand::new("rvals", &entities.varargs).with_doc("return values"); + ig.push( + Inst::new( + "call", + r#" + Direct function call. + + Call a function which has been declared in the preamble. The argument + types must match the function's signature. + "#, + &formats.call, + ) + .operands_in(vec![FN, args]) + .operands_out(vec![rvals]) + .is_call(true), + ); + + let SIG = &Operand::new("SIG", &entities.sig_ref).with_doc("function signature"); + let callee = &Operand::new("callee", iAddr).with_doc("address of function to call"); + let args = &Operand::new("args", &entities.varargs).with_doc("call arguments"); + let rvals = &Operand::new("rvals", &entities.varargs).with_doc("return values"); + ig.push( + Inst::new( + "call_indirect", + r#" + Indirect function call. + + Call the function pointed to by `callee` with the given arguments. The + called function must match the specified signature. + + Note that this is different from WebAssembly's ``call_indirect``; the + callee is a native address, rather than a table index. For WebAssembly, + `table_addr` and `load` are used to obtain a native address + from a table. + "#, + &formats.call_indirect, + ) + .operands_in(vec![SIG, callee, args]) + .operands_out(vec![rvals]) + .is_call(true), + ); + + let FN = &Operand::new("FN", &entities.func_ref) + .with_doc("function to call, declared by `function`"); + let addr = &Operand::new("addr", iAddr); + ig.push( + Inst::new( + "func_addr", + r#" + Get the address of a function. + + Compute the absolute address of a function declared in the preamble. + The returned address can be used as a ``callee`` argument to + `call_indirect`. This is also a method for calling functions that + are too far away to be addressable by a direct `call` + instruction. + "#, + &formats.func_addr, + ) + .operands_in(vec![FN]) + .operands_out(vec![addr]), + ); +} + +#[inline(never)] +fn define_simd_lane_access( + ig: &mut InstructionGroupBuilder, + formats: &Formats, + imm: &Immediates, + _: &EntityRefs, +) { + let TxN = &TypeVar::new( + "TxN", + "A SIMD vector type", + TypeSetBuilder::new() + .ints(Interval::All) + .floats(Interval::All) + .bools(Interval::All) + .simd_lanes(Interval::All) + .includes_scalars(false) + .build(), + ); + + let x = &Operand::new("x", &TxN.lane_of()).with_doc("Value to splat to all lanes"); + let a = &Operand::new("a", TxN); + + ig.push( + Inst::new( + "splat", + r#" + Vector splat. + + Return a vector whose lanes are all ``x``. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let I8x16 = &TypeVar::new( + "I8x16", + "A SIMD vector type consisting of 16 lanes of 8-bit integers", + TypeSetBuilder::new() + .ints(8..8) + .simd_lanes(16..16) + .includes_scalars(false) + .build(), + ); + let x = &Operand::new("x", I8x16).with_doc("Vector to modify by re-arranging lanes"); + let y = &Operand::new("y", I8x16).with_doc("Mask for re-arranging lanes"); + + ig.push( + Inst::new( + "swizzle", + r#" + Vector swizzle. + + Returns a new vector with byte-width lanes selected from the lanes of the first input + vector ``x`` specified in the second input vector ``s``. The indices ``i`` in range + ``[0, 15]`` select the ``i``-th element of ``x``. For indices outside of the range the + resulting lane is 0. Note that this operates on byte-width lanes. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + let x = &Operand::new("x", TxN).with_doc("The vector to modify"); + let y = &Operand::new("y", &TxN.lane_of()).with_doc("New lane value"); + let Idx = &Operand::new("Idx", &imm.uimm8).with_doc("Lane index"); + + ig.push( + Inst::new( + "insertlane", + r#" + Insert ``y`` as lane ``Idx`` in x. + + The lane index, ``Idx``, is an immediate value, not an SSA value. It + must indicate a valid lane index for the type of ``x``. + "#, + &formats.ternary_imm8, + ) + .operands_in(vec![x, y, Idx]) + .operands_out(vec![a]), + ); + + let x = &Operand::new("x", TxN); + let a = &Operand::new("a", &TxN.lane_of()); + + ig.push( + Inst::new( + "extractlane", + r#" + Extract lane ``Idx`` from ``x``. + + The lane index, ``Idx``, is an immediate value, not an SSA value. It + must indicate a valid lane index for the type of ``x``. Note that the upper bits of ``a`` + may or may not be zeroed depending on the ISA but the type system should prevent using + ``a`` as anything other than the extracted value. + "#, + &formats.binary_imm8, + ) + .operands_in(vec![x, Idx]) + .operands_out(vec![a]), + ); +} + +#[inline(never)] +fn define_simd_arithmetic( + ig: &mut InstructionGroupBuilder, + formats: &Formats, + _: &Immediates, + _: &EntityRefs, +) { + let Int = &TypeVar::new( + "Int", + "A scalar or vector integer type", + TypeSetBuilder::new() + .ints(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + + let a = &Operand::new("a", Int); + let x = &Operand::new("x", Int); + let y = &Operand::new("y", Int); + + ig.push( + Inst::new( + "imin", + r#" + Signed integer minimum. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "umin", + r#" + Unsigned integer minimum. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "imax", + r#" + Signed integer maximum. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "umax", + r#" + Unsigned integer maximum. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + let IxN = &TypeVar::new( + "IxN", + "A SIMD vector type containing integers", + TypeSetBuilder::new() + .ints(Interval::All) + .simd_lanes(Interval::All) + .includes_scalars(false) + .build(), + ); + + let a = &Operand::new("a", IxN); + let x = &Operand::new("x", IxN); + let y = &Operand::new("y", IxN); + + ig.push( + Inst::new( + "avg_round", + r#" + Unsigned average with rounding: `a := (x + y + 1) // 2` + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); +} + +#[allow(clippy::many_single_char_names)] +pub(crate) fn define( + all_instructions: &mut AllInstructions, + formats: &Formats, + imm: &Immediates, + entities: &EntityRefs, +) -> InstructionGroup { + let mut ig = InstructionGroupBuilder::new(all_instructions); + + define_control_flow(&mut ig, formats, imm, entities); + define_simd_lane_access(&mut ig, formats, imm, entities); + define_simd_arithmetic(&mut ig, formats, imm, entities); + + // Operand kind shorthands. + let iflags: &TypeVar = &ValueType::Special(types::Flag::IFlags.into()).into(); + let fflags: &TypeVar = &ValueType::Special(types::Flag::FFlags.into()).into(); + + let b1: &TypeVar = &ValueType::from(LaneType::from(types::Bool::B1)).into(); + let f32_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F32)).into(); + let f64_: &TypeVar = &ValueType::from(LaneType::from(types::Float::F64)).into(); + + // Starting definitions. + let Int = &TypeVar::new( + "Int", + "A scalar or vector integer type", + TypeSetBuilder::new() + .ints(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + + let Bool = &TypeVar::new( + "Bool", + "A scalar or vector boolean type", + TypeSetBuilder::new() + .bools(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + + let iB = &TypeVar::new( + "iB", + "A scalar integer type", + TypeSetBuilder::new().ints(Interval::All).build(), + ); + + let iAddr = &TypeVar::new( + "iAddr", + "An integer address type", + TypeSetBuilder::new().ints(32..64).refs(32..64).build(), + ); + + let Ref = &TypeVar::new( + "Ref", + "A scalar reference type", + TypeSetBuilder::new().refs(Interval::All).build(), + ); + + let Testable = &TypeVar::new( + "Testable", + "A scalar boolean or integer type", + TypeSetBuilder::new() + .ints(Interval::All) + .bools(Interval::All) + .build(), + ); + + let TxN = &TypeVar::new( + "TxN", + "A SIMD vector type", + TypeSetBuilder::new() + .ints(Interval::All) + .floats(Interval::All) + .bools(Interval::All) + .simd_lanes(Interval::All) + .includes_scalars(false) + .build(), + ); + let Any = &TypeVar::new( + "Any", + "Any integer, float, boolean, or reference scalar or vector type", + TypeSetBuilder::new() + .ints(Interval::All) + .floats(Interval::All) + .bools(Interval::All) + .refs(Interval::All) + .simd_lanes(Interval::All) + .includes_scalars(true) + .build(), + ); + + let AnyTo = &TypeVar::copy_from(Any, "AnyTo".to_string()); + + let Mem = &TypeVar::new( + "Mem", + "Any type that can be stored in memory", + TypeSetBuilder::new() + .ints(Interval::All) + .floats(Interval::All) + .simd_lanes(Interval::All) + .refs(Interval::All) + .build(), + ); + + let MemTo = &TypeVar::copy_from(Mem, "MemTo".to_string()); + + let addr = &Operand::new("addr", iAddr); + + let SS = &Operand::new("SS", &entities.stack_slot); + let Offset = &Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"); + let x = &Operand::new("x", Mem).with_doc("Value to be stored"); + let a = &Operand::new("a", Mem).with_doc("Value loaded"); + let p = &Operand::new("p", iAddr); + let MemFlags = &Operand::new("MemFlags", &imm.memflags); + let args = &Operand::new("args", &entities.varargs).with_doc("Address arguments"); + + ig.push( + Inst::new( + "load", + r#" + Load from memory at ``p + Offset``. + + This is a polymorphic instruction that can load any value type which + has a memory representation. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "load_complex", + r#" + Load from memory at ``sum(args) + Offset``. + + This is a polymorphic instruction that can load any value type which + has a memory representation. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "store", + r#" + Store ``x`` to memory at ``p + Offset``. + + This is a polymorphic instruction that can store any value type with a + memory representation. + "#, + &formats.store, + ) + .operands_in(vec![MemFlags, x, p, Offset]) + .can_store(true), + ); + + ig.push( + Inst::new( + "store_complex", + r#" + Store ``x`` to memory at ``sum(args) + Offset``. + + This is a polymorphic instruction that can store any value type with a + memory representation. + "#, + &formats.store_complex, + ) + .operands_in(vec![MemFlags, x, args, Offset]) + .can_store(true), + ); + + let iExt8 = &TypeVar::new( + "iExt8", + "An integer type with more than 8 bits", + TypeSetBuilder::new().ints(16..64).build(), + ); + let x = &Operand::new("x", iExt8); + let a = &Operand::new("a", iExt8); + + ig.push( + Inst::new( + "uload8", + r#" + Load 8 bits from memory at ``p + Offset`` and zero-extend. + + This is equivalent to ``load.i8`` followed by ``uextend``. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "uload8_complex", + r#" + Load 8 bits from memory at ``sum(args) + Offset`` and zero-extend. + + This is equivalent to ``load.i8`` followed by ``uextend``. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload8", + r#" + Load 8 bits from memory at ``p + Offset`` and sign-extend. + + This is equivalent to ``load.i8`` followed by ``sextend``. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload8_complex", + r#" + Load 8 bits from memory at ``sum(args) + Offset`` and sign-extend. + + This is equivalent to ``load.i8`` followed by ``sextend``. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "istore8", + r#" + Store the low 8 bits of ``x`` to memory at ``p + Offset``. + + This is equivalent to ``ireduce.i8`` followed by ``store.i8``. + "#, + &formats.store, + ) + .operands_in(vec![MemFlags, x, p, Offset]) + .can_store(true), + ); + + ig.push( + Inst::new( + "istore8_complex", + r#" + Store the low 8 bits of ``x`` to memory at ``sum(args) + Offset``. + + This is equivalent to ``ireduce.i8`` followed by ``store.i8``. + "#, + &formats.store_complex, + ) + .operands_in(vec![MemFlags, x, args, Offset]) + .can_store(true), + ); + + let iExt16 = &TypeVar::new( + "iExt16", + "An integer type with more than 16 bits", + TypeSetBuilder::new().ints(32..64).build(), + ); + let x = &Operand::new("x", iExt16); + let a = &Operand::new("a", iExt16); + + ig.push( + Inst::new( + "uload16", + r#" + Load 16 bits from memory at ``p + Offset`` and zero-extend. + + This is equivalent to ``load.i16`` followed by ``uextend``. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "uload16_complex", + r#" + Load 16 bits from memory at ``sum(args) + Offset`` and zero-extend. + + This is equivalent to ``load.i16`` followed by ``uextend``. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload16", + r#" + Load 16 bits from memory at ``p + Offset`` and sign-extend. + + This is equivalent to ``load.i16`` followed by ``sextend``. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload16_complex", + r#" + Load 16 bits from memory at ``sum(args) + Offset`` and sign-extend. + + This is equivalent to ``load.i16`` followed by ``sextend``. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "istore16", + r#" + Store the low 16 bits of ``x`` to memory at ``p + Offset``. + + This is equivalent to ``ireduce.i16`` followed by ``store.i16``. + "#, + &formats.store, + ) + .operands_in(vec![MemFlags, x, p, Offset]) + .can_store(true), + ); + + ig.push( + Inst::new( + "istore16_complex", + r#" + Store the low 16 bits of ``x`` to memory at ``sum(args) + Offset``. + + This is equivalent to ``ireduce.i16`` followed by ``store.i16``. + "#, + &formats.store_complex, + ) + .operands_in(vec![MemFlags, x, args, Offset]) + .can_store(true), + ); + + let iExt32 = &TypeVar::new( + "iExt32", + "An integer type with more than 32 bits", + TypeSetBuilder::new().ints(64..64).build(), + ); + let x = &Operand::new("x", iExt32); + let a = &Operand::new("a", iExt32); + + ig.push( + Inst::new( + "uload32", + r#" + Load 32 bits from memory at ``p + Offset`` and zero-extend. + + This is equivalent to ``load.i32`` followed by ``uextend``. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "uload32_complex", + r#" + Load 32 bits from memory at ``sum(args) + Offset`` and zero-extend. + + This is equivalent to ``load.i32`` followed by ``uextend``. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload32", + r#" + Load 32 bits from memory at ``p + Offset`` and sign-extend. + + This is equivalent to ``load.i32`` followed by ``sextend``. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload32_complex", + r#" + Load 32 bits from memory at ``sum(args) + Offset`` and sign-extend. + + This is equivalent to ``load.i32`` followed by ``sextend``. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "istore32", + r#" + Store the low 32 bits of ``x`` to memory at ``p + Offset``. + + This is equivalent to ``ireduce.i32`` followed by ``store.i32``. + "#, + &formats.store, + ) + .operands_in(vec![MemFlags, x, p, Offset]) + .can_store(true), + ); + + ig.push( + Inst::new( + "istore32_complex", + r#" + Store the low 32 bits of ``x`` to memory at ``sum(args) + Offset``. + + This is equivalent to ``ireduce.i32`` followed by ``store.i32``. + "#, + &formats.store_complex, + ) + .operands_in(vec![MemFlags, x, args, Offset]) + .can_store(true), + ); + + let I16x8 = &TypeVar::new( + "I16x8", + "A SIMD vector with exactly 8 lanes of 16-bit values", + TypeSetBuilder::new() + .ints(16..16) + .simd_lanes(8..8) + .includes_scalars(false) + .build(), + ); + let a = &Operand::new("a", I16x8).with_doc("Value loaded"); + + ig.push( + Inst::new( + "uload8x8", + r#" + Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i16x8 + vector. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "uload8x8_complex", + r#" + Load an 8x8 vector (64 bits) from memory at ``sum(args) + Offset`` and zero-extend into an + i16x8 vector. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload8x8", + r#" + Load an 8x8 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i16x8 + vector. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload8x8_complex", + r#" + Load an 8x8 vector (64 bits) from memory at ``sum(args) + Offset`` and sign-extend into an + i16x8 vector. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + let I32x4 = &TypeVar::new( + "I32x4", + "A SIMD vector with exactly 4 lanes of 32-bit values", + TypeSetBuilder::new() + .ints(32..32) + .simd_lanes(4..4) + .includes_scalars(false) + .build(), + ); + let a = &Operand::new("a", I32x4).with_doc("Value loaded"); + + ig.push( + Inst::new( + "uload16x4", + r#" + Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i32x4 + vector. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "uload16x4_complex", + r#" + Load a 16x4 vector (64 bits) from memory at ``sum(args) + Offset`` and zero-extend into an + i32x4 vector. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload16x4", + r#" + Load a 16x4 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i32x4 + vector. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload16x4_complex", + r#" + Load a 16x4 vector (64 bits) from memory at ``sum(args) + Offset`` and sign-extend into an + i32x4 vector. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + let I64x2 = &TypeVar::new( + "I64x2", + "A SIMD vector with exactly 2 lanes of 64-bit values", + TypeSetBuilder::new() + .ints(64..64) + .simd_lanes(2..2) + .includes_scalars(false) + .build(), + ); + let a = &Operand::new("a", I64x2).with_doc("Value loaded"); + + ig.push( + Inst::new( + "uload32x2", + r#" + Load an 32x2 vector (64 bits) from memory at ``p + Offset`` and zero-extend into an i64x2 + vector. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "uload32x2_complex", + r#" + Load a 32x2 vector (64 bits) from memory at ``sum(args) + Offset`` and zero-extend into an + i64x2 vector. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload32x2", + r#" + Load a 32x2 vector (64 bits) from memory at ``p + Offset`` and sign-extend into an i64x2 + vector. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "sload32x2_complex", + r#" + Load a 32x2 vector (64 bits) from memory at ``sum(args) + Offset`` and sign-extend into an + i64x2 vector. + "#, + &formats.load_complex, + ) + .operands_in(vec![MemFlags, args, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + let x = &Operand::new("x", Mem).with_doc("Value to be stored"); + let a = &Operand::new("a", Mem).with_doc("Value loaded"); + let Offset = + &Operand::new("Offset", &imm.offset32).with_doc("In-bounds offset into stack slot"); + + ig.push( + Inst::new( + "stack_load", + r#" + Load a value from a stack slot at the constant offset. + + This is a polymorphic instruction that can load any value type which + has a memory representation. + + The offset is an immediate constant, not an SSA value. The memory + access cannot go out of bounds, i.e. + `sizeof(a) + Offset <= sizeof(SS)`. + "#, + &formats.stack_load, + ) + .operands_in(vec![SS, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "stack_store", + r#" + Store a value to a stack slot at a constant offset. + + This is a polymorphic instruction that can store any value type with a + memory representation. + + The offset is an immediate constant, not an SSA value. The memory + access cannot go out of bounds, i.e. + `sizeof(a) + Offset <= sizeof(SS)`. + "#, + &formats.stack_store, + ) + .operands_in(vec![x, SS, Offset]) + .can_store(true), + ); + + ig.push( + Inst::new( + "stack_addr", + r#" + Get the address of a stack slot. + + Compute the absolute address of a byte in a stack slot. The offset must + refer to a byte inside the stack slot: + `0 <= Offset < sizeof(SS)`. + "#, + &formats.stack_load, + ) + .operands_in(vec![SS, Offset]) + .operands_out(vec![addr]), + ); + + let GV = &Operand::new("GV", &entities.global_value); + + ig.push( + Inst::new( + "global_value", + r#" + Compute the value of global GV. + "#, + &formats.unary_global_value, + ) + .operands_in(vec![GV]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "symbol_value", + r#" + Compute the value of global GV, which is a symbolic value. + "#, + &formats.unary_global_value, + ) + .operands_in(vec![GV]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "tls_value", + r#" + Compute the value of global GV, which is a TLS (thread local storage) value. + "#, + &formats.unary_global_value, + ) + .operands_in(vec![GV]) + .operands_out(vec![a]), + ); + + let HeapOffset = &TypeVar::new( + "HeapOffset", + "An unsigned heap offset", + TypeSetBuilder::new().ints(32..64).build(), + ); + + let H = &Operand::new("H", &entities.heap); + let p = &Operand::new("p", HeapOffset); + let Size = &Operand::new("Size", &imm.uimm32).with_doc("Size in bytes"); + + ig.push( + Inst::new( + "heap_addr", + r#" + Bounds check and compute absolute address of heap memory. + + Verify that the offset range ``p .. p + Size - 1`` is in bounds for the + heap H, and generate an absolute address that is safe to dereference. + + 1. If ``p + Size`` is not greater than the heap bound, return an + absolute address corresponding to a byte offset of ``p`` from the + heap's base address. + 2. If ``p + Size`` is greater than the heap bound, generate a trap. + "#, + &formats.heap_addr, + ) + .operands_in(vec![H, p, Size]) + .operands_out(vec![addr]), + ); + + // Note this instruction is marked as having other side-effects, so GVN won't try to hoist it, + // which would result in it being subject to spilling. While not hoisting would generally hurt + // performance, since a computed value used many times may need to be regenerated before each + // use, it is not the case here: this instruction doesn't generate any code. That's because, + // by definition the pinned register is never used by the register allocator, but is written to + // and read explicitly and exclusively by set_pinned_reg and get_pinned_reg. + ig.push( + Inst::new( + "get_pinned_reg", + r#" + Gets the content of the pinned register, when it's enabled. + "#, + &formats.nullary, + ) + .operands_out(vec![addr]) + .other_side_effects(true), + ); + + ig.push( + Inst::new( + "set_pinned_reg", + r#" + Sets the content of the pinned register, when it's enabled. + "#, + &formats.unary, + ) + .operands_in(vec![addr]) + .other_side_effects(true), + ); + + let TableOffset = &TypeVar::new( + "TableOffset", + "An unsigned table offset", + TypeSetBuilder::new().ints(32..64).build(), + ); + let T = &Operand::new("T", &entities.table); + let p = &Operand::new("p", TableOffset); + let Offset = + &Operand::new("Offset", &imm.offset32).with_doc("Byte offset from element address"); + + ig.push( + Inst::new( + "table_addr", + r#" + Bounds check and compute absolute address of a table entry. + + Verify that the offset ``p`` is in bounds for the table T, and generate + an absolute address that is safe to dereference. + + ``Offset`` must be less than the size of a table element. + + 1. If ``p`` is not greater than the table bound, return an absolute + address corresponding to a byte offset of ``p`` from the table's + base address. + 2. If ``p`` is greater than the table bound, generate a trap. + "#, + &formats.table_addr, + ) + .operands_in(vec![T, p, Offset]) + .operands_out(vec![addr]), + ); + + let N = &Operand::new("N", &imm.imm64); + let a = &Operand::new("a", Int).with_doc("A constant integer scalar or vector value"); + + ig.push( + Inst::new( + "iconst", + r#" + Integer constant. + + Create a scalar integer SSA value with an immediate constant value, or + an integer vector where all the lanes have the same value. + "#, + &formats.unary_imm, + ) + .operands_in(vec![N]) + .operands_out(vec![a]), + ); + + let N = &Operand::new("N", &imm.ieee32); + let a = &Operand::new("a", f32_).with_doc("A constant f32 scalar value"); + + ig.push( + Inst::new( + "f32const", + r#" + Floating point constant. + + Create a `f32` SSA value with an immediate constant value. + "#, + &formats.unary_ieee32, + ) + .operands_in(vec![N]) + .operands_out(vec![a]), + ); + + let N = &Operand::new("N", &imm.ieee64); + let a = &Operand::new("a", f64_).with_doc("A constant f64 scalar value"); + + ig.push( + Inst::new( + "f64const", + r#" + Floating point constant. + + Create a `f64` SSA value with an immediate constant value. + "#, + &formats.unary_ieee64, + ) + .operands_in(vec![N]) + .operands_out(vec![a]), + ); + + let N = &Operand::new("N", &imm.boolean); + let a = &Operand::new("a", Bool).with_doc("A constant boolean scalar or vector value"); + + ig.push( + Inst::new( + "bconst", + r#" + Boolean constant. + + Create a scalar boolean SSA value with an immediate constant value, or + a boolean vector where all the lanes have the same value. + "#, + &formats.unary_bool, + ) + .operands_in(vec![N]) + .operands_out(vec![a]), + ); + + let N = &Operand::new("N", &imm.pool_constant) + .with_doc("The 16 immediate bytes of a 128-bit vector"); + let a = &Operand::new("a", TxN).with_doc("A constant vector value"); + + ig.push( + Inst::new( + "vconst", + r#" + SIMD vector constant. + + Construct a vector with the given immediate bytes. + "#, + &formats.unary_const, + ) + .operands_in(vec![N]) + .operands_out(vec![a]), + ); + + let constant = + &Operand::new("constant", &imm.pool_constant).with_doc("A constant in the constant pool"); + let address = &Operand::new("address", iAddr); + ig.push( + Inst::new( + "const_addr", + r#" + Calculate the base address of a value in the constant pool. + "#, + &formats.unary_const, + ) + .operands_in(vec![constant]) + .operands_out(vec![address]), + ); + + let mask = &Operand::new("mask", &imm.uimm128) + .with_doc("The 16 immediate bytes used for selecting the elements to shuffle"); + let Tx16 = &TypeVar::new( + "Tx16", + "A SIMD vector with exactly 16 lanes of 8-bit values; eventually this may support other \ + lane counts and widths", + TypeSetBuilder::new() + .ints(8..8) + .bools(8..8) + .simd_lanes(16..16) + .includes_scalars(false) + .build(), + ); + let a = &Operand::new("a", Tx16).with_doc("A vector value"); + let b = &Operand::new("b", Tx16).with_doc("A vector value"); + + ig.push( + Inst::new( + "shuffle", + r#" + SIMD vector shuffle. + + Shuffle two vectors using the given immediate bytes. For each of the 16 bytes of the + immediate, a value i of 0-15 selects the i-th element of the first vector and a value i of + 16-31 selects the (i-16)th element of the second vector. Immediate values outside of the + 0-31 range place a 0 in the resulting vector lane. + "#, + &formats.shuffle, + ) + .operands_in(vec![a, b, mask]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", Ref).with_doc("A constant reference null value"); + + ig.push( + Inst::new( + "null", + r#" + Null constant value for reference types. + + Create a scalar reference SSA value with a constant null value. + "#, + &formats.nullary, + ) + .operands_out(vec![a]), + ); + + ig.push(Inst::new( + "nop", + r#" + Just a dummy instruction. + + Note: this doesn't compile to a machine code nop. + "#, + &formats.nullary, + )); + + let c = &Operand::new("c", Testable).with_doc("Controlling value to test"); + let x = &Operand::new("x", Any).with_doc("Value to use when `c` is true"); + let y = &Operand::new("y", Any).with_doc("Value to use when `c` is false"); + let a = &Operand::new("a", Any); + + ig.push( + Inst::new( + "select", + r#" + Conditional select. + + This instruction selects whole values. Use `vselect` for + lane-wise selection. + "#, + &formats.ternary, + ) + .operands_in(vec![c, x, y]) + .operands_out(vec![a]), + ); + + let cc = &Operand::new("cc", &imm.intcc).with_doc("Controlling condition code"); + let flags = &Operand::new("flags", iflags).with_doc("The machine's flag register"); + + ig.push( + Inst::new( + "selectif", + r#" + Conditional select, dependent on integer condition codes. + "#, + &formats.int_select, + ) + .operands_in(vec![cc, flags, x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "selectif_spectre_guard", + r#" + Conditional select intended for Spectre guards. + + This operation is semantically equivalent to a selectif instruction. + However, it is guaranteed to not be removed or otherwise altered by any + optimization pass, and is guaranteed to result in a conditional-move + instruction, not a branch-based lowering. As such, it is suitable + for use when producing Spectre guards. For example, a bounds-check + may guard against unsafe speculation past a bounds-check conditional + branch by passing the address or index to be accessed through a + conditional move, also gated on the same condition. Because no + Spectre-vulnerable processors are known to perform speculation on + conditional move instructions, this is guaranteed to pick the + correct input. If the selected input in case of overflow is a "safe" + value, for example a null pointer that causes an exception in the + speculative path, this ensures that no Spectre vulnerability will + exist. + "#, + &formats.int_select, + ) + .operands_in(vec![cc, flags, x, y]) + .operands_out(vec![a]) + .other_side_effects(true), + ); + + let c = &Operand::new("c", Any).with_doc("Controlling value to test"); + ig.push( + Inst::new( + "bitselect", + r#" + Conditional select of bits. + + For each bit in `c`, this instruction selects the corresponding bit from `x` if the bit + in `c` is 1 and the corresponding bit from `y` if the bit in `c` is 0. See also: + `select`, `vselect`. + "#, + &formats.ternary, + ) + .operands_in(vec![c, x, y]) + .operands_out(vec![a]), + ); + + let x = &Operand::new("x", Any); + + ig.push( + Inst::new( + "copy", + r#" + Register-register copy. + + This instruction copies its input, preserving the value type. + + A pure SSA-form program does not need to copy values, but this + instruction is useful for representing intermediate stages during + instruction transformations, and the register allocator needs a way of + representing register copies. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "spill", + r#" + Spill a register value to a stack slot. + + This instruction behaves exactly like `copy`, but the result + value is assigned to a spill slot. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .can_store(true), + ); + + ig.push( + Inst::new( + "fill", + r#" + Load a register value from a stack slot. + + This instruction behaves exactly like `copy`, but creates a new + SSA value for the spilled input value. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.push( + Inst::new( + "fill_nop", + r#" + This is identical to `fill`, except it has no encoding, since it is a no-op. + + This instruction is created only during late-stage redundant-reload removal, after all + registers and stack slots have been assigned. It is used to replace `fill`s that have + been identified as redundant. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .can_load(true), + ); + + let Sarg = &TypeVar::new( + "Sarg", + "Any scalar or vector type with at most 128 lanes", + TypeSetBuilder::new() + .specials(vec![crate::cdsl::types::SpecialType::StructArgument]) + .build(), + ); + let sarg_t = &Operand::new("sarg_t", Sarg); + + // FIXME remove once the old style codegen backends are removed. + ig.push( + Inst::new( + "dummy_sarg_t", + r#" + This creates a sarg_t + + This instruction is internal and should not be created by + Cranelift users. + "#, + &formats.nullary, + ) + .operands_in(vec![]) + .operands_out(vec![sarg_t]), + ); + + let src = &Operand::new("src", &imm.regunit); + let dst = &Operand::new("dst", &imm.regunit); + + ig.push( + Inst::new( + "regmove", + r#" + Temporarily divert ``x`` from ``src`` to ``dst``. + + This instruction moves the location of a value from one register to + another without creating a new SSA value. It is used by the register + allocator to temporarily rearrange register assignments in order to + satisfy instruction constraints. + + The register diversions created by this instruction must be undone + before the value leaves the block. At the entry to a new block, all live + values must be in their originally assigned registers. + "#, + &formats.reg_move, + ) + .operands_in(vec![x, src, dst]) + .other_side_effects(true), + ); + + ig.push( + Inst::new( + "copy_special", + r#" + Copies the contents of ''src'' register to ''dst'' register. + + This instructions copies the contents of one register to another + register without involving any SSA values. This is used for copying + special registers, e.g. copying the stack register to the frame + register in a function prologue. + "#, + &formats.copy_special, + ) + .operands_in(vec![src, dst]) + .other_side_effects(true), + ); + + ig.push( + Inst::new( + "copy_to_ssa", + r#" + Copies the contents of ''src'' register to ''a'' SSA name. + + This instruction copies the contents of one register, regardless of its SSA name, to + another register, creating a new SSA name. In that sense it is a one-sided version + of ''copy_special''. This instruction is internal and should not be created by + Cranelift users. + "#, + &formats.copy_to_ssa, + ) + .operands_in(vec![src]) + .operands_out(vec![a]) + .other_side_effects(true), + ); + + ig.push( + Inst::new( + "copy_nop", + r#" + Stack-slot-to-the-same-stack-slot copy, which is guaranteed to turn + into a no-op. This instruction is for use only within Cranelift itself. + + This instruction copies its input, preserving the value type. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let delta = &Operand::new("delta", Int); + + ig.push( + Inst::new( + "adjust_sp_down", + r#" + Subtracts ``delta`` offset value from the stack pointer register. + + This instruction is used to adjust the stack pointer by a dynamic amount. + "#, + &formats.unary, + ) + .operands_in(vec![delta]) + .other_side_effects(true), + ); + + let Offset = &Operand::new("Offset", &imm.imm64).with_doc("Offset from current stack pointer"); + + ig.push( + Inst::new( + "adjust_sp_up_imm", + r#" + Adds ``Offset`` immediate offset value to the stack pointer register. + + This instruction is used to adjust the stack pointer, primarily in function + prologues and epilogues. ``Offset`` is constrained to the size of a signed + 32-bit integer. + "#, + &formats.unary_imm, + ) + .operands_in(vec![Offset]) + .other_side_effects(true), + ); + + let Offset = &Operand::new("Offset", &imm.imm64).with_doc("Offset from current stack pointer"); + + ig.push( + Inst::new( + "adjust_sp_down_imm", + r#" + Subtracts ``Offset`` immediate offset value from the stack pointer + register. + + This instruction is used to adjust the stack pointer, primarily in function + prologues and epilogues. ``Offset`` is constrained to the size of a signed + 32-bit integer. + "#, + &formats.unary_imm, + ) + .operands_in(vec![Offset]) + .other_side_effects(true), + ); + + let f = &Operand::new("f", iflags); + + ig.push( + Inst::new( + "ifcmp_sp", + r#" + Compare ``addr`` with the stack pointer and set the CPU flags. + + This is like `ifcmp` where ``addr`` is the LHS operand and the stack + pointer is the RHS. + "#, + &formats.unary, + ) + .operands_in(vec![addr]) + .operands_out(vec![f]), + ); + + ig.push( + Inst::new( + "regspill", + r#" + Temporarily divert ``x`` from ``src`` to ``SS``. + + This instruction moves the location of a value from a register to a + stack slot without creating a new SSA value. It is used by the register + allocator to temporarily rearrange register assignments in order to + satisfy instruction constraints. + + See also `regmove`. + "#, + &formats.reg_spill, + ) + .operands_in(vec![x, src, SS]) + .other_side_effects(true), + ); + + ig.push( + Inst::new( + "regfill", + r#" + Temporarily divert ``x`` from ``SS`` to ``dst``. + + This instruction moves the location of a value from a stack slot to a + register without creating a new SSA value. It is used by the register + allocator to temporarily rearrange register assignments in order to + satisfy instruction constraints. + + See also `regmove`. + "#, + &formats.reg_fill, + ) + .operands_in(vec![x, SS, dst]) + .other_side_effects(true), + ); + + let N = + &Operand::new("args", &entities.varargs).with_doc("Variable number of args for StackMap"); + + ig.push( + Inst::new( + "safepoint", + r#" + This instruction will provide live reference values at a point in + the function. It can only be used by the compiler. + "#, + &formats.multiary, + ) + .operands_in(vec![N]) + .other_side_effects(true), + ); + + let x = &Operand::new("x", TxN).with_doc("Vector to split"); + let lo = &Operand::new("lo", &TxN.half_vector()).with_doc("Low-numbered lanes of `x`"); + let hi = &Operand::new("hi", &TxN.half_vector()).with_doc("High-numbered lanes of `x`"); + + ig.push( + Inst::new( + "vsplit", + r#" + Split a vector into two halves. + + Split the vector `x` into two separate values, each containing half of + the lanes from ``x``. The result may be two scalars if ``x`` only had + two lanes. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![lo, hi]) + .is_ghost(true), + ); + + let Any128 = &TypeVar::new( + "Any128", + "Any scalar or vector type with as most 128 lanes", + TypeSetBuilder::new() + .ints(Interval::All) + .floats(Interval::All) + .bools(Interval::All) + .simd_lanes(1..128) + .includes_scalars(true) + .build(), + ); + + let x = &Operand::new("x", Any128).with_doc("Low-numbered lanes"); + let y = &Operand::new("y", Any128).with_doc("High-numbered lanes"); + let a = &Operand::new("a", &Any128.double_vector()).with_doc("Concatenation of `x` and `y`"); + + ig.push( + Inst::new( + "vconcat", + r#" + Vector concatenation. + + Return a vector formed by concatenating ``x`` and ``y``. The resulting + vector type has twice as many lanes as each of the inputs. The lanes of + ``x`` appear as the low-numbered lanes, and the lanes of ``y`` become + the high-numbered lanes of ``a``. + + It is possible to form a vector by concatenating two scalars. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]) + .is_ghost(true), + ); + + let c = &Operand::new("c", &TxN.as_bool()).with_doc("Controlling vector"); + let x = &Operand::new("x", TxN).with_doc("Value to use where `c` is true"); + let y = &Operand::new("y", TxN).with_doc("Value to use where `c` is false"); + let a = &Operand::new("a", TxN); + + ig.push( + Inst::new( + "vselect", + r#" + Vector lane select. + + Select lanes from ``x`` or ``y`` controlled by the lanes of the boolean + vector ``c``. + "#, + &formats.ternary, + ) + .operands_in(vec![c, x, y]) + .operands_out(vec![a]), + ); + + let s = &Operand::new("s", b1); + + ig.push( + Inst::new( + "vany_true", + r#" + Reduce a vector to a scalar boolean. + + Return a scalar boolean true if any lane in ``a`` is non-zero, false otherwise. + "#, + &formats.unary, + ) + .operands_in(vec![a]) + .operands_out(vec![s]), + ); + + ig.push( + Inst::new( + "vall_true", + r#" + Reduce a vector to a scalar boolean. + + Return a scalar boolean true if all lanes in ``i`` are non-zero, false otherwise. + "#, + &formats.unary, + ) + .operands_in(vec![a]) + .operands_out(vec![s]), + ); + + let a = &Operand::new("a", TxN); + let x = &Operand::new("x", Int); + + ig.push( + Inst::new( + "vhigh_bits", + r#" + Reduce a vector to a scalar integer. + + Return a scalar integer, consisting of the concatenation of the most significant bit + of each lane of ``a``. + "#, + &formats.unary, + ) + .operands_in(vec![a]) + .operands_out(vec![x]), + ); + + let a = &Operand::new("a", &Int.as_bool()); + let Cond = &Operand::new("Cond", &imm.intcc); + let x = &Operand::new("x", Int); + let y = &Operand::new("y", Int); + + ig.push( + Inst::new( + "icmp", + r#" + Integer comparison. + + The condition code determines if the operands are interpreted as signed + or unsigned integers. + + | Signed | Unsigned | Condition | + |--------|----------|-----------------------| + | eq | eq | Equal | + | ne | ne | Not equal | + | slt | ult | Less than | + | sge | uge | Greater than or equal | + | sgt | ugt | Greater than | + | sle | ule | Less than or equal | + | of | * | Overflow | + | nof | * | No Overflow | + + \* The unsigned version of overflow conditions have ISA-specific + semantics and thus have been kept as methods on the TargetIsa trait as + [unsigned_add_overflow_condition][isa::TargetIsa::unsigned_add_overflow_condition] and + [unsigned_sub_overflow_condition][isa::TargetIsa::unsigned_sub_overflow_condition]. + + When this instruction compares integer vectors, it returns a boolean + vector of lane-wise comparisons. + "#, + &formats.int_compare, + ) + .operands_in(vec![Cond, x, y]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", b1); + let x = &Operand::new("x", iB); + let Y = &Operand::new("Y", &imm.imm64); + + ig.push( + Inst::new( + "icmp_imm", + r#" + Compare scalar integer to a constant. + + This is the same as the `icmp` instruction, except one operand is + an immediate constant. + + This instruction can only compare scalars. Use `icmp` for + lane-wise vector comparisons. + "#, + &formats.int_compare_imm, + ) + .operands_in(vec![Cond, x, Y]) + .operands_out(vec![a]), + ); + + let f = &Operand::new("f", iflags); + let x = &Operand::new("x", iB); + let y = &Operand::new("y", iB); + + ig.push( + Inst::new( + "ifcmp", + r#" + Compare scalar integers and return flags. + + Compare two scalar integer values and return integer CPU flags + representing the result. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![f]), + ); + + ig.push( + Inst::new( + "ifcmp_imm", + r#" + Compare scalar integer to a constant and return flags. + + Like `icmp_imm`, but returns integer CPU flags instead of testing + a specific condition code. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![f]), + ); + + let a = &Operand::new("a", Int); + let x = &Operand::new("x", Int); + let y = &Operand::new("y", Int); + + ig.push( + Inst::new( + "iadd", + r#" + Wrapping integer addition: `a := x + y \pmod{2^B}`. + + This instruction does not depend on the signed/unsigned interpretation + of the operands. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "uadd_sat", + r#" + Add with unsigned saturation. + + This is similar to `iadd` but the operands are interpreted as unsigned integers and their + summed result, instead of wrapping, will be saturated to the highest unsigned integer for + the controlling type (e.g. `0xFF` for i8). + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "sadd_sat", + r#" + Add with signed saturation. + + This is similar to `iadd` but the operands are interpreted as signed integers and their + summed result, instead of wrapping, will be saturated to the lowest or highest + signed integer for the controlling type (e.g. `0x80` or `0x7F` for i8). For example, + since an `sadd_sat.i8` of `0x70` and `0x70` is greater than `0x7F`, the result will be + clamped to `0x7F`. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "isub", + r#" + Wrapping integer subtraction: `a := x - y \pmod{2^B}`. + + This instruction does not depend on the signed/unsigned interpretation + of the operands. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "usub_sat", + r#" + Subtract with unsigned saturation. + + This is similar to `isub` but the operands are interpreted as unsigned integers and their + difference, instead of wrapping, will be saturated to the lowest unsigned integer for + the controlling type (e.g. `0x00` for i8). + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "ssub_sat", + r#" + Subtract with signed saturation. + + This is similar to `isub` but the operands are interpreted as signed integers and their + difference, instead of wrapping, will be saturated to the lowest or highest + signed integer for the controlling type (e.g. `0x80` or `0x7F` for i8). + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "ineg", + r#" + Integer negation: `a := -x \pmod{2^B}`. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "iabs", + r#" + Integer absolute value with wrapping: `a := |x|`. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "imul", + r#" + Wrapping integer multiplication: `a := x y \pmod{2^B}`. + + This instruction does not depend on the signed/unsigned interpretation + of the operands. + + Polymorphic over all integer types (vector and scalar). + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "umulhi", + r#" + Unsigned integer multiplication, producing the high half of a + double-length result. + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "smulhi", + r#" + Signed integer multiplication, producing the high half of a + double-length result. + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "udiv", + r#" + Unsigned integer division: `a := \lfloor {x \over y} \rfloor`. + + This operation traps if the divisor is zero. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]) + .can_trap(true), + ); + + ig.push( + Inst::new( + "sdiv", + r#" + Signed integer division rounded toward zero: `a := sign(xy) + \lfloor {|x| \over |y|}\rfloor`. + + This operation traps if the divisor is zero, or if the result is not + representable in `B` bits two's complement. This only happens + when `x = -2^{B-1}, y = -1`. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]) + .can_trap(true), + ); + + ig.push( + Inst::new( + "urem", + r#" + Unsigned integer remainder. + + This operation traps if the divisor is zero. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]) + .can_trap(true), + ); + + ig.push( + Inst::new( + "srem", + r#" + Signed integer remainder. The result has the sign of the dividend. + + This operation traps if the divisor is zero. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]) + .can_trap(true), + ); + + let a = &Operand::new("a", iB); + let x = &Operand::new("x", iB); + let Y = &Operand::new("Y", &imm.imm64); + + ig.push( + Inst::new( + "iadd_imm", + r#" + Add immediate integer. + + Same as `iadd`, but one operand is an immediate constant. + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "imul_imm", + r#" + Integer multiplication by immediate constant. + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "udiv_imm", + r#" + Unsigned integer division by an immediate constant. + + This operation traps if the divisor is zero. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "sdiv_imm", + r#" + Signed integer division by an immediate constant. + + This operation traps if the divisor is zero, or if the result is not + representable in `B` bits two's complement. This only happens + when `x = -2^{B-1}, Y = -1`. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "urem_imm", + r#" + Unsigned integer remainder with immediate divisor. + + This operation traps if the divisor is zero. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "srem_imm", + r#" + Signed integer remainder with immediate divisor. + + This operation traps if the divisor is zero. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "irsub_imm", + r#" + Immediate reverse wrapping subtraction: `a := Y - x \pmod{2^B}`. + + Also works as integer negation when `Y = 0`. Use `iadd_imm` + with a negative immediate operand for the reverse immediate + subtraction. + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", iB); + let x = &Operand::new("x", iB); + let y = &Operand::new("y", iB); + + let c_in = &Operand::new("c_in", b1).with_doc("Input carry flag"); + let c_out = &Operand::new("c_out", b1).with_doc("Output carry flag"); + let b_in = &Operand::new("b_in", b1).with_doc("Input borrow flag"); + let b_out = &Operand::new("b_out", b1).with_doc("Output borrow flag"); + + let c_if_in = &Operand::new("c_in", iflags); + let c_if_out = &Operand::new("c_out", iflags); + let b_if_in = &Operand::new("b_in", iflags); + let b_if_out = &Operand::new("b_out", iflags); + + ig.push( + Inst::new( + "iadd_cin", + r#" + Add integers with carry in. + + Same as `iadd` with an additional carry input. Computes: + + ```text + a = x + y + c_{in} \pmod 2^B + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.ternary, + ) + .operands_in(vec![x, y, c_in]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "iadd_ifcin", + r#" + Add integers with carry in. + + Same as `iadd` with an additional carry flag input. Computes: + + ```text + a = x + y + c_{in} \pmod 2^B + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.ternary, + ) + .operands_in(vec![x, y, c_if_in]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "iadd_cout", + r#" + Add integers with carry out. + + Same as `iadd` with an additional carry output. + + ```text + a &= x + y \pmod 2^B \\ + c_{out} &= x+y >= 2^B + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a, c_out]), + ); + + ig.push( + Inst::new( + "iadd_ifcout", + r#" + Add integers with carry out. + + Same as `iadd` with an additional carry flag output. + + ```text + a &= x + y \pmod 2^B \\ + c_{out} &= x+y >= 2^B + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a, c_if_out]), + ); + + ig.push( + Inst::new( + "iadd_carry", + r#" + Add integers with carry in and out. + + Same as `iadd` with an additional carry input and output. + + ```text + a &= x + y + c_{in} \pmod 2^B \\ + c_{out} &= x + y + c_{in} >= 2^B + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.ternary, + ) + .operands_in(vec![x, y, c_in]) + .operands_out(vec![a, c_out]), + ); + + ig.push( + Inst::new( + "iadd_ifcarry", + r#" + Add integers with carry in and out. + + Same as `iadd` with an additional carry flag input and output. + + ```text + a &= x + y + c_{in} \pmod 2^B \\ + c_{out} &= x + y + c_{in} >= 2^B + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.ternary, + ) + .operands_in(vec![x, y, c_if_in]) + .operands_out(vec![a, c_if_out]), + ); + + ig.push( + Inst::new( + "isub_bin", + r#" + Subtract integers with borrow in. + + Same as `isub` with an additional borrow flag input. Computes: + + ```text + a = x - (y + b_{in}) \pmod 2^B + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.ternary, + ) + .operands_in(vec![x, y, b_in]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "isub_ifbin", + r#" + Subtract integers with borrow in. + + Same as `isub` with an additional borrow flag input. Computes: + + ```text + a = x - (y + b_{in}) \pmod 2^B + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.ternary, + ) + .operands_in(vec![x, y, b_if_in]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "isub_bout", + r#" + Subtract integers with borrow out. + + Same as `isub` with an additional borrow flag output. + + ```text + a &= x - y \pmod 2^B \\ + b_{out} &= x < y + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a, b_out]), + ); + + ig.push( + Inst::new( + "isub_ifbout", + r#" + Subtract integers with borrow out. + + Same as `isub` with an additional borrow flag output. + + ```text + a &= x - y \pmod 2^B \\ + b_{out} &= x < y + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a, b_if_out]), + ); + + ig.push( + Inst::new( + "isub_borrow", + r#" + Subtract integers with borrow in and out. + + Same as `isub` with an additional borrow flag input and output. + + ```text + a &= x - (y + b_{in}) \pmod 2^B \\ + b_{out} &= x < y + b_{in} + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.ternary, + ) + .operands_in(vec![x, y, b_in]) + .operands_out(vec![a, b_out]), + ); + + ig.push( + Inst::new( + "isub_ifborrow", + r#" + Subtract integers with borrow in and out. + + Same as `isub` with an additional borrow flag input and output. + + ```text + a &= x - (y + b_{in}) \pmod 2^B \\ + b_{out} &= x < y + b_{in} + ``` + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.ternary, + ) + .operands_in(vec![x, y, b_if_in]) + .operands_out(vec![a, b_if_out]), + ); + + let bits = &TypeVar::new( + "bits", + "Any integer, float, or boolean scalar or vector type", + TypeSetBuilder::new() + .ints(Interval::All) + .floats(Interval::All) + .bools(Interval::All) + .simd_lanes(Interval::All) + .includes_scalars(true) + .build(), + ); + let x = &Operand::new("x", bits); + let y = &Operand::new("y", bits); + let a = &Operand::new("a", bits); + + ig.push( + Inst::new( + "band", + r#" + Bitwise and. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "bor", + r#" + Bitwise or. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "bxor", + r#" + Bitwise xor. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "bnot", + r#" + Bitwise not. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "band_not", + r#" + Bitwise and not. + + Computes `x & ~y`. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "bor_not", + r#" + Bitwise or not. + + Computes `x | ~y`. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "bxor_not", + r#" + Bitwise xor not. + + Computes `x ^ ~y`. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + let x = &Operand::new("x", iB); + let Y = &Operand::new("Y", &imm.imm64); + let a = &Operand::new("a", iB); + + ig.push( + Inst::new( + "band_imm", + r#" + Bitwise and with immediate. + + Same as `band`, but one operand is an immediate constant. + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "bor_imm", + r#" + Bitwise or with immediate. + + Same as `bor`, but one operand is an immediate constant. + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "bxor_imm", + r#" + Bitwise xor with immediate. + + Same as `bxor`, but one operand is an immediate constant. + + Polymorphic over all scalar integer types, but does not support vector + types. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + let x = &Operand::new("x", Int).with_doc("Scalar or vector value to shift"); + let y = &Operand::new("y", iB).with_doc("Number of bits to shift"); + let Y = &Operand::new("Y", &imm.imm64); + let a = &Operand::new("a", Int); + + ig.push( + Inst::new( + "rotl", + r#" + Rotate left. + + Rotate the bits in ``x`` by ``y`` places. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "rotr", + r#" + Rotate right. + + Rotate the bits in ``x`` by ``y`` places. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "rotl_imm", + r#" + Rotate left by immediate. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "rotr_imm", + r#" + Rotate right by immediate. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "ishl", + r#" + Integer shift left. Shift the bits in ``x`` towards the MSB by ``y`` + places. Shift in zero bits to the LSB. + + The shift amount is masked to the size of ``x``. + + When shifting a B-bits integer type, this instruction computes: + + ```text + s &:= y \pmod B, + a &:= x \cdot 2^s \pmod{2^B}. + ``` + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "ushr", + r#" + Unsigned shift right. Shift bits in ``x`` towards the LSB by ``y`` + places, shifting in zero bits to the MSB. Also called a *logical + shift*. + + The shift amount is masked to the size of the register. + + When shifting a B-bits integer type, this instruction computes: + + ```text + s &:= y \pmod B, + a &:= \lfloor x \cdot 2^{-s} \rfloor. + ``` + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "sshr", + r#" + Signed shift right. Shift bits in ``x`` towards the LSB by ``y`` + places, shifting in sign bits to the MSB. Also called an *arithmetic + shift*. + + The shift amount is masked to the size of the register. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "ishl_imm", + r#" + Integer shift left by immediate. + + The shift amount is masked to the size of ``x``. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "ushr_imm", + r#" + Unsigned shift right by immediate. + + The shift amount is masked to the size of the register. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "sshr_imm", + r#" + Signed shift right by immediate. + + The shift amount is masked to the size of the register. + "#, + &formats.binary_imm64, + ) + .operands_in(vec![x, Y]) + .operands_out(vec![a]), + ); + + let x = &Operand::new("x", iB); + let a = &Operand::new("a", iB); + + ig.push( + Inst::new( + "bitrev", + r#" + Reverse the bits of a integer. + + Reverses the bits in ``x``. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "clz", + r#" + Count leading zero bits. + + Starting from the MSB in ``x``, count the number of zero bits before + reaching the first one bit. When ``x`` is zero, returns the size of x + in bits. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "cls", + r#" + Count leading sign bits. + + Starting from the MSB after the sign bit in ``x``, count the number of + consecutive bits identical to the sign bit. When ``x`` is 0 or -1, + returns one less than the size of x in bits. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "ctz", + r#" + Count trailing zeros. + + Starting from the LSB in ``x``, count the number of zero bits before + reaching the first one bit. When ``x`` is zero, returns the size of x + in bits. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "popcnt", + r#" + Population count + + Count the number of one bits in ``x``. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let Float = &TypeVar::new( + "Float", + "A scalar or vector floating point number", + TypeSetBuilder::new() + .floats(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + let Cond = &Operand::new("Cond", &imm.floatcc); + let x = &Operand::new("x", Float); + let y = &Operand::new("y", Float); + let a = &Operand::new("a", &Float.as_bool()); + + ig.push( + Inst::new( + "fcmp", + r#" + Floating point comparison. + + Two IEEE 754-2008 floating point numbers, `x` and `y`, relate to each + other in exactly one of four ways: + + == ========================================== + UN Unordered when one or both numbers is NaN. + EQ When `x = y`. (And `0.0 = -0.0`). + LT When `x < y`. + GT When `x > y`. + == ========================================== + + The 14 `floatcc` condition codes each correspond to a subset of + the four relations, except for the empty set which would always be + false, and the full set which would always be true. + + The condition codes are divided into 7 'ordered' conditions which don't + include UN, and 7 unordered conditions which all include UN. + + +-------+------------+---------+------------+-------------------------+ + |Ordered |Unordered |Condition | + +=======+============+=========+============+=========================+ + |ord |EQ | LT | GT|uno |UN |NaNs absent / present. | + +-------+------------+---------+------------+-------------------------+ + |eq |EQ |ueq |UN | EQ |Equal | + +-------+------------+---------+------------+-------------------------+ + |one |LT | GT |ne |UN | LT | GT|Not equal | + +-------+------------+---------+------------+-------------------------+ + |lt |LT |ult |UN | LT |Less than | + +-------+------------+---------+------------+-------------------------+ + |le |LT | EQ |ule |UN | LT | EQ|Less than or equal | + +-------+------------+---------+------------+-------------------------+ + |gt |GT |ugt |UN | GT |Greater than | + +-------+------------+---------+------------+-------------------------+ + |ge |GT | EQ |uge |UN | GT | EQ|Greater than or equal | + +-------+------------+---------+------------+-------------------------+ + + The standard C comparison operators, `<, <=, >, >=`, are all ordered, + so they are false if either operand is NaN. The C equality operator, + `==`, is ordered, and since inequality is defined as the logical + inverse it is *unordered*. They map to the `floatcc` condition + codes as follows: + + ==== ====== ============ + C `Cond` Subset + ==== ====== ============ + `==` eq EQ + `!=` ne UN | LT | GT + `<` lt LT + `<=` le LT | EQ + `>` gt GT + `>=` ge GT | EQ + ==== ====== ============ + + This subset of condition codes also corresponds to the WebAssembly + floating point comparisons of the same name. + + When this instruction compares floating point vectors, it returns a + boolean vector with the results of lane-wise comparisons. + "#, + &formats.float_compare, + ) + .operands_in(vec![Cond, x, y]) + .operands_out(vec![a]), + ); + + let f = &Operand::new("f", fflags); + + ig.push( + Inst::new( + "ffcmp", + r#" + Floating point comparison returning flags. + + Compares two numbers like `fcmp`, but returns floating point CPU + flags instead of testing a specific condition. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![f]), + ); + + let x = &Operand::new("x", Float); + let y = &Operand::new("y", Float); + let z = &Operand::new("z", Float); + let a = &Operand::new("a", Float).with_doc("Result of applying operator to each lane"); + + ig.push( + Inst::new( + "fadd", + r#" + Floating point addition. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "fsub", + r#" + Floating point subtraction. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "fmul", + r#" + Floating point multiplication. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "fdiv", + r#" + Floating point division. + + Unlike the integer division instructions ` and + `udiv`, this can't trap. Division by zero is infinity or + NaN, depending on the dividend. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "sqrt", + r#" + Floating point square root. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "fma", + r#" + Floating point fused multiply-and-add. + + Computes `a := xy+z` without any intermediate rounding of the + product. + "#, + &formats.ternary, + ) + .operands_in(vec![x, y, z]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", Float).with_doc("``x`` with its sign bit inverted"); + + ig.push( + Inst::new( + "fneg", + r#" + Floating point negation. + + Note that this is a pure bitwise operation. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", Float).with_doc("``x`` with its sign bit cleared"); + + ig.push( + Inst::new( + "fabs", + r#" + Floating point absolute value. + + Note that this is a pure bitwise operation. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", Float).with_doc("``x`` with its sign bit changed to that of ``y``"); + + ig.push( + Inst::new( + "fcopysign", + r#" + Floating point copy sign. + + Note that this is a pure bitwise operation. The sign bit from ``y`` is + copied to the sign bit of ``x``. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", Float).with_doc("The smaller of ``x`` and ``y``"); + + ig.push( + Inst::new( + "fmin", + r#" + Floating point minimum, propagating NaNs. + + If either operand is NaN, this returns a NaN. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "fmin_pseudo", + r#" + Floating point pseudo-minimum, propagating NaNs. This behaves differently from ``fmin``. + See https://github.com/WebAssembly/simd/pull/122 for background. + + The behaviour is defined as ``fmin_pseudo(a, b) = (b < a) ? b : a``, and the behaviour + for zero or NaN inputs follows from the behaviour of ``<`` with such inputs. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", Float).with_doc("The larger of ``x`` and ``y``"); + + ig.push( + Inst::new( + "fmax", + r#" + Floating point maximum, propagating NaNs. + + If either operand is NaN, this returns a NaN. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "fmax_pseudo", + r#" + Floating point pseudo-maximum, propagating NaNs. This behaves differently from ``fmax``. + See https://github.com/WebAssembly/simd/pull/122 for background. + + The behaviour is defined as ``fmax_pseudo(a, b) = (a < b) ? b : a``, and the behaviour + for zero or NaN inputs follows from the behaviour of ``<`` with such inputs. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", Float).with_doc("``x`` rounded to integral value"); + + ig.push( + Inst::new( + "ceil", + r#" + Round floating point round to integral, towards positive infinity. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "floor", + r#" + Round floating point round to integral, towards negative infinity. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "trunc", + r#" + Round floating point round to integral, towards zero. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "nearest", + r#" + Round floating point round to integral, towards nearest with ties to + even. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", b1); + let x = &Operand::new("x", Ref); + + ig.push( + Inst::new( + "is_null", + r#" + Reference verification. + + The condition code determines if the reference type in question is + null or not. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", b1); + let x = &Operand::new("x", Ref); + + ig.push( + Inst::new( + "is_invalid", + r#" + Reference verification. + + The condition code determines if the reference type in question is + invalid or not. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let Cond = &Operand::new("Cond", &imm.intcc); + let f = &Operand::new("f", iflags); + let a = &Operand::new("a", b1); + + ig.push( + Inst::new( + "trueif", + r#" + Test integer CPU flags for a specific condition. + + Check the CPU flags in ``f`` against the ``Cond`` condition code and + return true when the condition code is satisfied. + "#, + &formats.int_cond, + ) + .operands_in(vec![Cond, f]) + .operands_out(vec![a]), + ); + + let Cond = &Operand::new("Cond", &imm.floatcc); + let f = &Operand::new("f", fflags); + + ig.push( + Inst::new( + "trueff", + r#" + Test floating point CPU flags for a specific condition. + + Check the CPU flags in ``f`` against the ``Cond`` condition code and + return true when the condition code is satisfied. + "#, + &formats.float_cond, + ) + .operands_in(vec![Cond, f]) + .operands_out(vec![a]), + ); + + let x = &Operand::new("x", Mem); + let a = &Operand::new("a", MemTo).with_doc("Bits of `x` reinterpreted"); + + ig.push( + Inst::new( + "bitcast", + r#" + Reinterpret the bits in `x` as a different type. + + The input and output types must be storable to memory and of the same + size. A bitcast is equivalent to storing one type and loading the other + type from the same address. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let x = &Operand::new("x", Any); + let a = &Operand::new("a", AnyTo).with_doc("Bits of `x` reinterpreted"); + + ig.push( + Inst::new( + "raw_bitcast", + r#" + Cast the bits in `x` as a different type of the same bit width. + + This instruction does not change the data's representation but allows + data in registers to be used as different types, e.g. an i32x4 as a + b8x16. The only constraint on the result `a` is that it can be + `raw_bitcast` back to the original type. Also, in a raw_bitcast between + vector types with the same number of lanes, the value of each result + lane is a raw_bitcast of the corresponding operand lane. TODO there is + currently no mechanism for enforcing the bit width constraint. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let a = &Operand::new("a", TxN).with_doc("A vector value"); + let s = &Operand::new("s", &TxN.lane_of()).with_doc("A scalar value"); + + ig.push( + Inst::new( + "scalar_to_vector", + r#" + Copies a scalar value to a vector value. The scalar is copied into the + least significant lane of the vector, and all other lanes will be zero. + "#, + &formats.unary, + ) + .operands_in(vec![s]) + .operands_out(vec![a]), + ); + + let Bool = &TypeVar::new( + "Bool", + "A scalar or vector boolean type", + TypeSetBuilder::new() + .bools(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + + let BoolTo = &TypeVar::new( + "BoolTo", + "A smaller boolean type with the same number of lanes", + TypeSetBuilder::new() + .bools(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + + let x = &Operand::new("x", Bool); + let a = &Operand::new("a", BoolTo); + + ig.push( + Inst::new( + "breduce", + r#" + Convert `x` to a smaller boolean type in the platform-defined way. + + The result type must have the same number of vector lanes as the input, + and each lane must not have more bits that the input lanes. If the + input and output types are the same, this is a no-op. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .constraints(vec![WiderOrEq(Bool.clone(), BoolTo.clone())]), + ); + + let BoolTo = &TypeVar::new( + "BoolTo", + "A larger boolean type with the same number of lanes", + TypeSetBuilder::new() + .bools(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + let x = &Operand::new("x", Bool); + let a = &Operand::new("a", BoolTo); + + ig.push( + Inst::new( + "bextend", + r#" + Convert `x` to a larger boolean type in the platform-defined way. + + The result type must have the same number of vector lanes as the input, + and each lane must not have fewer bits that the input lanes. If the + input and output types are the same, this is a no-op. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .constraints(vec![WiderOrEq(BoolTo.clone(), Bool.clone())]), + ); + + let IntTo = &TypeVar::new( + "IntTo", + "An integer type with the same number of lanes", + TypeSetBuilder::new() + .ints(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + let x = &Operand::new("x", Bool); + let a = &Operand::new("a", IntTo); + + ig.push( + Inst::new( + "bint", + r#" + Convert `x` to an integer. + + True maps to 1 and false maps to 0. The result type must have the same + number of vector lanes as the input. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "bmask", + r#" + Convert `x` to an integer mask. + + True maps to all 1s and false maps to all 0s. The result type must have + the same number of vector lanes as the input. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let Int = &TypeVar::new( + "Int", + "A scalar or vector integer type", + TypeSetBuilder::new() + .ints(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + + let IntTo = &TypeVar::new( + "IntTo", + "A smaller integer type with the same number of lanes", + TypeSetBuilder::new() + .ints(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + let x = &Operand::new("x", Int); + let a = &Operand::new("a", IntTo); + + ig.push( + Inst::new( + "ireduce", + r#" + Convert `x` to a smaller integer type by dropping high bits. + + Each lane in `x` is converted to a smaller integer type by discarding + the most significant bits. This is the same as reducing modulo + `2^n`. + + The result type must have the same number of vector lanes as the input, + and each lane must not have more bits that the input lanes. If the + input and output types are the same, this is a no-op. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .constraints(vec![WiderOrEq(Int.clone(), IntTo.clone())]), + ); + + let I16or32xN = &TypeVar::new( + "I16or32xN", + "A SIMD vector type containing integer lanes 16 or 32 bits wide", + TypeSetBuilder::new() + .ints(16..32) + .simd_lanes(4..8) + .includes_scalars(false) + .build(), + ); + + let x = &Operand::new("x", I16or32xN); + let y = &Operand::new("y", I16or32xN); + let a = &Operand::new("a", &I16or32xN.split_lanes()); + + ig.push( + Inst::new( + "snarrow", + r#" + Combine `x` and `y` into a vector with twice the lanes but half the integer width while + saturating overflowing values to the signed maximum and minimum. + + The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4` + and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value + returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "unarrow", + r#" + Combine `x` and `y` into a vector with twice the lanes but half the integer width while + saturating overflowing values to the unsigned maximum and minimum. + + Note that all input lanes are considered signed: any negative lanes will overflow and be + replaced with the unsigned minimum, `0x00`. + + The lanes will be concatenated after narrowing. For example, when `x` and `y` are `i32x4` + and `x = [x3, x2, x1, x0]` and `y = [y3, y2, y1, y0]`, then after narrowing the value + returned is an `i16x8`: `a = [y3', y2', y1', y0', x3', x2', x1', x0']`. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + let I8or16xN = &TypeVar::new( + "I8or16xN", + "A SIMD vector type containing integer lanes 8 or 16 bits wide.", + TypeSetBuilder::new() + .ints(8..16) + .simd_lanes(8..16) + .includes_scalars(false) + .build(), + ); + + let x = &Operand::new("x", I8or16xN); + let a = &Operand::new("a", &I8or16xN.merge_lanes()); + + ig.push( + Inst::new( + "swiden_low", + r#" + Widen the low lanes of `x` using signed extension. + + This will double the lane width and halve the number of lanes. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "swiden_high", + r#" + Widen the high lanes of `x` using signed extension. + + This will double the lane width and halve the number of lanes. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "uwiden_low", + r#" + Widen the low lanes of `x` using unsigned extension. + + This will double the lane width and halve the number of lanes. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "uwiden_high", + r#" + Widen the high lanes of `x` using unsigned extension. + + This will double the lane width and halve the number of lanes. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let I16x8 = &TypeVar::new( + "I16x8", + "A SIMD vector type containing 8 integer lanes each 16 bits wide.", + TypeSetBuilder::new() + .ints(16..16) + .simd_lanes(8..8) + .includes_scalars(false) + .build(), + ); + + let x = &Operand::new("x", I16x8); + let y = &Operand::new("y", I16x8); + let a = &Operand::new("a", &I16x8.merge_lanes()); + + ig.push( + Inst::new( + "widening_pairwise_dot_product_s", + r#" + Takes corresponding elements in `x` and `y`, performs a sign-extending length-doubling + multiplication on them, then adds adjacent pairs of elements to form the result. For + example, if the input vectors are `[x3, x2, x1, x0]` and `[y3, y2, y1, y0]`, it produces + the vector `[r1, r0]`, where `r1 = sx(x3) * sx(y3) + sx(x2) * sx(y2)` and + `r0 = sx(x1) * sx(y1) + sx(x0) * sx(y0)`, and `sx(n)` sign-extends `n` to twice its width. + + This will double the lane width and halve the number of lanes. So the resulting + vector has the same number of bits as `x` and `y` do (individually). + + See https://github.com/WebAssembly/simd/pull/127 for background info. + "#, + &formats.binary, + ) + .operands_in(vec![x, y]) + .operands_out(vec![a]), + ); + + let IntTo = &TypeVar::new( + "IntTo", + "A larger integer type with the same number of lanes", + TypeSetBuilder::new() + .ints(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + let x = &Operand::new("x", Int); + let a = &Operand::new("a", IntTo); + + ig.push( + Inst::new( + "uextend", + r#" + Convert `x` to a larger integer type by zero-extending. + + Each lane in `x` is converted to a larger integer type by adding + zeroes. The result has the same numerical value as `x` when both are + interpreted as unsigned integers. + + The result type must have the same number of vector lanes as the input, + and each lane must not have fewer bits that the input lanes. If the + input and output types are the same, this is a no-op. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .constraints(vec![WiderOrEq(IntTo.clone(), Int.clone())]), + ); + + ig.push( + Inst::new( + "sextend", + r#" + Convert `x` to a larger integer type by sign-extending. + + Each lane in `x` is converted to a larger integer type by replicating + the sign bit. The result has the same numerical value as `x` when both + are interpreted as signed integers. + + The result type must have the same number of vector lanes as the input, + and each lane must not have fewer bits that the input lanes. If the + input and output types are the same, this is a no-op. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .constraints(vec![WiderOrEq(IntTo.clone(), Int.clone())]), + ); + + let FloatTo = &TypeVar::new( + "FloatTo", + "A scalar or vector floating point number", + TypeSetBuilder::new() + .floats(Interval::All) + .simd_lanes(Interval::All) + .build(), + ); + let x = &Operand::new("x", Float); + let a = &Operand::new("a", FloatTo); + + ig.push( + Inst::new( + "fpromote", + r#" + Convert `x` to a larger floating point format. + + Each lane in `x` is converted to the destination floating point format. + This is an exact operation. + + Cranelift currently only supports two floating point formats + - `f32` and `f64`. This may change in the future. + + The result type must have the same number of vector lanes as the input, + and the result lanes must not have fewer bits than the input lanes. If + the input and output types are the same, this is a no-op. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .constraints(vec![WiderOrEq(FloatTo.clone(), Float.clone())]), + ); + + ig.push( + Inst::new( + "fdemote", + r#" + Convert `x` to a smaller floating point format. + + Each lane in `x` is converted to the destination floating point format + by rounding to nearest, ties to even. + + Cranelift currently only supports two floating point formats + - `f32` and `f64`. This may change in the future. + + The result type must have the same number of vector lanes as the input, + and the result lanes must not have more bits than the input lanes. If + the input and output types are the same, this is a no-op. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .constraints(vec![WiderOrEq(Float.clone(), FloatTo.clone())]), + ); + + let x = &Operand::new("x", Float); + let a = &Operand::new("a", IntTo); + + ig.push( + Inst::new( + "fcvt_to_uint", + r#" + Convert floating point to unsigned integer. + + Each lane in `x` is converted to an unsigned integer by rounding + towards zero. If `x` is NaN or if the unsigned integral value cannot be + represented in the result type, this instruction traps. + + The result type must have the same number of vector lanes as the input. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .can_trap(true), + ); + + ig.push( + Inst::new( + "fcvt_to_uint_sat", + r#" + Convert floating point to unsigned integer as fcvt_to_uint does, but + saturates the input instead of trapping. NaN and negative values are + converted to 0. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "fcvt_to_sint", + r#" + Convert floating point to signed integer. + + Each lane in `x` is converted to a signed integer by rounding towards + zero. If `x` is NaN or if the signed integral value cannot be + represented in the result type, this instruction traps. + + The result type must have the same number of vector lanes as the input. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]) + .can_trap(true), + ); + + ig.push( + Inst::new( + "fcvt_to_sint_sat", + r#" + Convert floating point to signed integer as fcvt_to_sint does, but + saturates the input instead of trapping. NaN values are converted to 0. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let x = &Operand::new("x", Int); + let a = &Operand::new("a", FloatTo); + + ig.push( + Inst::new( + "fcvt_from_uint", + r#" + Convert unsigned integer to floating point. + + Each lane in `x` is interpreted as an unsigned integer and converted to + floating point using round to nearest, ties to even. + + The result type must have the same number of vector lanes as the input. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + ig.push( + Inst::new( + "fcvt_from_sint", + r#" + Convert signed integer to floating point. + + Each lane in `x` is interpreted as a signed integer and converted to + floating point using round to nearest, ties to even. + + The result type must have the same number of vector lanes as the input. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![a]), + ); + + let WideInt = &TypeVar::new( + "WideInt", + "An integer type with lanes from `i16` upwards", + TypeSetBuilder::new() + .ints(16..128) + .simd_lanes(Interval::All) + .build(), + ); + let x = &Operand::new("x", WideInt); + let lo = &Operand::new("lo", &WideInt.half_width()).with_doc("The low bits of `x`"); + let hi = &Operand::new("hi", &WideInt.half_width()).with_doc("The high bits of `x`"); + + ig.push( + Inst::new( + "isplit", + r#" + Split an integer into low and high parts. + + Vectors of integers are split lane-wise, so the results have the same + number of lanes as the input, but the lanes are half the size. + + Returns the low half of `x` and the high half of `x` as two independent + values. + "#, + &formats.unary, + ) + .operands_in(vec![x]) + .operands_out(vec![lo, hi]) + .is_ghost(true), + ); + + let NarrowInt = &TypeVar::new( + "NarrowInt", + "An integer type with lanes type to `i64`", + TypeSetBuilder::new() + .ints(8..64) + .simd_lanes(Interval::All) + .build(), + ); + + let lo = &Operand::new("lo", NarrowInt); + let hi = &Operand::new("hi", NarrowInt); + let a = &Operand::new("a", &NarrowInt.double_width()) + .with_doc("The concatenation of `lo` and `hi`"); + + ig.push( + Inst::new( + "iconcat", + r#" + Concatenate low and high bits to form a larger integer type. + + Vectors of integers are concatenated lane-wise such that the result has + the same number of lanes as the inputs, but the lanes are twice the + size. + "#, + &formats.binary, + ) + .operands_in(vec![lo, hi]) + .operands_out(vec![a]) + .is_ghost(true), + ); + + // Instructions relating to atomic memory accesses and fences + let AtomicMem = &TypeVar::new( + "AtomicMem", + "Any type that can be stored in memory, which can be used in an atomic operation", + TypeSetBuilder::new().ints(8..64).build(), + ); + let x = &Operand::new("x", AtomicMem).with_doc("Value to be atomically stored"); + let a = &Operand::new("a", AtomicMem).with_doc("Value atomically loaded"); + let e = &Operand::new("e", AtomicMem).with_doc("Expected value in CAS"); + let p = &Operand::new("p", iAddr); + let MemFlags = &Operand::new("MemFlags", &imm.memflags); + let AtomicRmwOp = &Operand::new("AtomicRmwOp", &imm.atomic_rmw_op); + + ig.push( + Inst::new( + "atomic_rmw", + r#" + Atomically read-modify-write memory at `p`, with second operand `x`. The old value is + returned. `p` has the type of the target word size, and `x` may be an integer type of + 8, 16, 32 or 64 bits, even on a 32-bit target. The type of the returned value is the + same as the type of `x`. This operation is sequentially consistent and creates + happens-before edges that order normal (non-atomic) loads and stores. + "#, + &formats.atomic_rmw, + ) + .operands_in(vec![MemFlags, AtomicRmwOp, p, x]) + .operands_out(vec![a]) + .can_load(true) + .can_store(true) + .other_side_effects(true), + ); + + ig.push( + Inst::new( + "atomic_cas", + r#" + Perform an atomic compare-and-swap operation on memory at `p`, with expected value `e`, + storing `x` if the value at `p` equals `e`. The old value at `p` is returned, + regardless of whether the operation succeeds or fails. `p` has the type of the target + word size, and `x` and `e` must have the same type and the same size, which may be an + integer type of 8, 16, 32 or 64 bits, even on a 32-bit target. The type of the returned + value is the same as the type of `x` and `e`. This operation is sequentially + consistent and creates happens-before edges that order normal (non-atomic) loads and + stores. + "#, + &formats.atomic_cas, + ) + .operands_in(vec![MemFlags, p, e, x]) + .operands_out(vec![a]) + .can_load(true) + .can_store(true) + .other_side_effects(true), + ); + + ig.push( + Inst::new( + "atomic_load", + r#" + Atomically load from memory at `p`. + + This is a polymorphic instruction that can load any value type which has a memory + representation. It should only be used for integer types with 8, 16, 32 or 64 bits. + This operation is sequentially consistent and creates happens-before edges that order + normal (non-atomic) loads and stores. + "#, + &formats.load_no_offset, + ) + .operands_in(vec![MemFlags, p]) + .operands_out(vec![a]) + .can_load(true) + .other_side_effects(true), + ); + + ig.push( + Inst::new( + "atomic_store", + r#" + Atomically store `x` to memory at `p`. + + This is a polymorphic instruction that can store any value type with a memory + representation. It should only be used for integer types with 8, 16, 32 or 64 bits. + This operation is sequentially consistent and creates happens-before edges that order + normal (non-atomic) loads and stores. + "#, + &formats.store_no_offset, + ) + .operands_in(vec![MemFlags, x, p]) + .can_store(true) + .other_side_effects(true), + ); + + ig.push( + Inst::new( + "fence", + r#" + A memory fence. This must provide ordering to ensure that, at a minimum, neither loads + nor stores of any kind may move forwards or backwards across the fence. This operation + is sequentially consistent. + "#, + &formats.nullary, + ) + .other_side_effects(true), + ); + + let Offset = &Operand::new("Offset", &imm.offset32).with_doc("Byte offset from base address"); + let a = &Operand::new("a", TxN); + + ig.push( + Inst::new( + "load_splat", + r#" + Load an element from memory at ``p + Offset`` and return a vector + whose lanes are all set to that element. + + This is equivalent to ``load`` followed by ``splat``. + "#, + &formats.load, + ) + .operands_in(vec![MemFlags, p, Offset]) + .operands_out(vec![a]) + .can_load(true), + ); + + ig.build() +} diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/legalize.rs b/third_party/rust/cranelift-codegen-meta/src/shared/legalize.rs new file mode 100644 index 0000000000..9a0d6cffde --- /dev/null +++ b/third_party/rust/cranelift-codegen-meta/src/shared/legalize.rs @@ -0,0 +1,1087 @@ +use crate::cdsl::ast::{var, ExprBuilder, Literal}; +use crate::cdsl::instructions::{Bindable, Instruction, InstructionGroup}; +use crate::cdsl::xform::{TransformGroupBuilder, TransformGroups}; + +use crate::shared::immediates::Immediates; +use crate::shared::types::Float::{F32, F64}; +use crate::shared::types::Int::{I128, I16, I32, I64, I8}; +use cranelift_codegen_shared::condcodes::{CondCode, IntCC}; + +#[allow(clippy::many_single_char_names, clippy::cognitive_complexity)] +pub(crate) fn define(insts: &InstructionGroup, imm: &Immediates) -> TransformGroups { + let mut narrow = TransformGroupBuilder::new( + "narrow", + r#" + Legalize instructions by narrowing. + + The transformations in the 'narrow' group work by expressing + instructions in terms of smaller types. Operations on vector types are + expressed in terms of vector types with fewer lanes, and integer + operations are expressed in terms of smaller integer types. + "#, + ); + + let mut widen = TransformGroupBuilder::new( + "widen", + r#" + Legalize instructions by widening. + + The transformations in the 'widen' group work by expressing + instructions in terms of larger types. + "#, + ); + + let mut expand = TransformGroupBuilder::new( + "expand", + r#" + Legalize instructions by expansion. + + Rewrite instructions in terms of other instructions, generally + operating on the same types as the original instructions. + "#, + ); + + // List of instructions. + let band = insts.by_name("band"); + let band_imm = insts.by_name("band_imm"); + let band_not = insts.by_name("band_not"); + let bint = insts.by_name("bint"); + let bitrev = insts.by_name("bitrev"); + let bnot = insts.by_name("bnot"); + let bor = insts.by_name("bor"); + let bor_imm = insts.by_name("bor_imm"); + let bor_not = insts.by_name("bor_not"); + let brnz = insts.by_name("brnz"); + let brz = insts.by_name("brz"); + let br_icmp = insts.by_name("br_icmp"); + let br_table = insts.by_name("br_table"); + let bxor = insts.by_name("bxor"); + let bxor_imm = insts.by_name("bxor_imm"); + let bxor_not = insts.by_name("bxor_not"); + let cls = insts.by_name("cls"); + let clz = insts.by_name("clz"); + let ctz = insts.by_name("ctz"); + let copy = insts.by_name("copy"); + let fabs = insts.by_name("fabs"); + let f32const = insts.by_name("f32const"); + let f64const = insts.by_name("f64const"); + let fcopysign = insts.by_name("fcopysign"); + let fcvt_from_sint = insts.by_name("fcvt_from_sint"); + let fneg = insts.by_name("fneg"); + let iadd = insts.by_name("iadd"); + let iadd_cin = insts.by_name("iadd_cin"); + let iadd_cout = insts.by_name("iadd_cout"); + let iadd_carry = insts.by_name("iadd_carry"); + let iadd_ifcin = insts.by_name("iadd_ifcin"); + let iadd_ifcout = insts.by_name("iadd_ifcout"); + let iadd_imm = insts.by_name("iadd_imm"); + let icmp = insts.by_name("icmp"); + let icmp_imm = insts.by_name("icmp_imm"); + let iconcat = insts.by_name("iconcat"); + let iconst = insts.by_name("iconst"); + let ifcmp = insts.by_name("ifcmp"); + let ifcmp_imm = insts.by_name("ifcmp_imm"); + let imul = insts.by_name("imul"); + let imul_imm = insts.by_name("imul_imm"); + let ireduce = insts.by_name("ireduce"); + let irsub_imm = insts.by_name("irsub_imm"); + let ishl = insts.by_name("ishl"); + let ishl_imm = insts.by_name("ishl_imm"); + let isplit = insts.by_name("isplit"); + let istore8 = insts.by_name("istore8"); + let istore16 = insts.by_name("istore16"); + let isub = insts.by_name("isub"); + let isub_bin = insts.by_name("isub_bin"); + let isub_bout = insts.by_name("isub_bout"); + let isub_borrow = insts.by_name("isub_borrow"); + let isub_ifbin = insts.by_name("isub_ifbin"); + let isub_ifbout = insts.by_name("isub_ifbout"); + let jump = insts.by_name("jump"); + let load = insts.by_name("load"); + let popcnt = insts.by_name("popcnt"); + let resumable_trapnz = insts.by_name("resumable_trapnz"); + let rotl = insts.by_name("rotl"); + let rotl_imm = insts.by_name("rotl_imm"); + let rotr = insts.by_name("rotr"); + let rotr_imm = insts.by_name("rotr_imm"); + let sdiv = insts.by_name("sdiv"); + let sdiv_imm = insts.by_name("sdiv_imm"); + let select = insts.by_name("select"); + let sextend = insts.by_name("sextend"); + let sshr = insts.by_name("sshr"); + let sshr_imm = insts.by_name("sshr_imm"); + let srem = insts.by_name("srem"); + let srem_imm = insts.by_name("srem_imm"); + let store = insts.by_name("store"); + let udiv = insts.by_name("udiv"); + let udiv_imm = insts.by_name("udiv_imm"); + let uextend = insts.by_name("uextend"); + let uload8 = insts.by_name("uload8"); + let uload16 = insts.by_name("uload16"); + let umulhi = insts.by_name("umulhi"); + let ushr = insts.by_name("ushr"); + let ushr_imm = insts.by_name("ushr_imm"); + let urem = insts.by_name("urem"); + let urem_imm = insts.by_name("urem_imm"); + let trapif = insts.by_name("trapif"); + let trapnz = insts.by_name("trapnz"); + let trapz = insts.by_name("trapz"); + + // Custom expansions for memory objects. + expand.custom_legalize(insts.by_name("global_value"), "expand_global_value"); + expand.custom_legalize(insts.by_name("heap_addr"), "expand_heap_addr"); + expand.custom_legalize(insts.by_name("table_addr"), "expand_table_addr"); + + // Custom expansions for calls. + expand.custom_legalize(insts.by_name("call"), "expand_call"); + + // Custom expansions that need to change the CFG. + // TODO: Add sufficient XForm syntax that we don't need to hand-code these. + expand.custom_legalize(trapz, "expand_cond_trap"); + expand.custom_legalize(trapnz, "expand_cond_trap"); + expand.custom_legalize(resumable_trapnz, "expand_cond_trap"); + expand.custom_legalize(br_table, "expand_br_table"); + expand.custom_legalize(select, "expand_select"); + widen.custom_legalize(select, "expand_select"); // small ints + + // Custom expansions for floating point constants. + // These expansions require bit-casting or creating constant pool entries. + expand.custom_legalize(f32const, "expand_fconst"); + expand.custom_legalize(f64const, "expand_fconst"); + + // Custom expansions for stack memory accesses. + expand.custom_legalize(insts.by_name("stack_load"), "expand_stack_load"); + expand.custom_legalize(insts.by_name("stack_store"), "expand_stack_store"); + + // Custom expansions for small stack memory acccess. + widen.custom_legalize(insts.by_name("stack_load"), "expand_stack_load"); + widen.custom_legalize(insts.by_name("stack_store"), "expand_stack_store"); + + // List of variables to reuse in patterns. + let x = var("x"); + let y = var("y"); + let z = var("z"); + let a = var("a"); + let a1 = var("a1"); + let a2 = var("a2"); + let a3 = var("a3"); + let a4 = var("a4"); + let b = var("b"); + let b1 = var("b1"); + let b2 = var("b2"); + let b3 = var("b3"); + let b4 = var("b4"); + let b_in = var("b_in"); + let b_int = var("b_int"); + let c = var("c"); + let c1 = var("c1"); + let c2 = var("c2"); + let c3 = var("c3"); + let c4 = var("c4"); + let c_in = var("c_in"); + let c_int = var("c_int"); + let d = var("d"); + let d1 = var("d1"); + let d2 = var("d2"); + let d3 = var("d3"); + let d4 = var("d4"); + let e = var("e"); + let e1 = var("e1"); + let e2 = var("e2"); + let e3 = var("e3"); + let e4 = var("e4"); + let f = var("f"); + let f1 = var("f1"); + let f2 = var("f2"); + let xl = var("xl"); + let xh = var("xh"); + let yl = var("yl"); + let yh = var("yh"); + let al = var("al"); + let ah = var("ah"); + let cc = var("cc"); + let block = var("block"); + let ptr = var("ptr"); + let flags = var("flags"); + let offset = var("off"); + let vararg = var("vararg"); + + narrow.custom_legalize(load, "narrow_load"); + narrow.custom_legalize(store, "narrow_store"); + + // iconst.i64 can't be legalized in the meta langage (because integer literals can't be + // embedded as part of arguments), so use a custom legalization for now. + narrow.custom_legalize(iconst, "narrow_iconst"); + + for &(ty, ty_half) in &[(I128, I64), (I64, I32)] { + let inst = uextend.bind(ty).bind(ty_half); + narrow.legalize( + def!(a = inst(x)), + vec![ + def!(ah = iconst(Literal::constant(&imm.imm64, 0))), + def!(a = iconcat(x, ah)), + ], + ); + } + + for &(ty, ty_half, shift) in &[(I128, I64, 63), (I64, I32, 31)] { + let inst = sextend.bind(ty).bind(ty_half); + narrow.legalize( + def!(a = inst(x)), + vec![ + def!(ah = sshr_imm(x, Literal::constant(&imm.imm64, shift))), // splat sign bit to whole number + def!(a = iconcat(x, ah)), + ], + ); + } + + for &bin_op in &[band, bor, bxor, band_not, bor_not, bxor_not] { + narrow.legalize( + def!(a = bin_op(x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + def!(al = bin_op(xl, yl)), + def!(ah = bin_op(xh, yh)), + def!(a = iconcat(al, ah)), + ], + ); + } + + narrow.legalize( + def!(a = bnot(x)), + vec![ + def!((xl, xh) = isplit(x)), + def!(al = bnot(xl)), + def!(ah = bnot(xh)), + def!(a = iconcat(al, ah)), + ], + ); + + narrow.legalize( + def!(a = select(c, x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + def!(al = select(c, xl, yl)), + def!(ah = select(c, xh, yh)), + def!(a = iconcat(al, ah)), + ], + ); + + for &ty in &[I128, I64] { + let block = var("block"); + let block1 = var("block1"); + let block2 = var("block2"); + + narrow.legalize( + def!(brz.ty(x, block, vararg)), + vec![ + def!((xl, xh) = isplit(x)), + def!( + a = icmp_imm( + Literal::enumerator_for(&imm.intcc, "eq"), + xl, + Literal::constant(&imm.imm64, 0) + ) + ), + def!( + b = icmp_imm( + Literal::enumerator_for(&imm.intcc, "eq"), + xh, + Literal::constant(&imm.imm64, 0) + ) + ), + def!(c = band(a, b)), + def!(brnz(c, block, vararg)), + ], + ); + + narrow.legalize( + def!(brnz.ty(x, block1, vararg)), + vec![ + def!((xl, xh) = isplit(x)), + def!(brnz(xl, block1, vararg)), + def!(jump(block2, Literal::empty_vararg())), + block!(block2), + def!(brnz(xh, block1, vararg)), + ], + ); + } + + narrow.legalize( + def!(a = popcnt.I128(x)), + vec![ + def!((xl, xh) = isplit(x)), + def!(e1 = popcnt(xl)), + def!(e2 = popcnt(xh)), + def!(e3 = iadd(e1, e2)), + def!(a = uextend(e3)), + ], + ); + + // TODO(ryzokuken): benchmark this and decide if branching is a faster + // approach than evaluating boolean expressions. + + narrow.custom_legalize(icmp_imm, "narrow_icmp_imm"); + + let intcc_eq = Literal::enumerator_for(&imm.intcc, "eq"); + let intcc_ne = Literal::enumerator_for(&imm.intcc, "ne"); + for &(int_ty, int_ty_half) in &[(I64, I32), (I128, I64)] { + narrow.legalize( + def!(b = icmp.int_ty(intcc_eq, x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + def!(b1 = icmp.int_ty_half(intcc_eq, xl, yl)), + def!(b2 = icmp.int_ty_half(intcc_eq, xh, yh)), + def!(b = band(b1, b2)), + ], + ); + + narrow.legalize( + def!(b = icmp.int_ty(intcc_ne, x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + def!(b1 = icmp.int_ty_half(intcc_ne, xl, yl)), + def!(b2 = icmp.int_ty_half(intcc_ne, xh, yh)), + def!(b = bor(b1, b2)), + ], + ); + + use IntCC::*; + for cc in &[ + SignedGreaterThan, + SignedGreaterThanOrEqual, + SignedLessThan, + SignedLessThanOrEqual, + UnsignedGreaterThan, + UnsignedGreaterThanOrEqual, + UnsignedLessThan, + UnsignedLessThanOrEqual, + ] { + let intcc_cc = Literal::enumerator_for(&imm.intcc, cc.to_static_str()); + let cc1 = Literal::enumerator_for(&imm.intcc, cc.without_equal().to_static_str()); + let cc2 = + Literal::enumerator_for(&imm.intcc, cc.inverse().without_equal().to_static_str()); + let cc3 = Literal::enumerator_for(&imm.intcc, cc.unsigned().to_static_str()); + narrow.legalize( + def!(b = icmp.int_ty(intcc_cc, x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + // X = cc1 || (!cc2 && cc3) + def!(b1 = icmp.int_ty_half(cc1, xh, yh)), + def!(b2 = icmp.int_ty_half(cc2, xh, yh)), + def!(b3 = icmp.int_ty_half(cc3, xl, yl)), + def!(c1 = bnot(b2)), + def!(c2 = band(c1, b3)), + def!(b = bor(b1, c2)), + ], + ); + } + } + + // TODO(ryzokuken): explore the perf diff w/ x86_umulx and consider have a + // separate legalization for x86. + for &ty in &[I64, I128] { + narrow.legalize( + def!(a = imul.ty(x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + def!(a1 = imul(xh, yl)), + def!(a2 = imul(xl, yh)), + def!(a3 = iadd(a1, a2)), + def!(a4 = umulhi(xl, yl)), + def!(ah = iadd(a3, a4)), + def!(al = imul(xl, yl)), + def!(a = iconcat(al, ah)), + ], + ); + } + + let zero = Literal::constant(&imm.imm64, 0); + narrow.legalize( + def!(a = iadd_imm.I128(x, c)), + vec![ + def!(yh = iconst.I64(zero)), + def!(yl = iconst.I64(c)), + def!(y = iconcat.I64(yh, yl)), + def!(a = iadd(x, y)), + ], + ); + + // Widen instructions with one input operand. + for &op in &[bnot, popcnt] { + for &int_ty in &[I8, I16] { + widen.legalize( + def!(a = op.int_ty(b)), + vec![ + def!(x = uextend.I32(b)), + def!(z = op.I32(x)), + def!(a = ireduce.int_ty(z)), + ], + ); + } + } + + // Widen instructions with two input operands. + let mut widen_two_arg = |signed: bool, op: &Instruction| { + for &int_ty in &[I8, I16] { + let sign_ext_op = if signed { sextend } else { uextend }; + widen.legalize( + def!(a = op.int_ty(b, c)), + vec![ + def!(x = sign_ext_op.I32(b)), + def!(y = sign_ext_op.I32(c)), + def!(z = op.I32(x, y)), + def!(a = ireduce.int_ty(z)), + ], + ); + } + }; + + for bin_op in &[ + iadd, isub, imul, udiv, urem, band, bor, bxor, band_not, bor_not, bxor_not, + ] { + widen_two_arg(false, bin_op); + } + for bin_op in &[sdiv, srem] { + widen_two_arg(true, bin_op); + } + + // Widen instructions using immediate operands. + let mut widen_imm = |signed: bool, op: &Instruction| { + for &int_ty in &[I8, I16] { + let sign_ext_op = if signed { sextend } else { uextend }; + widen.legalize( + def!(a = op.int_ty(b, c)), + vec![ + def!(x = sign_ext_op.I32(b)), + def!(z = op.I32(x, c)), + def!(a = ireduce.int_ty(z)), + ], + ); + } + }; + + for bin_op in &[ + iadd_imm, imul_imm, udiv_imm, urem_imm, band_imm, bor_imm, bxor_imm, irsub_imm, + ] { + widen_imm(false, bin_op); + } + for bin_op in &[sdiv_imm, srem_imm] { + widen_imm(true, bin_op); + } + + for &(int_ty, num) in &[(I8, 24), (I16, 16)] { + let imm = Literal::constant(&imm.imm64, -num); + + widen.legalize( + def!(a = clz.int_ty(b)), + vec![ + def!(c = uextend.I32(b)), + def!(d = clz.I32(c)), + def!(e = iadd_imm(d, imm)), + def!(a = ireduce.int_ty(e)), + ], + ); + + widen.legalize( + def!(a = cls.int_ty(b)), + vec![ + def!(c = sextend.I32(b)), + def!(d = cls.I32(c)), + def!(e = iadd_imm(d, imm)), + def!(a = ireduce.int_ty(e)), + ], + ); + } + + for &(int_ty, num) in &[(I8, 1 << 8), (I16, 1 << 16)] { + let num = Literal::constant(&imm.imm64, num); + widen.legalize( + def!(a = ctz.int_ty(b)), + vec![ + def!(c = uextend.I32(b)), + // When `b` is zero, returns the size of x in bits. + def!(d = bor_imm(c, num)), + def!(e = ctz.I32(d)), + def!(a = ireduce.int_ty(e)), + ], + ); + } + + // iconst + for &int_ty in &[I8, I16] { + widen.legalize( + def!(a = iconst.int_ty(b)), + vec![def!(c = iconst.I32(b)), def!(a = ireduce.int_ty(c))], + ); + } + + for &extend_op in &[uextend, sextend] { + // The sign extension operators have two typevars: the result has one and controls the + // instruction, then the input has one. + let bound = extend_op.bind(I16).bind(I8); + widen.legalize( + def!(a = bound(b)), + vec![def!(c = extend_op.I32(b)), def!(a = ireduce(c))], + ); + } + + widen.legalize( + def!(store.I8(flags, a, ptr, offset)), + vec![ + def!(b = uextend.I32(a)), + def!(istore8(flags, b, ptr, offset)), + ], + ); + + widen.legalize( + def!(store.I16(flags, a, ptr, offset)), + vec![ + def!(b = uextend.I32(a)), + def!(istore16(flags, b, ptr, offset)), + ], + ); + + widen.legalize( + def!(a = load.I8(flags, ptr, offset)), + vec![ + def!(b = uload8.I32(flags, ptr, offset)), + def!(a = ireduce(b)), + ], + ); + + widen.legalize( + def!(a = load.I16(flags, ptr, offset)), + vec![ + def!(b = uload16.I32(flags, ptr, offset)), + def!(a = ireduce(b)), + ], + ); + + for &int_ty in &[I8, I16] { + widen.legalize( + def!(br_table.int_ty(x, y, z)), + vec![def!(b = uextend.I32(x)), def!(br_table(b, y, z))], + ); + } + + for &int_ty in &[I8, I16] { + widen.legalize( + def!(a = bint.int_ty(b)), + vec![def!(x = bint.I32(b)), def!(a = ireduce.int_ty(x))], + ); + } + + for &int_ty in &[I8, I16] { + for &op in &[ishl, ishl_imm, ushr, ushr_imm] { + widen.legalize( + def!(a = op.int_ty(b, c)), + vec![ + def!(x = uextend.I32(b)), + def!(z = op.I32(x, c)), + def!(a = ireduce.int_ty(z)), + ], + ); + } + + for &op in &[sshr, sshr_imm] { + widen.legalize( + def!(a = op.int_ty(b, c)), + vec![ + def!(x = sextend.I32(b)), + def!(z = op.I32(x, c)), + def!(a = ireduce.int_ty(z)), + ], + ); + } + + for cc in &["eq", "ne", "ugt", "ult", "uge", "ule"] { + let w_cc = Literal::enumerator_for(&imm.intcc, cc); + widen.legalize( + def!(a = icmp_imm.int_ty(w_cc, b, c)), + vec![def!(x = uextend.I32(b)), def!(a = icmp_imm(w_cc, x, c))], + ); + widen.legalize( + def!(a = icmp.int_ty(w_cc, b, c)), + vec![ + def!(x = uextend.I32(b)), + def!(y = uextend.I32(c)), + def!(a = icmp.I32(w_cc, x, y)), + ], + ); + } + + for cc in &["sgt", "slt", "sge", "sle"] { + let w_cc = Literal::enumerator_for(&imm.intcc, cc); + widen.legalize( + def!(a = icmp_imm.int_ty(w_cc, b, c)), + vec![def!(x = sextend.I32(b)), def!(a = icmp_imm(w_cc, x, c))], + ); + + widen.legalize( + def!(a = icmp.int_ty(w_cc, b, c)), + vec![ + def!(x = sextend.I32(b)), + def!(y = sextend.I32(c)), + def!(a = icmp(w_cc, x, y)), + ], + ); + } + } + + for &ty in &[I8, I16] { + widen.legalize( + def!(brz.ty(x, block, vararg)), + vec![def!(a = uextend.I32(x)), def!(brz(a, block, vararg))], + ); + + widen.legalize( + def!(brnz.ty(x, block, vararg)), + vec![def!(a = uextend.I32(x)), def!(brnz(a, block, vararg))], + ); + } + + for &(ty_half, ty) in &[(I64, I128), (I32, I64)] { + let inst = ireduce.bind(ty_half).bind(ty); + expand.legalize( + def!(a = inst(x)), + vec![def!((b, c) = isplit(x)), def!(a = copy(b))], + ); + } + + // Expand integer operations with carry for RISC architectures that don't have + // the flags. + let intcc_ult = Literal::enumerator_for(&imm.intcc, "ult"); + expand.legalize( + def!((a, c) = iadd_cout(x, y)), + vec![def!(a = iadd(x, y)), def!(c = icmp(intcc_ult, a, x))], + ); + + let intcc_ugt = Literal::enumerator_for(&imm.intcc, "ugt"); + expand.legalize( + def!((a, b) = isub_bout(x, y)), + vec![def!(a = isub(x, y)), def!(b = icmp(intcc_ugt, a, x))], + ); + + expand.legalize( + def!(a = iadd_cin(x, y, c)), + vec![ + def!(a1 = iadd(x, y)), + def!(c_int = bint(c)), + def!(a = iadd(a1, c_int)), + ], + ); + + expand.legalize( + def!(a = isub_bin(x, y, b)), + vec![ + def!(a1 = isub(x, y)), + def!(b_int = bint(b)), + def!(a = isub(a1, b_int)), + ], + ); + + expand.legalize( + def!((a, c) = iadd_carry(x, y, c_in)), + vec![ + def!((a1, c1) = iadd_cout(x, y)), + def!(c_int = bint(c_in)), + def!((a, c2) = iadd_cout(a1, c_int)), + def!(c = bor(c1, c2)), + ], + ); + + expand.legalize( + def!((a, b) = isub_borrow(x, y, b_in)), + vec![ + def!((a1, b1) = isub_bout(x, y)), + def!(b_int = bint(b_in)), + def!((a, b2) = isub_bout(a1, b_int)), + def!(b = bor(b1, b2)), + ], + ); + + // Expansion for fcvt_from_sint for smaller integer types. + // This uses expand and not widen because the controlling type variable for + // this instruction is f32/f64, which is legalized as part of the expand + // group. + for &dest_ty in &[F32, F64] { + for &src_ty in &[I8, I16] { + let bound_inst = fcvt_from_sint.bind(dest_ty).bind(src_ty); + expand.legalize( + def!(a = bound_inst(b)), + vec![ + def!(x = sextend.I32(b)), + def!(a = fcvt_from_sint.dest_ty(x)), + ], + ); + } + } + + // Expansions for immediate operands that are out of range. + for &(inst_imm, inst) in &[ + (iadd_imm, iadd), + (imul_imm, imul), + (sdiv_imm, sdiv), + (udiv_imm, udiv), + (srem_imm, srem), + (urem_imm, urem), + (band_imm, band), + (bor_imm, bor), + (bxor_imm, bxor), + (ifcmp_imm, ifcmp), + ] { + expand.legalize( + def!(a = inst_imm(x, y)), + vec![def!(a1 = iconst(y)), def!(a = inst(x, a1))], + ); + } + + expand.legalize( + def!(a = irsub_imm(y, x)), + vec![def!(a1 = iconst(x)), def!(a = isub(a1, y))], + ); + + // Rotates and shifts. + for &(inst_imm, inst) in &[ + (rotl_imm, rotl), + (rotr_imm, rotr), + (ishl_imm, ishl), + (sshr_imm, sshr), + (ushr_imm, ushr), + ] { + expand.legalize( + def!(a = inst_imm(x, y)), + vec![def!(a1 = iconst.I32(y)), def!(a = inst(x, a1))], + ); + } + + expand.legalize( + def!(a = icmp_imm(cc, x, y)), + vec![def!(a1 = iconst(y)), def!(a = icmp(cc, x, a1))], + ); + + //# Expansions for *_not variants of bitwise ops. + for &(inst_not, inst) in &[(band_not, band), (bor_not, bor), (bxor_not, bxor)] { + expand.legalize( + def!(a = inst_not(x, y)), + vec![def!(a1 = bnot(y)), def!(a = inst(x, a1))], + ); + } + + //# Expand bnot using xor. + let minus_one = Literal::constant(&imm.imm64, -1); + expand.legalize( + def!(a = bnot(x)), + vec![def!(y = iconst(minus_one)), def!(a = bxor(x, y))], + ); + + //# Expand bitrev + //# Adapted from Stack Overflow. + //# https://stackoverflow.com/questions/746171/most-efficient-algorithm-for-bit-reversal-from-msb-lsb-to-lsb-msb-in-c + let imm64_1 = Literal::constant(&imm.imm64, 1); + let imm64_2 = Literal::constant(&imm.imm64, 2); + let imm64_4 = Literal::constant(&imm.imm64, 4); + + widen.legalize( + def!(a = bitrev.I8(x)), + vec![ + def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaa))), + def!(a2 = ushr_imm(a1, imm64_1)), + def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x55))), + def!(a4 = ishl_imm(a3, imm64_1)), + def!(b = bor(a2, a4)), + def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcc))), + def!(b2 = ushr_imm(b1, imm64_2)), + def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x33))), + def!(b4 = ishl_imm(b3, imm64_2)), + def!(c = bor(b2, b4)), + def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0))), + def!(c2 = ushr_imm(c1, imm64_4)), + def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f))), + def!(c4 = ishl_imm(c3, imm64_4)), + def!(a = bor(c2, c4)), + ], + ); + + let imm64_8 = Literal::constant(&imm.imm64, 8); + + widen.legalize( + def!(a = bitrev.I16(x)), + vec![ + def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaaaa))), + def!(a2 = ushr_imm(a1, imm64_1)), + def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x5555))), + def!(a4 = ishl_imm(a3, imm64_1)), + def!(b = bor(a2, a4)), + def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcccc))), + def!(b2 = ushr_imm(b1, imm64_2)), + def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x3333))), + def!(b4 = ishl_imm(b3, imm64_2)), + def!(c = bor(b2, b4)), + def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0f0))), + def!(c2 = ushr_imm(c1, imm64_4)), + def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f0f))), + def!(c4 = ishl_imm(c3, imm64_4)), + def!(d = bor(c2, c4)), + def!(d1 = band_imm(d, Literal::constant(&imm.imm64, 0xff00))), + def!(d2 = ushr_imm(d1, imm64_8)), + def!(d3 = band_imm(d, Literal::constant(&imm.imm64, 0x00ff))), + def!(d4 = ishl_imm(d3, imm64_8)), + def!(a = bor(d2, d4)), + ], + ); + + let imm64_16 = Literal::constant(&imm.imm64, 16); + + expand.legalize( + def!(a = bitrev.I32(x)), + vec![ + def!(a1 = band_imm(x, Literal::constant(&imm.imm64, 0xaaaa_aaaa))), + def!(a2 = ushr_imm(a1, imm64_1)), + def!(a3 = band_imm(x, Literal::constant(&imm.imm64, 0x5555_5555))), + def!(a4 = ishl_imm(a3, imm64_1)), + def!(b = bor(a2, a4)), + def!(b1 = band_imm(b, Literal::constant(&imm.imm64, 0xcccc_cccc))), + def!(b2 = ushr_imm(b1, imm64_2)), + def!(b3 = band_imm(b, Literal::constant(&imm.imm64, 0x3333_3333))), + def!(b4 = ishl_imm(b3, imm64_2)), + def!(c = bor(b2, b4)), + def!(c1 = band_imm(c, Literal::constant(&imm.imm64, 0xf0f0_f0f0))), + def!(c2 = ushr_imm(c1, imm64_4)), + def!(c3 = band_imm(c, Literal::constant(&imm.imm64, 0x0f0f_0f0f))), + def!(c4 = ishl_imm(c3, imm64_4)), + def!(d = bor(c2, c4)), + def!(d1 = band_imm(d, Literal::constant(&imm.imm64, 0xff00_ff00))), + def!(d2 = ushr_imm(d1, imm64_8)), + def!(d3 = band_imm(d, Literal::constant(&imm.imm64, 0x00ff_00ff))), + def!(d4 = ishl_imm(d3, imm64_8)), + def!(e = bor(d2, d4)), + def!(e1 = ushr_imm(e, imm64_16)), + def!(e2 = ishl_imm(e, imm64_16)), + def!(a = bor(e1, e2)), + ], + ); + + #[allow(overflowing_literals)] + let imm64_0xaaaaaaaaaaaaaaaa = Literal::constant(&imm.imm64, 0xaaaa_aaaa_aaaa_aaaa); + let imm64_0x5555555555555555 = Literal::constant(&imm.imm64, 0x5555_5555_5555_5555); + #[allow(overflowing_literals)] + let imm64_0xcccccccccccccccc = Literal::constant(&imm.imm64, 0xcccc_cccc_cccc_cccc); + let imm64_0x3333333333333333 = Literal::constant(&imm.imm64, 0x3333_3333_3333_3333); + #[allow(overflowing_literals)] + let imm64_0xf0f0f0f0f0f0f0f0 = Literal::constant(&imm.imm64, 0xf0f0_f0f0_f0f0_f0f0); + let imm64_0x0f0f0f0f0f0f0f0f = Literal::constant(&imm.imm64, 0x0f0f_0f0f_0f0f_0f0f); + #[allow(overflowing_literals)] + let imm64_0xff00ff00ff00ff00 = Literal::constant(&imm.imm64, 0xff00_ff00_ff00_ff00); + let imm64_0x00ff00ff00ff00ff = Literal::constant(&imm.imm64, 0x00ff_00ff_00ff_00ff); + #[allow(overflowing_literals)] + let imm64_0xffff0000ffff0000 = Literal::constant(&imm.imm64, 0xffff_0000_ffff_0000); + let imm64_0x0000ffff0000ffff = Literal::constant(&imm.imm64, 0x0000_ffff_0000_ffff); + let imm64_32 = Literal::constant(&imm.imm64, 32); + + expand.legalize( + def!(a = bitrev.I64(x)), + vec![ + def!(a1 = band_imm(x, imm64_0xaaaaaaaaaaaaaaaa)), + def!(a2 = ushr_imm(a1, imm64_1)), + def!(a3 = band_imm(x, imm64_0x5555555555555555)), + def!(a4 = ishl_imm(a3, imm64_1)), + def!(b = bor(a2, a4)), + def!(b1 = band_imm(b, imm64_0xcccccccccccccccc)), + def!(b2 = ushr_imm(b1, imm64_2)), + def!(b3 = band_imm(b, imm64_0x3333333333333333)), + def!(b4 = ishl_imm(b3, imm64_2)), + def!(c = bor(b2, b4)), + def!(c1 = band_imm(c, imm64_0xf0f0f0f0f0f0f0f0)), + def!(c2 = ushr_imm(c1, imm64_4)), + def!(c3 = band_imm(c, imm64_0x0f0f0f0f0f0f0f0f)), + def!(c4 = ishl_imm(c3, imm64_4)), + def!(d = bor(c2, c4)), + def!(d1 = band_imm(d, imm64_0xff00ff00ff00ff00)), + def!(d2 = ushr_imm(d1, imm64_8)), + def!(d3 = band_imm(d, imm64_0x00ff00ff00ff00ff)), + def!(d4 = ishl_imm(d3, imm64_8)), + def!(e = bor(d2, d4)), + def!(e1 = band_imm(e, imm64_0xffff0000ffff0000)), + def!(e2 = ushr_imm(e1, imm64_16)), + def!(e3 = band_imm(e, imm64_0x0000ffff0000ffff)), + def!(e4 = ishl_imm(e3, imm64_16)), + def!(f = bor(e2, e4)), + def!(f1 = ushr_imm(f, imm64_32)), + def!(f2 = ishl_imm(f, imm64_32)), + def!(a = bor(f1, f2)), + ], + ); + + narrow.legalize( + def!(a = bitrev.I128(x)), + vec![ + def!((xl, xh) = isplit(x)), + def!(yh = bitrev(xl)), + def!(yl = bitrev(xh)), + def!(a = iconcat(yl, yh)), + ], + ); + + // Floating-point sign manipulations. + for &(ty, const_inst, minus_zero) in &[ + (F32, f32const, &Literal::bits(&imm.ieee32, 0x8000_0000)), + ( + F64, + f64const, + &Literal::bits(&imm.ieee64, 0x8000_0000_0000_0000), + ), + ] { + expand.legalize( + def!(a = fabs.ty(x)), + vec![def!(b = const_inst(minus_zero)), def!(a = band_not(x, b))], + ); + + expand.legalize( + def!(a = fneg.ty(x)), + vec![def!(b = const_inst(minus_zero)), def!(a = bxor(x, b))], + ); + + expand.legalize( + def!(a = fcopysign.ty(x, y)), + vec![ + def!(b = const_inst(minus_zero)), + def!(a1 = band_not(x, b)), + def!(a2 = band(y, b)), + def!(a = bor(a1, a2)), + ], + ); + } + + expand.custom_legalize(br_icmp, "expand_br_icmp"); + + let mut groups = TransformGroups::new(); + + let narrow_id = narrow.build_and_add_to(&mut groups); + let expand_id = expand.build_and_add_to(&mut groups); + + // Expansions using CPU flags. + let mut expand_flags = TransformGroupBuilder::new( + "expand_flags", + r#" + Instruction expansions for architectures with flags. + + Expand some instructions using CPU flags, then fall back to the normal + expansions. Not all architectures support CPU flags, so these patterns + are kept separate. + "#, + ) + .chain_with(expand_id); + + let imm64_0 = Literal::constant(&imm.imm64, 0); + let intcc_ne = Literal::enumerator_for(&imm.intcc, "ne"); + let intcc_eq = Literal::enumerator_for(&imm.intcc, "eq"); + + expand_flags.legalize( + def!(trapnz(x, c)), + vec![ + def!(a = ifcmp_imm(x, imm64_0)), + def!(trapif(intcc_ne, a, c)), + ], + ); + + expand_flags.legalize( + def!(trapz(x, c)), + vec![ + def!(a = ifcmp_imm(x, imm64_0)), + def!(trapif(intcc_eq, a, c)), + ], + ); + + expand_flags.build_and_add_to(&mut groups); + + // Narrow legalizations using CPU flags. + let mut narrow_flags = TransformGroupBuilder::new( + "narrow_flags", + r#" + Narrow instructions for architectures with flags. + + Narrow some instructions using CPU flags, then fall back to the normal + legalizations. Not all architectures support CPU flags, so these + patterns are kept separate. + "#, + ) + .chain_with(narrow_id); + + narrow_flags.legalize( + def!(a = iadd(x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + def!((al, c) = iadd_ifcout(xl, yl)), + def!(ah = iadd_ifcin(xh, yh, c)), + def!(a = iconcat(al, ah)), + ], + ); + + narrow_flags.legalize( + def!(a = isub(x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + def!((al, b) = isub_ifbout(xl, yl)), + def!(ah = isub_ifbin(xh, yh, b)), + def!(a = iconcat(al, ah)), + ], + ); + + narrow_flags.build_and_add_to(&mut groups); + + // TODO(ryzokuken): figure out a way to legalize iadd_c* to iadd_ifc* (and + // similarly isub_b* to isub_ifb*) on expand_flags so that this isn't required. + // Narrow legalizations for ISAs that don't have CPU flags. + let mut narrow_no_flags = TransformGroupBuilder::new( + "narrow_no_flags", + r#" + Narrow instructions for architectures without flags. + + Narrow some instructions avoiding the use of CPU flags, then fall back + to the normal legalizations. Not all architectures support CPU flags, + so these patterns are kept separate. + "#, + ) + .chain_with(narrow_id); + + narrow_no_flags.legalize( + def!(a = iadd(x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + def!((al, c) = iadd_cout(xl, yl)), + def!(ah = iadd_cin(xh, yh, c)), + def!(a = iconcat(al, ah)), + ], + ); + + narrow_no_flags.legalize( + def!(a = isub(x, y)), + vec![ + def!((xl, xh) = isplit(x)), + def!((yl, yh) = isplit(y)), + def!((al, b) = isub_bout(xl, yl)), + def!(ah = isub_bin(xh, yh, b)), + def!(a = iconcat(al, ah)), + ], + ); + + narrow_no_flags.build_and_add_to(&mut groups); + + // TODO The order of declarations unfortunately matters to be compatible with the Python code. + // When it's all migrated, we can put this next to the narrow/expand build_and_add_to calls + // above. + widen.build_and_add_to(&mut groups); + + groups +} diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/mod.rs b/third_party/rust/cranelift-codegen-meta/src/shared/mod.rs new file mode 100644 index 0000000000..b185262ccd --- /dev/null +++ b/third_party/rust/cranelift-codegen-meta/src/shared/mod.rs @@ -0,0 +1,101 @@ +//! Shared definitions for the Cranelift intermediate language. + +pub mod entities; +pub mod formats; +pub mod immediates; +pub mod instructions; +pub mod legalize; +pub mod settings; +pub mod types; + +use crate::cdsl::formats::{FormatStructure, InstructionFormat}; +use crate::cdsl::instructions::{AllInstructions, InstructionGroup}; +use crate::cdsl::settings::SettingGroup; +use crate::cdsl::xform::TransformGroups; + +use crate::shared::entities::EntityRefs; +use crate::shared::formats::Formats; +use crate::shared::immediates::Immediates; + +use std::collections::HashMap; +use std::iter::FromIterator; +use std::rc::Rc; + +pub(crate) struct Definitions { + pub settings: SettingGroup, + pub all_instructions: AllInstructions, + pub instructions: InstructionGroup, + pub imm: Immediates, + pub formats: Formats, + pub transform_groups: TransformGroups, + pub entities: EntityRefs, +} + +pub(crate) fn define() -> Definitions { + let mut all_instructions = AllInstructions::new(); + + let immediates = Immediates::new(); + let entities = EntityRefs::new(); + let formats = Formats::new(&immediates, &entities); + let instructions = + instructions::define(&mut all_instructions, &formats, &immediates, &entities); + let transform_groups = legalize::define(&instructions, &immediates); + + Definitions { + settings: settings::define(), + all_instructions, + instructions, + imm: immediates, + formats, + transform_groups, + entities, + } +} + +impl Definitions { + /// Verifies certain properties of formats. + /// + /// - Formats must be uniquely named: if two formats have the same name, they must refer to the + /// same data. Otherwise, two format variants in the codegen crate would have the same name. + /// - Formats must be structurally different from each other. Otherwise, this would lead to + /// code duplicate in the codegen crate. + /// + /// Returns a list of all the instruction formats effectively used. + pub fn verify_instruction_formats(&self) -> Vec<&InstructionFormat> { + let mut format_names: HashMap<&'static str, &Rc<InstructionFormat>> = HashMap::new(); + + // A structure is: number of input value operands / whether there's varargs or not / names + // of immediate fields. + let mut format_structures: HashMap<FormatStructure, &InstructionFormat> = HashMap::new(); + + for inst in self.all_instructions.values() { + // Check name. + if let Some(existing_format) = format_names.get(&inst.format.name) { + assert!( + Rc::ptr_eq(&existing_format, &inst.format), + "formats must uniquely named; there's a\ + conflict on the name '{}', please make sure it is used only once.", + existing_format.name + ); + } else { + format_names.insert(inst.format.name, &inst.format); + } + + // Check structure. + let key = inst.format.structure(); + if let Some(existing_format) = format_structures.get(&key) { + assert_eq!( + existing_format.name, inst.format.name, + "duplicate instruction formats {} and {}; please remove one.", + existing_format.name, inst.format.name + ); + } else { + format_structures.insert(key, &inst.format); + } + } + + let mut result = Vec::from_iter(format_structures.into_iter().map(|(_, v)| v)); + result.sort_by_key(|format| format.name); + result + } +} diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/settings.rs b/third_party/rust/cranelift-codegen-meta/src/shared/settings.rs new file mode 100644 index 0000000000..1ddc445927 --- /dev/null +++ b/third_party/rust/cranelift-codegen-meta/src/shared/settings.rs @@ -0,0 +1,287 @@ +use crate::cdsl::settings::{SettingGroup, SettingGroupBuilder}; + +pub(crate) fn define() -> SettingGroup { + let mut settings = SettingGroupBuilder::new("shared"); + + settings.add_enum( + "regalloc", + r#"Register allocator to use with the MachInst backend. + + This selects the register allocator as an option among those offered by the `regalloc.rs` + crate. Please report register allocation bugs to the maintainers of this crate whenever + possible. + + Note: this only applies to target that use the MachInst backend. As of 2020-04-17, this + means the x86_64 backend doesn't use this yet. + + Possible values: + + - `backtracking` is a greedy, backtracking register allocator as implemented in + Spidermonkey's optimizing tier IonMonkey. It may take more time to allocate registers, but + it should generate better code in general, resulting in better throughput of generated + code. + - `backtracking_checked` is the backtracking allocator with additional self checks that may + take some time to run, and thus these checks are disabled by default. + - `experimental_linear_scan` is an experimental linear scan allocator. It may take less + time to allocate registers, but generated code's quality may be inferior. As of + 2020-04-17, it is still experimental and it should not be used in production settings. + - `experimental_linear_scan_checked` is the linear scan allocator with additional self + checks that may take some time to run, and thus these checks are disabled by default. + "#, + vec![ + "backtracking", + "backtracking_checked", + "experimental_linear_scan", + "experimental_linear_scan_checked", + ], + ); + + settings.add_enum( + "opt_level", + r#" + Optimization level: + + - none: Minimise compile time by disabling most optimizations. + - speed: Generate the fastest possible code + - speed_and_size: like "speed", but also perform transformations + aimed at reducing code size. + "#, + vec!["none", "speed", "speed_and_size"], + ); + + settings.add_bool( + "enable_verifier", + r#" + Run the Cranelift IR verifier at strategic times during compilation. + + This makes compilation slower but catches many bugs. The verifier is always enabled by + default, which is useful during development. + "#, + true, + ); + + // Note that Cranelift doesn't currently need an is_pie flag, because PIE is + // just PIC where symbols can't be pre-empted, which can be expressed with the + // `colocated` flag on external functions and global values. + settings.add_bool( + "is_pic", + "Enable Position-Independent Code generation", + false, + ); + + settings.add_bool( + "use_colocated_libcalls", + r#" + Use colocated libcalls. + + Generate code that assumes that libcalls can be declared "colocated", + meaning they will be defined along with the current function, such that + they can use more efficient addressing. + "#, + false, + ); + + settings.add_bool( + "avoid_div_traps", + r#" + Generate explicit checks around native division instructions to avoid + their trapping. + + This is primarily used by SpiderMonkey which doesn't install a signal + handler for SIGFPE, but expects a SIGILL trap for division by zero. + + On ISAs like ARM where the native division instructions don't trap, + this setting has no effect - explicit checks are always inserted. + "#, + false, + ); + + settings.add_bool( + "enable_float", + r#" + Enable the use of floating-point instructions + + Disabling use of floating-point instructions is not yet implemented. + "#, + true, + ); + + settings.add_bool( + "enable_nan_canonicalization", + r#" + Enable NaN canonicalization + + This replaces NaNs with a single canonical value, for users requiring + entirely deterministic WebAssembly computation. This is not required + by the WebAssembly spec, so it is not enabled by default. + "#, + false, + ); + + settings.add_bool( + "enable_pinned_reg", + r#"Enable the use of the pinned register. + + This register is excluded from register allocation, and is completely under the control of + the end-user. It is possible to read it via the get_pinned_reg instruction, and to set it + with the set_pinned_reg instruction. + "#, + false, + ); + + settings.add_bool( + "use_pinned_reg_as_heap_base", + r#"Use the pinned register as the heap base. + + Enabling this requires the enable_pinned_reg setting to be set to true. It enables a custom + legalization of the `heap_addr` instruction so it will use the pinned register as the heap + base, instead of fetching it from a global value. + + Warning! Enabling this means that the pinned register *must* be maintained to contain the + heap base address at all times, during the lifetime of a function. Using the pinned + register for other purposes when this is set is very likely to cause crashes. + "#, + false, + ); + + settings.add_bool("enable_simd", "Enable the use of SIMD instructions.", false); + + settings.add_bool( + "enable_atomics", + "Enable the use of atomic instructions", + true, + ); + + settings.add_bool( + "enable_safepoints", + r#" + Enable safepoint instruction insertions. + + This will allow the emit_stack_maps() function to insert the safepoint + instruction on top of calls and interrupt traps in order to display the + live reference values at that point in the program. + "#, + false, + ); + + settings.add_enum( + "tls_model", + r#" + Defines the model used to perform TLS accesses. + "#, + vec!["none", "elf_gd", "macho", "coff"], + ); + + // Settings specific to the `baldrdash` calling convention. + + settings.add_enum( + "libcall_call_conv", + r#" + Defines the calling convention to use for LibCalls call expansion, + since it may be different from the ISA default calling convention. + + The default value is to use the same calling convention as the ISA + default calling convention. + + This list should be kept in sync with the list of calling + conventions available in isa/call_conv.rs. + "#, + vec![ + "isa_default", + "fast", + "cold", + "system_v", + "windows_fastcall", + "baldrdash_system_v", + "baldrdash_windows", + "baldrdash_2020", + "probestack", + ], + ); + + settings.add_num( + "baldrdash_prologue_words", + r#" + Number of pointer-sized words pushed by the baldrdash prologue. + + Functions with the `baldrdash` calling convention don't generate their + own prologue and epilogue. They depend on externally generated code + that pushes a fixed number of words in the prologue and restores them + in the epilogue. + + This setting configures the number of pointer-sized words pushed on the + stack when the Cranelift-generated code is entered. This includes the + pushed return address on x86. + "#, + 0, + ); + + // BaldrMonkey requires that not-yet-relocated function addresses be encoded + // as all-ones bitpatterns. + settings.add_bool( + "emit_all_ones_funcaddrs", + "Emit not-yet-relocated function addresses as all-ones bit patterns.", + false, + ); + + // Stack probing options. + + settings.add_bool( + "enable_probestack", + r#" + Enable the use of stack probes, for calling conventions which support this + functionality. + "#, + true, + ); + + settings.add_bool( + "probestack_func_adjusts_sp", + r#" + Set this to true of the stack probe function modifies the stack pointer + itself. + "#, + false, + ); + + settings.add_num( + "probestack_size_log2", + r#" + The log2 of the size of the stack guard region. + + Stack frames larger than this size will have stack overflow checked + by calling the probestack function. + + The default is 12, which translates to a size of 4096. + "#, + 12, + ); + + // Jump table options. + + settings.add_bool( + "enable_jump_tables", + "Enable the use of jump tables in generated machine code.", + true, + ); + + // Spectre options. + + settings.add_bool( + "enable_heap_access_spectre_mitigation", + r#" + Enable Spectre mitigation on heap bounds checks. + + This is a no-op for any heap that needs no bounds checks; e.g., + if the limit is static and the guard region is large enough that + the index cannot reach past it. + + This option is enabled by default because it is highly + recommended for secure sandboxing. The embedder should consider + the security implications carefully before disabling this option. + "#, + true, + ); + + settings.build() +} diff --git a/third_party/rust/cranelift-codegen-meta/src/shared/types.rs b/third_party/rust/cranelift-codegen-meta/src/shared/types.rs new file mode 100644 index 0000000000..631e5433e9 --- /dev/null +++ b/third_party/rust/cranelift-codegen-meta/src/shared/types.rs @@ -0,0 +1,236 @@ +//! This module predefines all the Cranelift scalar types. + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub(crate) enum Bool { + /// 1-bit bool. + B1 = 1, + /// 8-bit bool. + B8 = 8, + /// 16-bit bool. + B16 = 16, + /// 32-bit bool. + B32 = 32, + /// 64-bit bool. + B64 = 64, + /// 128-bit bool. + B128 = 128, +} + +/// This provides an iterator through all of the supported bool variants. +pub(crate) struct BoolIterator { + index: u8, +} + +impl BoolIterator { + pub fn new() -> Self { + Self { index: 0 } + } +} + +impl Iterator for BoolIterator { + type Item = Bool; + fn next(&mut self) -> Option<Self::Item> { + let res = match self.index { + 0 => Some(Bool::B1), + 1 => Some(Bool::B8), + 2 => Some(Bool::B16), + 3 => Some(Bool::B32), + 4 => Some(Bool::B64), + 5 => Some(Bool::B128), + _ => return None, + }; + self.index += 1; + res + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub(crate) enum Int { + /// 8-bit int. + I8 = 8, + /// 16-bit int. + I16 = 16, + /// 32-bit int. + I32 = 32, + /// 64-bit int. + I64 = 64, + /// 128-bit int. + I128 = 128, +} + +/// This provides an iterator through all of the supported int variants. +pub(crate) struct IntIterator { + index: u8, +} + +impl IntIterator { + pub fn new() -> Self { + Self { index: 0 } + } +} + +impl Iterator for IntIterator { + type Item = Int; + fn next(&mut self) -> Option<Self::Item> { + let res = match self.index { + 0 => Some(Int::I8), + 1 => Some(Int::I16), + 2 => Some(Int::I32), + 3 => Some(Int::I64), + 4 => Some(Int::I128), + _ => return None, + }; + self.index += 1; + res + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub(crate) enum Float { + F32 = 32, + F64 = 64, +} + +/// Iterator through the variants of the Float enum. +pub(crate) struct FloatIterator { + index: u8, +} + +impl FloatIterator { + pub fn new() -> Self { + Self { index: 0 } + } +} + +/// This provides an iterator through all of the supported float variants. +impl Iterator for FloatIterator { + type Item = Float; + fn next(&mut self) -> Option<Self::Item> { + let res = match self.index { + 0 => Some(Float::F32), + 1 => Some(Float::F64), + _ => return None, + }; + self.index += 1; + res + } +} + +/// A type representing CPU flags. +/// +/// Flags can't be stored in memory. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub(crate) enum Flag { + /// CPU flags from an integer comparison. + IFlags, + /// CPU flags from a floating point comparison. + FFlags, +} + +/// Iterator through the variants of the Flag enum. +pub(crate) struct FlagIterator { + index: u8, +} + +impl FlagIterator { + pub fn new() -> Self { + Self { index: 0 } + } +} + +impl Iterator for FlagIterator { + type Item = Flag; + fn next(&mut self) -> Option<Self::Item> { + let res = match self.index { + 0 => Some(Flag::IFlags), + 1 => Some(Flag::FFlags), + _ => return None, + }; + self.index += 1; + res + } +} + +#[derive(Debug, Clone, Copy, Eq, PartialEq, Hash)] +pub(crate) enum Reference { + /// 32-bit reference. + R32 = 32, + /// 64-bit reference. + R64 = 64, +} + +/// This provides an iterator through all of the supported reference variants. +pub(crate) struct ReferenceIterator { + index: u8, +} + +impl ReferenceIterator { + pub fn new() -> Self { + Self { index: 0 } + } +} + +impl Iterator for ReferenceIterator { + type Item = Reference; + fn next(&mut self) -> Option<Self::Item> { + let res = match self.index { + 0 => Some(Reference::R32), + 1 => Some(Reference::R64), + _ => return None, + }; + self.index += 1; + res + } +} + +#[cfg(test)] +mod iter_tests { + use super::*; + + #[test] + fn bool_iter_works() { + let mut bool_iter = BoolIterator::new(); + assert_eq!(bool_iter.next(), Some(Bool::B1)); + assert_eq!(bool_iter.next(), Some(Bool::B8)); + assert_eq!(bool_iter.next(), Some(Bool::B16)); + assert_eq!(bool_iter.next(), Some(Bool::B32)); + assert_eq!(bool_iter.next(), Some(Bool::B64)); + assert_eq!(bool_iter.next(), Some(Bool::B128)); + assert_eq!(bool_iter.next(), None); + } + + #[test] + fn int_iter_works() { + let mut int_iter = IntIterator::new(); + assert_eq!(int_iter.next(), Some(Int::I8)); + assert_eq!(int_iter.next(), Some(Int::I16)); + assert_eq!(int_iter.next(), Some(Int::I32)); + assert_eq!(int_iter.next(), Some(Int::I64)); + assert_eq!(int_iter.next(), Some(Int::I128)); + assert_eq!(int_iter.next(), None); + } + + #[test] + fn float_iter_works() { + let mut float_iter = FloatIterator::new(); + assert_eq!(float_iter.next(), Some(Float::F32)); + assert_eq!(float_iter.next(), Some(Float::F64)); + assert_eq!(float_iter.next(), None); + } + + #[test] + fn flag_iter_works() { + let mut flag_iter = FlagIterator::new(); + assert_eq!(flag_iter.next(), Some(Flag::IFlags)); + assert_eq!(flag_iter.next(), Some(Flag::FFlags)); + assert_eq!(flag_iter.next(), None); + } + + #[test] + fn reference_iter_works() { + let mut reference_iter = ReferenceIterator::new(); + assert_eq!(reference_iter.next(), Some(Reference::R32)); + assert_eq!(reference_iter.next(), Some(Reference::R64)); + assert_eq!(reference_iter.next(), None); + } +} |