Adding upstream version 86.0.1.upstream/86.0.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1330 insertions, 0 deletions

diff --git a/third_party/rust/cranelift-codegen/src/peepmatic.rs b/third_party/rust/cranelift-codegen/src/peepmatic.rs
new file mode 100644
index 0000000000..bf0f440865
--- /dev/null
+++ b/third_party/rust/cranelift-codegen/src/peepmatic.rs
@@ -0,0 +1,1330 @@
+//! Glue for working with `peepmatic`-generated peephole optimizers.
+
+use crate::cursor::{Cursor, FuncCursor};
+use crate::ir::{
+    dfg::DataFlowGraph,
+    entities::{Inst, Value},
+    immediates::{Imm64, Uimm64},
+    instructions::{InstructionData, Opcode},
+    types, InstBuilder,
+};
+use crate::isa::TargetIsa;
+use cranelift_codegen_shared::condcodes::IntCC;
+use peepmatic_runtime::{
+    cc::ConditionCode,
+    instruction_set::InstructionSet,
+    part::{Constant, Part},
+    r#type::{BitWidth, Kind, Type},
+    PeepholeOptimizations, PeepholeOptimizer,
+};
+use std::borrow::Cow;
+use std::boxed::Box;
+use std::convert::{TryFrom, TryInto};
+use std::iter;
+use std::ptr;
+use std::sync::atomic::{AtomicPtr, Ordering};
+
+peepmatic_traits::define_parse_and_typing_rules_for_operator! {
+    Opcode {
+        adjust_sp_down => AdjustSpDown {
+            parameters(iNN);
+            result(void);
+        }
+        adjust_sp_down_imm => AdjustSpDownImm {
+            immediates(iNN);
+            result(void);
+        }
+        band => Band {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        band_imm => BandImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        bconst => Bconst {
+            immediates(b1);
+            result(bNN);
+        }
+        bint => Bint {
+            parameters(bNN);
+            result(iNN);
+        }
+        bor => Bor {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        bor_imm => BorImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        brnz => Brnz {
+            parameters(bool_or_int);
+            result(void);
+        }
+        brz => Brz {
+            parameters(bool_or_int);
+            result(void);
+        }
+        bxor => Bxor {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        bxor_imm => BxorImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        iadd => Iadd {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        iadd_imm => IaddImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        icmp => Icmp {
+            immediates(cc);
+            parameters(iNN, iNN);
+            result(b1);
+        }
+        icmp_imm => IcmpImm {
+            immediates(cc, iNN);
+            parameters(iNN);
+            result(b1);
+        }
+        iconst => Iconst {
+            immediates(iNN);
+            result(iNN);
+        }
+        ifcmp => Ifcmp {
+            parameters(iNN, iNN);
+            result(cpu_flags);
+        }
+        ifcmp_imm => IfcmpImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(cpu_flags);
+        }
+        imul => Imul {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        imul_imm => ImulImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        ireduce => Ireduce {
+            parameters(iNN);
+            result(iMM);
+            is_reduce(true);
+        }
+        irsub_imm => IrsubImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        ishl => Ishl {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        ishl_imm => IshlImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        isub => Isub {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        rotl => Rotl {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        rotl_imm => RotlImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        rotr => Rotr {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        rotr_imm => RotrImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        sdiv => Sdiv {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        sdiv_imm => SdivImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        select => Select {
+            parameters(bool_or_int, any_t, any_t);
+            result(any_t);
+        }
+        sextend => Sextend {
+            parameters(iNN);
+            result(iMM);
+            is_extend(true);
+        }
+        srem => Srem {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        srem_imm => SremImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        sshr => Sshr {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        sshr_imm => SshrImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        trapnz => Trapnz {
+            parameters(bool_or_int);
+            result(void);
+        }
+        trapz => Trapz {
+            parameters(bool_or_int);
+            result(void);
+        }
+        udiv => Udiv {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        udiv_imm => UdivImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        uextend => Uextend {
+            parameters(iNN);
+            result(iMM);
+            is_extend(true);
+        }
+        urem => Urem {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        urem_imm => UremImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+        ushr => Ushr {
+            parameters(iNN, iNN);
+            result(iNN);
+        }
+        ushr_imm => UshrImm {
+            immediates(iNN);
+            parameters(iNN);
+            result(iNN);
+        }
+    }
+    parse_cfg(feature = "rebuild-peephole-optimizers");
+}
+
+/// Code required to rebuild Peepmatic-based peephole optimizers.
+///
+/// This module is used to scope imports and dependencies that are only required
+/// for building peephole optimizers (as opposed to just using pre-built
+/// peephole optimizers). This helps ensure that our regular builds using
+/// pre-built peephole optimizers stay lean.
+#[cfg(feature = "rebuild-peephole-optimizers")]
+mod rebuild {
+    use super::*;
+    use alloc::vec::Vec;
+    use std::fs;
+    use std::path::Path;
+
+    /// Rebuild the `preopt.peepmatic` peephole optimizer.
+    ///
+    /// Saves and overwrites the old `preopt.serialized` build and returns a
+    /// copy of the result.
+    pub fn rebuild_preopt() -> Vec<u8> {
+        let codegen_path = Path::new(include_str!(concat!(
+            env!("OUT_DIR"),
+            "/CRANELIFT_CODEGEN_PATH"
+        )));
+        let source_path = codegen_path.join("src").join("preopt.peepmatic");
+
+        let preopt = peepmatic::compile_file::<Opcode>(&source_path)
+            .expect("failed to compile `src/preopt.peepmatic`");
+
+        let serialized_path = codegen_path.join("src").join("preopt.serialized");
+        preopt
+            .serialize_to_file(&serialized_path)
+            .expect("failed to serialize peephole optimizer to `src/preopt.serialized`");
+        fs::read(&serialized_path).expect("failed to read `src/preopt.serialized`")
+    }
+}
+
+/// Get the `preopt.peepmatic` peephole optimizer.
+pub(crate) fn preopt<'a, 'b>(
+    isa: &'b dyn TargetIsa,
+) -> PeepholeOptimizer<'static, 'a, &'b dyn TargetIsa> {
+    #[cfg(feature = "rebuild-peephole-optimizers")]
+    fn get_serialized() -> Cow<'static, [u8]> {
+        rebuild::rebuild_preopt().into()
+    }
+
+    #[cfg(not(feature = "rebuild-peephole-optimizers"))]
+    fn get_serialized() -> Cow<'static, [u8]> {
+        static SERIALIZED: &[u8] = include_bytes!("preopt.serialized");
+        SERIALIZED.into()
+    }
+
+    // Once initialized, this must never be re-assigned. The initialized value
+    // is semantically "static data" and is intentionally leaked for the whole
+    // program's lifetime.
+    static DESERIALIZED: AtomicPtr<PeepholeOptimizations<Opcode>> = AtomicPtr::new(ptr::null_mut());
+
+    // If `DESERIALIZED` has already been initialized, then just use it.
+    let ptr = DESERIALIZED.load(Ordering::SeqCst);
+    if let Some(peep_opts) = unsafe { ptr.as_ref() } {
+        return peep_opts.optimizer(isa);
+    }
+
+    // Otherwise, if `DESERIALIZED` hasn't been initialized, then we need to
+    // deserialize the peephole optimizations and initialize it. However,
+    // another thread could be doing the same thing concurrently, so there is a
+    // race to see who initializes `DESERIALIZED` first, and we need to be
+    // prepared to both win or lose that race.
+    let peep_opts = PeepholeOptimizations::deserialize(&get_serialized())
+        .expect("should always be able to deserialize `preopt.serialized`");
+    let peep_opts = Box::into_raw(Box::new(peep_opts));
+
+    // Only update `DESERIALIZE` if it is still null, attempting to perform the
+    // one-time transition from null -> non-null.
+    if DESERIALIZED
+        .compare_and_swap(ptr::null_mut(), peep_opts, Ordering::SeqCst)
+        .is_null()
+    {
+        // We won the race to initialize `DESERIALIZED`.
+        debug_assert_eq!(DESERIALIZED.load(Ordering::SeqCst), peep_opts);
+        let peep_opts = unsafe { &*peep_opts };
+        return peep_opts.optimizer(isa);
+    }
+
+    // We lost the race to initialize `DESERIALIZED`. Drop our no-longer-needed
+    // instance of `peep_opts` and get the pointer to the instance that won the
+    // race.
+    let _ = unsafe { Box::from_raw(peep_opts) };
+    let peep_opts = DESERIALIZED.load(Ordering::SeqCst);
+    let peep_opts = unsafe { peep_opts.as_ref().unwrap() };
+    peep_opts.optimizer(isa)
+}
+
+/// Either a `Value` or an `Inst`.
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub enum ValueOrInst {
+    Value(Value),
+    Inst(Inst),
+}
+
+impl ValueOrInst {
+    /// Get the underlying `Value` if any.
+    pub fn value(&self) -> Option<Value> {
+        match *self {
+            Self::Value(v) => Some(v),
+            Self::Inst(_) => None,
+        }
+    }
+
+    /// Get the underlying `Inst` if any.
+    pub fn inst(&self) -> Option<Inst> {
+        match *self {
+            Self::Inst(i) => Some(i),
+            Self::Value(_) => None,
+        }
+    }
+
+    /// Unwrap the underlying `Value`, panicking if it is not a `Value.
+    pub fn unwrap_value(&self) -> Value {
+        self.value().unwrap()
+    }
+
+    /// Unwrap the underlying `Inst`, panicking if it is not a `Inst.
+    pub fn unwrap_inst(&self) -> Inst {
+        self.inst().unwrap()
+    }
+
+    /// Is this a `Value`?
+    pub fn is_value(&self) -> bool {
+        self.value().is_some()
+    }
+
+    /// Is this an `Inst`?
+    pub fn is_inst(&self) -> bool {
+        self.inst().is_some()
+    }
+
+    fn resolve_inst(&self, dfg: &DataFlowGraph) -> Option<Inst> {
+        match *self {
+            ValueOrInst::Inst(i) => Some(i),
+            ValueOrInst::Value(v) => dfg.value_def(v).inst(),
+        }
+    }
+
+    fn result_bit_width(&self, dfg: &DataFlowGraph) -> u8 {
+        match *self {
+            ValueOrInst::Value(v) => dfg.value_type(v).bits().try_into().unwrap(),
+            ValueOrInst::Inst(inst) => {
+                let result = dfg.first_result(inst);
+                dfg.value_type(result).bits().try_into().unwrap()
+            }
+        }
+    }
+
+    fn to_constant(&self, pos: &mut FuncCursor) -> Option<Constant> {
+        let inst = self.resolve_inst(&pos.func.dfg)?;
+        match pos.func.dfg[inst] {
+            InstructionData::UnaryImm {
+                opcode: Opcode::Iconst,
+                imm,
+            } => {
+                let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap();
+                let x: i64 = imm.into();
+                Some(Constant::Int(x as u64, width))
+            }
+            InstructionData::UnaryBool {
+                opcode: Opcode::Bconst,
+                imm,
+            } => {
+                let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap();
+                Some(Constant::Bool(imm, width))
+            }
+            _ => None,
+        }
+    }
+}
+
+impl From<Value> for ValueOrInst {
+    fn from(v: Value) -> ValueOrInst {
+        ValueOrInst::Value(v)
+    }
+}
+
+impl From<Inst> for ValueOrInst {
+    fn from(i: Inst) -> ValueOrInst {
+        ValueOrInst::Inst(i)
+    }
+}
+
+/// Get the fixed bit width of `bit_width`, or if it is polymorphic, the bit
+/// width of `root`.
+fn bit_width(dfg: &DataFlowGraph, bit_width: BitWidth, root: Inst) -> u8 {
+    bit_width.fixed_width().unwrap_or_else(|| {
+        let tyvar = dfg.ctrl_typevar(root);
+        let ty = dfg.compute_result_type(root, 0, tyvar).unwrap();
+        u8::try_from(ty.bits()).unwrap()
+    })
+}
+
+/// Convert the constant `c` into an instruction.
+fn const_to_value<'a>(builder: impl InstBuilder<'a>, c: Constant, root: Inst) -> Value {
+    match c {
+        Constant::Bool(b, width) => {
+            let width = bit_width(builder.data_flow_graph(), width, root);
+            let ty = match width {
+                1 => types::B1,
+                8 => types::B8,
+                16 => types::B16,
+                32 => types::B32,
+                64 => types::B64,
+                128 => types::B128,
+                _ => unreachable!(),
+            };
+            builder.bconst(ty, b)
+        }
+        Constant::Int(x, width) => {
+            let width = bit_width(builder.data_flow_graph(), width, root);
+            let ty = match width {
+                8 => types::I8,
+                16 => types::I16,
+                32 => types::I32,
+                64 => types::I64,
+                128 => types::I128,
+                _ => unreachable!(),
+            };
+            builder.iconst(ty, x as i64)
+        }
+    }
+}
+
+fn part_to_value(pos: &mut FuncCursor, root: Inst, part: Part<ValueOrInst>) -> Option<Value> {
+    match part {
+        Part::Instruction(ValueOrInst::Inst(inst)) => {
+            pos.func.dfg.inst_results(inst).first().copied()
+        }
+        Part::Instruction(ValueOrInst::Value(v)) => Some(v),
+        Part::Constant(c) => Some(const_to_value(pos.ins(), c, root)),
+        Part::ConditionCode(_) => None,
+    }
+}
+
+impl TryFrom<Constant> for Imm64 {
+    type Error = &'static str;
+
+    fn try_from(c: Constant) -> Result<Self, Self::Error> {
+        match c {
+            Constant::Int(x, _) => Ok(Imm64::from(x as i64)),
+            Constant::Bool(..) => Err("cannot create Imm64 from Constant::Bool"),
+        }
+    }
+}
+
+impl Into<Constant> for Imm64 {
+    #[inline]
+    fn into(self) -> Constant {
+        let x: i64 = self.into();
+        Constant::Int(x as _, BitWidth::SixtyFour)
+    }
+}
+
+impl Into<Part<ValueOrInst>> for Imm64 {
+    #[inline]
+    fn into(self) -> Part<ValueOrInst> {
+        let c: Constant = self.into();
+        c.into()
+    }
+}
+
+fn part_to_imm64(pos: &mut FuncCursor, part: Part<ValueOrInst>) -> Imm64 {
+    return match part {
+        Part::Instruction(x) => match x.to_constant(pos).unwrap_or_else(|| cannot_convert()) {
+            Constant::Int(x, _) => (x as i64).into(),
+            Constant::Bool(..) => cannot_convert(),
+        },
+        Part::Constant(Constant::Int(x, _)) => (x as i64).into(),
+        Part::ConditionCode(_) | Part::Constant(Constant::Bool(..)) => cannot_convert(),
+    };
+
+    #[inline(never)]
+    #[cold]
+    fn cannot_convert() -> ! {
+        panic!("cannot convert part into `Imm64`")
+    }
+}
+
+impl Into<Constant> for Uimm64 {
+    #[inline]
+    fn into(self) -> Constant {
+        let x: u64 = self.into();
+        Constant::Int(x, BitWidth::SixtyFour)
+    }
+}
+
+impl Into<Part<ValueOrInst>> for Uimm64 {
+    #[inline]
+    fn into(self) -> Part<ValueOrInst> {
+        let c: Constant = self.into();
+        c.into()
+    }
+}
+
+fn peepmatic_to_intcc(cc: ConditionCode) -> IntCC {
+    match cc {
+        ConditionCode::Eq => IntCC::Equal,
+        ConditionCode::Ne => IntCC::NotEqual,
+        ConditionCode::Slt => IntCC::SignedLessThan,
+        ConditionCode::Sle => IntCC::SignedGreaterThanOrEqual,
+        ConditionCode::Sgt => IntCC::SignedGreaterThan,
+        ConditionCode::Sge => IntCC::SignedLessThanOrEqual,
+        ConditionCode::Ult => IntCC::UnsignedLessThan,
+        ConditionCode::Uge => IntCC::UnsignedGreaterThanOrEqual,
+        ConditionCode::Ugt => IntCC::UnsignedGreaterThan,
+        ConditionCode::Ule => IntCC::UnsignedLessThanOrEqual,
+        ConditionCode::Of => IntCC::Overflow,
+        ConditionCode::Nof => IntCC::NotOverflow,
+    }
+}
+
+fn intcc_to_peepmatic(cc: IntCC) -> ConditionCode {
+    match cc {
+        IntCC::Equal => ConditionCode::Eq,
+        IntCC::NotEqual => ConditionCode::Ne,
+        IntCC::SignedLessThan => ConditionCode::Slt,
+        IntCC::SignedGreaterThanOrEqual => ConditionCode::Sle,
+        IntCC::SignedGreaterThan => ConditionCode::Sgt,
+        IntCC::SignedLessThanOrEqual => ConditionCode::Sge,
+        IntCC::UnsignedLessThan => ConditionCode::Ult,
+        IntCC::UnsignedGreaterThanOrEqual => ConditionCode::Uge,
+        IntCC::UnsignedGreaterThan => ConditionCode::Ugt,
+        IntCC::UnsignedLessThanOrEqual => ConditionCode::Ule,
+        IntCC::Overflow => ConditionCode::Of,
+        IntCC::NotOverflow => ConditionCode::Nof,
+    }
+}
+
+fn peepmatic_ty_to_ir_ty(ty: Type, dfg: &DataFlowGraph, root: Inst) -> types::Type {
+    match (ty.kind, bit_width(dfg, ty.bit_width, root)) {
+        (Kind::Int, 8) => types::I8,
+        (Kind::Int, 16) => types::I16,
+        (Kind::Int, 32) => types::I32,
+        (Kind::Int, 64) => types::I64,
+        (Kind::Int, 128) => types::I128,
+        (Kind::Bool, 1) => types::B1,
+        (Kind::Bool, 8) => types::I8,
+        (Kind::Bool, 16) => types::I16,
+        (Kind::Bool, 32) => types::I32,
+        (Kind::Bool, 64) => types::I64,
+        (Kind::Bool, 128) => types::I128,
+        _ => unreachable!(),
+    }
+}
+
+// NB: the unsafe contract we must uphold here is that our implementation of
+// `instruction_result_bit_width` must always return a valid, non-zero bit
+// width.
+unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa {
+    type Context = FuncCursor<'b>;
+
+    type Operator = Opcode;
+
+    type Instruction = ValueOrInst;
+
+    fn replace_instruction(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        old: ValueOrInst,
+        new: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("replace {:?} with {:?}", old, new);
+        let old_inst = old.resolve_inst(&pos.func.dfg).unwrap();
+
+        // Try to convert `new` to an instruction, because we prefer replacing
+        // an old instruction with a new one wholesale. However, if the
+        // replacement cannot be converted to an instruction (e.g. the
+        // right-hand side is a block/function parameter value) then we change
+        // the old instruction's result to an alias of the new value.
+        let new_inst = match new {
+            Part::Instruction(ValueOrInst::Inst(inst)) => Some(inst),
+            Part::Instruction(ValueOrInst::Value(_)) => {
+                // Do not try and follow the value definition. If we transplant
+                // this value's instruction, and there are other uses of this
+                // value, then we could mess up ordering between instructions.
+                None
+            }
+            Part::Constant(c) => {
+                let v = const_to_value(pos.ins(), c, old_inst);
+                let inst = pos.func.dfg.value_def(v).unwrap_inst();
+                Some(inst)
+            }
+            Part::ConditionCode(_) => None,
+        };
+
+        match new_inst {
+            Some(new_inst) => {
+                pos.func.transplant_inst(old_inst, new_inst);
+                debug_assert_eq!(pos.current_inst(), Some(old_inst));
+                old_inst.into()
+            }
+            None => {
+                let new_value = part_to_value(pos, old_inst, new).unwrap();
+
+                let old_results = pos.func.dfg.detach_results(old_inst);
+                let old_results = old_results.as_slice(&pos.func.dfg.value_lists);
+                assert_eq!(old_results.len(), 1);
+                let old_value = old_results[0];
+
+                pos.func.dfg.change_to_alias(old_value, new_value);
+                pos.func.dfg.replace(old_inst).nop();
+
+                new_value.into()
+            }
+        }
+    }
+
+    fn operator<E>(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        value_or_inst: ValueOrInst,
+        operands: &mut E,
+    ) -> Option<Opcode>
+    where
+        E: Extend<Part<Self::Instruction>>,
+    {
+        let inst = value_or_inst.resolve_inst(&pos.func.dfg)?;
+        Some(match pos.func.dfg[inst] {
+            InstructionData::Binary {
+                opcode: opcode @ Opcode::Band,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Bor,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Bxor,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Iadd,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Ifcmp,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Imul,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Ishl,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Isub,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Rotl,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Rotr,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Sdiv,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Srem,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Sshr,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Udiv,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Urem,
+                args,
+            }
+            | InstructionData::Binary {
+                opcode: opcode @ Opcode::Ushr,
+                args,
+            } => {
+                operands.extend(args.iter().map(|v| Part::Instruction((*v).into())));
+                opcode
+            }
+
+            InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::BandImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::BorImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::BxorImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::IaddImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::IfcmpImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::ImulImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::IrsubImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::IshlImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::RotlImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::RotrImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::SdivImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::SremImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::SshrImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::UdivImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::UremImm,
+                imm,
+                arg,
+            }
+            | InstructionData::BinaryImm64 {
+                opcode: opcode @ Opcode::UshrImm,
+                imm,
+                arg,
+            } => {
+                operands.extend(
+                    iter::once(imm.into()).chain(iter::once(Part::Instruction(arg.into()))),
+                );
+                opcode
+            }
+
+            InstructionData::Branch {
+                opcode: opcode @ Opcode::Brnz,
+                ref args,
+                destination: _,
+            }
+            | InstructionData::Branch {
+                opcode: opcode @ Opcode::Brz,
+                ref args,
+                destination: _,
+            } => {
+                operands.extend(
+                    args.as_slice(&pos.func.dfg.value_lists)
+                        .iter()
+                        .map(|v| Part::Instruction((*v).into()))
+                        // NB: Peepmatic only knows about the condition, not any
+                        // of the arguments to the block, which are special
+                        // cased elsewhere, if/when we actually replace the
+                        // instruction.
+                        .take(1),
+                );
+                opcode
+            }
+
+            InstructionData::CondTrap {
+                opcode: opcode @ Opcode::Trapnz,
+                arg,
+                code: _,
+            }
+            | InstructionData::CondTrap {
+                opcode: opcode @ Opcode::Trapz,
+                arg,
+                code: _,
+            } => {
+                operands.extend(iter::once(Part::Instruction(arg.into())));
+                opcode
+            }
+
+            InstructionData::IntCompare {
+                opcode: opcode @ Opcode::Icmp,
+                cond,
+                args,
+            } => {
+                operands.extend(
+                    iter::once(intcc_to_peepmatic(cond).into())
+                        .chain(args.iter().map(|v| Part::Instruction((*v).into()))),
+                );
+                opcode
+            }
+
+            InstructionData::IntCompareImm {
+                opcode: opcode @ Opcode::IcmpImm,
+                cond,
+                imm,
+                arg,
+            } => {
+                operands.extend(
+                    iter::once(intcc_to_peepmatic(cond).into())
+                        .chain(iter::once(Part::Constant(imm.into())))
+                        .chain(iter::once(Part::Instruction(arg.into()))),
+                );
+                opcode
+            }
+
+            InstructionData::Ternary {
+                opcode: opcode @ Opcode::Select,
+                ref args,
+            } => {
+                operands.extend(args.iter().map(|v| Part::Instruction((*v).into())));
+                opcode
+            }
+
+            InstructionData::Unary {
+                opcode: opcode @ Opcode::AdjustSpDown,
+                arg,
+            }
+            | InstructionData::Unary {
+                opcode: opcode @ Opcode::Bint,
+                arg,
+            }
+            | InstructionData::Unary {
+                opcode: opcode @ Opcode::Ireduce,
+                arg,
+            }
+            | InstructionData::Unary {
+                opcode: opcode @ Opcode::Sextend,
+                arg,
+            }
+            | InstructionData::Unary {
+                opcode: opcode @ Opcode::Uextend,
+                arg,
+            } => {
+                operands.extend(iter::once(Part::Instruction(arg.into())));
+                opcode
+            }
+
+            InstructionData::UnaryBool { opcode, imm } => {
+                operands.extend(iter::once(Part::Constant(Constant::Bool(
+                    imm,
+                    BitWidth::Polymorphic,
+                ))));
+                opcode
+            }
+
+            InstructionData::UnaryImm {
+                opcode: opcode @ Opcode::AdjustSpDownImm,
+                imm,
+            }
+            | InstructionData::UnaryImm {
+                opcode: opcode @ Opcode::Iconst,
+                imm,
+            } => {
+                operands.extend(iter::once(imm.into()));
+                opcode
+            }
+            ref otherwise => {
+                log::trace!("Not supported by Peepmatic: {:?}", otherwise);
+                return None;
+            }
+        })
+    }
+
+    fn make_inst_1(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        operator: Opcode,
+        r#type: Type,
+        a: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("make_inst_1: {:?}({:?})", operator, a);
+
+        let root = root.resolve_inst(&pos.func.dfg).unwrap();
+        match operator {
+            Opcode::AdjustSpDown => {
+                let a = part_to_value(pos, root, a).unwrap();
+                pos.ins().adjust_sp_down(a).into()
+            }
+            Opcode::AdjustSpDownImm => {
+                let c = a.unwrap_constant();
+                let imm = Imm64::try_from(c).unwrap();
+                pos.ins().adjust_sp_down_imm(imm).into()
+            }
+            Opcode::Bconst => {
+                let c = a.unwrap_constant();
+                let val = const_to_value(pos.ins(), c, root);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Bint => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                let val = pos.ins().bint(ty, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Bnot => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let val = pos.ins().bnot(a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Brnz => {
+                let a = part_to_value(pos, root, a).unwrap();
+
+                // NB: branching instructions must be the root of an
+                // optimization's right-hand side, so we get the destination
+                // block and arguments from the left-hand side's root. Peepmatic
+                // doesn't currently represent labels or varargs.
+                let block = pos.func.dfg[root].branch_destination().unwrap();
+                let args = pos.func.dfg.inst_args(root)[1..].to_vec();
+
+                pos.ins().brnz(a, block, &args).into()
+            }
+            Opcode::Brz => {
+                let a = part_to_value(pos, root, a).unwrap();
+
+                // See the comment in the `Opcode::Brnz` match argm.
+                let block = pos.func.dfg[root].branch_destination().unwrap();
+                let args = pos.func.dfg.inst_args(root)[1..].to_vec();
+
+                pos.ins().brz(a, block, &args).into()
+            }
+            Opcode::Iconst => {
+                let a = a.unwrap_constant();
+                let val = const_to_value(pos.ins(), a, root);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Ireduce => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                let val = pos.ins().ireduce(ty, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Sextend => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                let val = pos.ins().sextend(ty, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Trapnz => {
+                let a = part_to_value(pos, root, a).unwrap();
+
+                // NB: similar to branching instructions (see comment in the
+                // `Opcode::Brnz` match arm) trapping instructions must be the
+                // root of an optimization's right-hand side, and we get the
+                // trap code from the root of the left-hand side. Peepmatic
+                // doesn't currently represent trap codes.
+                let code = pos.func.dfg[root].trap_code().unwrap();
+
+                pos.ins().trapnz(a, code).into()
+            }
+            Opcode::Trapz => {
+                let a = part_to_value(pos, root, a).unwrap();
+                // See comment in the `Opcode::Trapnz` match arm.
+                let code = pos.func.dfg[root].trap_code().unwrap();
+                pos.ins().trapz(a, code).into()
+            }
+            Opcode::Uextend => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root);
+                let val = pos.ins().uextend(ty, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn make_inst_2(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        operator: Opcode,
+        _: Type,
+        a: Part<ValueOrInst>,
+        b: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("make_inst_2: {:?}({:?}, {:?})", operator, a, b);
+
+        let root = root.resolve_inst(&pos.func.dfg).unwrap();
+        match operator {
+            Opcode::Band => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().band(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::BandImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().band_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Bor => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().bor(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::BorImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().bor_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Bxor => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().bxor(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::BxorImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().bxor_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Iadd => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().iadd(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::IaddImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().iadd_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Ifcmp => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ifcmp(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::IfcmpImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ifcmp_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Imul => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().imul(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::ImulImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().imul_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::IrsubImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().irsub_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Ishl => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ishl(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::IshlImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ishl_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Isub => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().isub(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Rotl => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().rotl(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::RotlImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().rotl_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Rotr => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().rotr(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::RotrImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().rotr_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Sdiv => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().sdiv(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::SdivImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().sdiv_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Srem => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().srem(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::SremImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().srem_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Sshr => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().sshr(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::SshrImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().sshr_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Udiv => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().udiv(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::UdivImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().udiv_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Urem => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().urem(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::UremImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().urem_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Ushr => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ushr(a, b);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::UshrImm => {
+                let a = part_to_imm64(pos, a);
+                let b = part_to_value(pos, root, b).unwrap();
+                let val = pos.ins().ushr_imm(b, a);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn make_inst_3(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        root: ValueOrInst,
+        operator: Opcode,
+        _: Type,
+        a: Part<ValueOrInst>,
+        b: Part<ValueOrInst>,
+        c: Part<ValueOrInst>,
+    ) -> ValueOrInst {
+        log::trace!("make_inst_3: {:?}({:?}, {:?}, {:?})", operator, a, b, c);
+
+        let root = root.resolve_inst(&pos.func.dfg).unwrap();
+        match operator {
+            Opcode::Icmp => {
+                let cond = a.unwrap_condition_code();
+                let cond = peepmatic_to_intcc(cond);
+                let b = part_to_value(pos, root, b).unwrap();
+                let c = part_to_value(pos, root, c).unwrap();
+                let val = pos.ins().icmp(cond, b, c);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::IcmpImm => {
+                let cond = a.unwrap_condition_code();
+                let cond = peepmatic_to_intcc(cond);
+                let imm = part_to_imm64(pos, b);
+                let c = part_to_value(pos, root, c).unwrap();
+                let val = pos.ins().icmp_imm(cond, c, imm);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            Opcode::Select => {
+                let a = part_to_value(pos, root, a).unwrap();
+                let b = part_to_value(pos, root, b).unwrap();
+                let c = part_to_value(pos, root, c).unwrap();
+                let val = pos.ins().select(a, b, c);
+                pos.func.dfg.value_def(val).unwrap_inst().into()
+            }
+            _ => unreachable!(),
+        }
+    }
+
+    fn instruction_to_constant(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        value_or_inst: ValueOrInst,
+    ) -> Option<Constant> {
+        value_or_inst.to_constant(pos)
+    }
+
+    fn instruction_result_bit_width(
+        &self,
+        pos: &mut FuncCursor<'b>,
+        value_or_inst: ValueOrInst,
+    ) -> u8 {
+        value_or_inst.result_bit_width(&pos.func.dfg)
+    }
+
+    fn native_word_size_in_bits(&self, _pos: &mut FuncCursor<'b>) -> u8 {
+        self.pointer_bits()
+    }
+}
+
+#[cfg(test)]
+#[cfg(feature = "x86")]
+mod tests {
+    use super::*;
+    use crate::isa::lookup;
+    use crate::settings::{builder, Flags};
+    use std::str::FromStr;
+    use target_lexicon::triple;
+
+    #[test]
+    fn get_peepmatic_preopt() {
+        let isa = lookup(triple!("x86_64"))
+            .expect("expect x86 ISA")
+            .finish(Flags::new(builder()));
+        let _ = preopt(&*isa);
+    }
+}