//! Glue for working with `peepmatic`-generated peephole optimizers. use crate::cursor::{Cursor, FuncCursor}; use crate::ir::{ dfg::DataFlowGraph, entities::{Inst, Value}, immediates::{Imm64, Uimm64}, instructions::{InstructionData, Opcode}, types, InstBuilder, }; use crate::isa::TargetIsa; use cranelift_codegen_shared::condcodes::IntCC; use peepmatic_runtime::{ cc::ConditionCode, instruction_set::InstructionSet, part::{Constant, Part}, r#type::{BitWidth, Kind, Type}, PeepholeOptimizations, PeepholeOptimizer, }; use std::borrow::Cow; use std::boxed::Box; use std::convert::{TryFrom, TryInto}; use std::iter; use std::ptr; use std::sync::atomic::{AtomicPtr, Ordering}; peepmatic_traits::define_parse_and_typing_rules_for_operator! { Opcode { adjust_sp_down => AdjustSpDown { parameters(iNN); result(void); } adjust_sp_down_imm => AdjustSpDownImm { immediates(iNN); result(void); } band => Band { parameters(iNN, iNN); result(iNN); } band_imm => BandImm { immediates(iNN); parameters(iNN); result(iNN); } bconst => Bconst { immediates(b1); result(bNN); } bint => Bint { parameters(bNN); result(iNN); } bor => Bor { parameters(iNN, iNN); result(iNN); } bor_imm => BorImm { immediates(iNN); parameters(iNN); result(iNN); } brnz => Brnz { parameters(bool_or_int); result(void); } brz => Brz { parameters(bool_or_int); result(void); } bxor => Bxor { parameters(iNN, iNN); result(iNN); } bxor_imm => BxorImm { immediates(iNN); parameters(iNN); result(iNN); } iadd => Iadd { parameters(iNN, iNN); result(iNN); } iadd_imm => IaddImm { immediates(iNN); parameters(iNN); result(iNN); } icmp => Icmp { immediates(cc); parameters(iNN, iNN); result(b1); } icmp_imm => IcmpImm { immediates(cc, iNN); parameters(iNN); result(b1); } iconst => Iconst { immediates(iNN); result(iNN); } ifcmp => Ifcmp { parameters(iNN, iNN); result(cpu_flags); } ifcmp_imm => IfcmpImm { immediates(iNN); parameters(iNN); result(cpu_flags); } imul => Imul { parameters(iNN, iNN); result(iNN); } imul_imm => ImulImm { immediates(iNN); parameters(iNN); result(iNN); } ireduce => Ireduce { parameters(iNN); result(iMM); is_reduce(true); } irsub_imm => IrsubImm { immediates(iNN); parameters(iNN); result(iNN); } ishl => Ishl { parameters(iNN, iNN); result(iNN); } ishl_imm => IshlImm { immediates(iNN); parameters(iNN); result(iNN); } isub => Isub { parameters(iNN, iNN); result(iNN); } rotl => Rotl { parameters(iNN, iNN); result(iNN); } rotl_imm => RotlImm { immediates(iNN); parameters(iNN); result(iNN); } rotr => Rotr { parameters(iNN, iNN); result(iNN); } rotr_imm => RotrImm { immediates(iNN); parameters(iNN); result(iNN); } sdiv => Sdiv { parameters(iNN, iNN); result(iNN); } sdiv_imm => SdivImm { immediates(iNN); parameters(iNN); result(iNN); } select => Select { parameters(bool_or_int, any_t, any_t); result(any_t); } sextend => Sextend { parameters(iNN); result(iMM); is_extend(true); } srem => Srem { parameters(iNN, iNN); result(iNN); } srem_imm => SremImm { immediates(iNN); parameters(iNN); result(iNN); } sshr => Sshr { parameters(iNN, iNN); result(iNN); } sshr_imm => SshrImm { immediates(iNN); parameters(iNN); result(iNN); } trapnz => Trapnz { parameters(bool_or_int); result(void); } trapz => Trapz { parameters(bool_or_int); result(void); } udiv => Udiv { parameters(iNN, iNN); result(iNN); } udiv_imm => UdivImm { immediates(iNN); parameters(iNN); result(iNN); } uextend => Uextend { parameters(iNN); result(iMM); is_extend(true); } urem => Urem { parameters(iNN, iNN); result(iNN); } urem_imm => UremImm { immediates(iNN); parameters(iNN); result(iNN); } ushr => Ushr { parameters(iNN, iNN); result(iNN); } ushr_imm => UshrImm { immediates(iNN); parameters(iNN); result(iNN); } } parse_cfg(feature = "rebuild-peephole-optimizers"); } /// Code required to rebuild Peepmatic-based peephole optimizers. /// /// This module is used to scope imports and dependencies that are only required /// for building peephole optimizers (as opposed to just using pre-built /// peephole optimizers). This helps ensure that our regular builds using /// pre-built peephole optimizers stay lean. #[cfg(feature = "rebuild-peephole-optimizers")] mod rebuild { use super::*; use alloc::vec::Vec; use std::fs; use std::path::Path; /// Rebuild the `preopt.peepmatic` peephole optimizer. /// /// Saves and overwrites the old `preopt.serialized` build and returns a /// copy of the result. pub fn rebuild_preopt() -> Vec { let codegen_path = Path::new(include_str!(concat!( env!("OUT_DIR"), "/CRANELIFT_CODEGEN_PATH" ))); let source_path = codegen_path.join("src").join("preopt.peepmatic"); let preopt = peepmatic::compile_file::(&source_path) .expect("failed to compile `src/preopt.peepmatic`"); let serialized_path = codegen_path.join("src").join("preopt.serialized"); preopt .serialize_to_file(&serialized_path) .expect("failed to serialize peephole optimizer to `src/preopt.serialized`"); fs::read(&serialized_path).expect("failed to read `src/preopt.serialized`") } } /// Get the `preopt.peepmatic` peephole optimizer. pub(crate) fn preopt<'a, 'b>( isa: &'b dyn TargetIsa, ) -> PeepholeOptimizer<'static, 'a, &'b dyn TargetIsa> { #[cfg(feature = "rebuild-peephole-optimizers")] fn get_serialized() -> Cow<'static, [u8]> { rebuild::rebuild_preopt().into() } #[cfg(not(feature = "rebuild-peephole-optimizers"))] fn get_serialized() -> Cow<'static, [u8]> { static SERIALIZED: &[u8] = include_bytes!("preopt.serialized"); SERIALIZED.into() } // Once initialized, this must never be re-assigned. The initialized value // is semantically "static data" and is intentionally leaked for the whole // program's lifetime. static DESERIALIZED: AtomicPtr> = AtomicPtr::new(ptr::null_mut()); // If `DESERIALIZED` has already been initialized, then just use it. let ptr = DESERIALIZED.load(Ordering::SeqCst); if let Some(peep_opts) = unsafe { ptr.as_ref() } { return peep_opts.optimizer(isa); } // Otherwise, if `DESERIALIZED` hasn't been initialized, then we need to // deserialize the peephole optimizations and initialize it. However, // another thread could be doing the same thing concurrently, so there is a // race to see who initializes `DESERIALIZED` first, and we need to be // prepared to both win or lose that race. let peep_opts = PeepholeOptimizations::deserialize(&get_serialized()) .expect("should always be able to deserialize `preopt.serialized`"); let peep_opts = Box::into_raw(Box::new(peep_opts)); // Only update `DESERIALIZE` if it is still null, attempting to perform the // one-time transition from null -> non-null. if DESERIALIZED .compare_and_swap(ptr::null_mut(), peep_opts, Ordering::SeqCst) .is_null() { // We won the race to initialize `DESERIALIZED`. debug_assert_eq!(DESERIALIZED.load(Ordering::SeqCst), peep_opts); let peep_opts = unsafe { &*peep_opts }; return peep_opts.optimizer(isa); } // We lost the race to initialize `DESERIALIZED`. Drop our no-longer-needed // instance of `peep_opts` and get the pointer to the instance that won the // race. let _ = unsafe { Box::from_raw(peep_opts) }; let peep_opts = DESERIALIZED.load(Ordering::SeqCst); let peep_opts = unsafe { peep_opts.as_ref().unwrap() }; peep_opts.optimizer(isa) } /// Either a `Value` or an `Inst`. #[derive(Clone, Copy, Debug, Eq, PartialEq)] pub enum ValueOrInst { Value(Value), Inst(Inst), } impl ValueOrInst { /// Get the underlying `Value` if any. pub fn value(&self) -> Option { match *self { Self::Value(v) => Some(v), Self::Inst(_) => None, } } /// Get the underlying `Inst` if any. pub fn inst(&self) -> Option { match *self { Self::Inst(i) => Some(i), Self::Value(_) => None, } } /// Unwrap the underlying `Value`, panicking if it is not a `Value. pub fn unwrap_value(&self) -> Value { self.value().unwrap() } /// Unwrap the underlying `Inst`, panicking if it is not a `Inst. pub fn unwrap_inst(&self) -> Inst { self.inst().unwrap() } /// Is this a `Value`? pub fn is_value(&self) -> bool { self.value().is_some() } /// Is this an `Inst`? pub fn is_inst(&self) -> bool { self.inst().is_some() } fn resolve_inst(&self, dfg: &DataFlowGraph) -> Option { match *self { ValueOrInst::Inst(i) => Some(i), ValueOrInst::Value(v) => dfg.value_def(v).inst(), } } fn result_bit_width(&self, dfg: &DataFlowGraph) -> u8 { match *self { ValueOrInst::Value(v) => dfg.value_type(v).bits().try_into().unwrap(), ValueOrInst::Inst(inst) => { let result = dfg.first_result(inst); dfg.value_type(result).bits().try_into().unwrap() } } } fn to_constant(&self, pos: &mut FuncCursor) -> Option { let inst = self.resolve_inst(&pos.func.dfg)?; match pos.func.dfg[inst] { InstructionData::UnaryImm { opcode: Opcode::Iconst, imm, } => { let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap(); let x: i64 = imm.into(); Some(Constant::Int(x as u64, width)) } InstructionData::UnaryBool { opcode: Opcode::Bconst, imm, } => { let width = self.result_bit_width(&pos.func.dfg).try_into().unwrap(); Some(Constant::Bool(imm, width)) } _ => None, } } } impl From for ValueOrInst { fn from(v: Value) -> ValueOrInst { ValueOrInst::Value(v) } } impl From for ValueOrInst { fn from(i: Inst) -> ValueOrInst { ValueOrInst::Inst(i) } } /// Get the fixed bit width of `bit_width`, or if it is polymorphic, the bit /// width of `root`. fn bit_width(dfg: &DataFlowGraph, bit_width: BitWidth, root: Inst) -> u8 { bit_width.fixed_width().unwrap_or_else(|| { let tyvar = dfg.ctrl_typevar(root); let ty = dfg.compute_result_type(root, 0, tyvar).unwrap(); u8::try_from(ty.bits()).unwrap() }) } /// Convert the constant `c` into an instruction. fn const_to_value<'a>(builder: impl InstBuilder<'a>, c: Constant, root: Inst) -> Value { match c { Constant::Bool(b, width) => { let width = bit_width(builder.data_flow_graph(), width, root); let ty = match width { 1 => types::B1, 8 => types::B8, 16 => types::B16, 32 => types::B32, 64 => types::B64, 128 => types::B128, _ => unreachable!(), }; builder.bconst(ty, b) } Constant::Int(x, width) => { let width = bit_width(builder.data_flow_graph(), width, root); let ty = match width { 8 => types::I8, 16 => types::I16, 32 => types::I32, 64 => types::I64, 128 => types::I128, _ => unreachable!(), }; builder.iconst(ty, x as i64) } } } fn part_to_value(pos: &mut FuncCursor, root: Inst, part: Part) -> Option { match part { Part::Instruction(ValueOrInst::Inst(inst)) => { pos.func.dfg.inst_results(inst).first().copied() } Part::Instruction(ValueOrInst::Value(v)) => Some(v), Part::Constant(c) => Some(const_to_value(pos.ins(), c, root)), Part::ConditionCode(_) => None, } } impl TryFrom for Imm64 { type Error = &'static str; fn try_from(c: Constant) -> Result { match c { Constant::Int(x, _) => Ok(Imm64::from(x as i64)), Constant::Bool(..) => Err("cannot create Imm64 from Constant::Bool"), } } } impl Into for Imm64 { #[inline] fn into(self) -> Constant { let x: i64 = self.into(); Constant::Int(x as _, BitWidth::SixtyFour) } } impl Into> for Imm64 { #[inline] fn into(self) -> Part { let c: Constant = self.into(); c.into() } } fn part_to_imm64(pos: &mut FuncCursor, part: Part) -> Imm64 { return match part { Part::Instruction(x) => match x.to_constant(pos).unwrap_or_else(|| cannot_convert()) { Constant::Int(x, _) => (x as i64).into(), Constant::Bool(..) => cannot_convert(), }, Part::Constant(Constant::Int(x, _)) => (x as i64).into(), Part::ConditionCode(_) | Part::Constant(Constant::Bool(..)) => cannot_convert(), }; #[inline(never)] #[cold] fn cannot_convert() -> ! { panic!("cannot convert part into `Imm64`") } } impl Into for Uimm64 { #[inline] fn into(self) -> Constant { let x: u64 = self.into(); Constant::Int(x, BitWidth::SixtyFour) } } impl Into> for Uimm64 { #[inline] fn into(self) -> Part { let c: Constant = self.into(); c.into() } } fn peepmatic_to_intcc(cc: ConditionCode) -> IntCC { match cc { ConditionCode::Eq => IntCC::Equal, ConditionCode::Ne => IntCC::NotEqual, ConditionCode::Slt => IntCC::SignedLessThan, ConditionCode::Sle => IntCC::SignedGreaterThanOrEqual, ConditionCode::Sgt => IntCC::SignedGreaterThan, ConditionCode::Sge => IntCC::SignedLessThanOrEqual, ConditionCode::Ult => IntCC::UnsignedLessThan, ConditionCode::Uge => IntCC::UnsignedGreaterThanOrEqual, ConditionCode::Ugt => IntCC::UnsignedGreaterThan, ConditionCode::Ule => IntCC::UnsignedLessThanOrEqual, ConditionCode::Of => IntCC::Overflow, ConditionCode::Nof => IntCC::NotOverflow, } } fn intcc_to_peepmatic(cc: IntCC) -> ConditionCode { match cc { IntCC::Equal => ConditionCode::Eq, IntCC::NotEqual => ConditionCode::Ne, IntCC::SignedLessThan => ConditionCode::Slt, IntCC::SignedGreaterThanOrEqual => ConditionCode::Sle, IntCC::SignedGreaterThan => ConditionCode::Sgt, IntCC::SignedLessThanOrEqual => ConditionCode::Sge, IntCC::UnsignedLessThan => ConditionCode::Ult, IntCC::UnsignedGreaterThanOrEqual => ConditionCode::Uge, IntCC::UnsignedGreaterThan => ConditionCode::Ugt, IntCC::UnsignedLessThanOrEqual => ConditionCode::Ule, IntCC::Overflow => ConditionCode::Of, IntCC::NotOverflow => ConditionCode::Nof, } } fn peepmatic_ty_to_ir_ty(ty: Type, dfg: &DataFlowGraph, root: Inst) -> types::Type { match (ty.kind, bit_width(dfg, ty.bit_width, root)) { (Kind::Int, 8) => types::I8, (Kind::Int, 16) => types::I16, (Kind::Int, 32) => types::I32, (Kind::Int, 64) => types::I64, (Kind::Int, 128) => types::I128, (Kind::Bool, 1) => types::B1, (Kind::Bool, 8) => types::I8, (Kind::Bool, 16) => types::I16, (Kind::Bool, 32) => types::I32, (Kind::Bool, 64) => types::I64, (Kind::Bool, 128) => types::I128, _ => unreachable!(), } } // NB: the unsafe contract we must uphold here is that our implementation of // `instruction_result_bit_width` must always return a valid, non-zero bit // width. unsafe impl<'a, 'b> InstructionSet<'b> for &'a dyn TargetIsa { type Context = FuncCursor<'b>; type Operator = Opcode; type Instruction = ValueOrInst; fn replace_instruction( &self, pos: &mut FuncCursor<'b>, old: ValueOrInst, new: Part, ) -> ValueOrInst { log::trace!("replace {:?} with {:?}", old, new); let old_inst = old.resolve_inst(&pos.func.dfg).unwrap(); // Try to convert `new` to an instruction, because we prefer replacing // an old instruction with a new one wholesale. However, if the // replacement cannot be converted to an instruction (e.g. the // right-hand side is a block/function parameter value) then we change // the old instruction's result to an alias of the new value. let new_inst = match new { Part::Instruction(ValueOrInst::Inst(inst)) => Some(inst), Part::Instruction(ValueOrInst::Value(_)) => { // Do not try and follow the value definition. If we transplant // this value's instruction, and there are other uses of this // value, then we could mess up ordering between instructions. None } Part::Constant(c) => { let v = const_to_value(pos.ins(), c, old_inst); let inst = pos.func.dfg.value_def(v).unwrap_inst(); Some(inst) } Part::ConditionCode(_) => None, }; match new_inst { Some(new_inst) => { pos.func.transplant_inst(old_inst, new_inst); debug_assert_eq!(pos.current_inst(), Some(old_inst)); old_inst.into() } None => { let new_value = part_to_value(pos, old_inst, new).unwrap(); let old_results = pos.func.dfg.detach_results(old_inst); let old_results = old_results.as_slice(&pos.func.dfg.value_lists); assert_eq!(old_results.len(), 1); let old_value = old_results[0]; pos.func.dfg.change_to_alias(old_value, new_value); pos.func.dfg.replace(old_inst).nop(); new_value.into() } } } fn operator( &self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst, operands: &mut E, ) -> Option where E: Extend>, { let inst = value_or_inst.resolve_inst(&pos.func.dfg)?; Some(match pos.func.dfg[inst] { InstructionData::Binary { opcode: opcode @ Opcode::Band, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Bor, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Bxor, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Iadd, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Ifcmp, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Imul, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Ishl, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Isub, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Rotl, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Rotr, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Sdiv, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Srem, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Sshr, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Udiv, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Urem, args, } | InstructionData::Binary { opcode: opcode @ Opcode::Ushr, args, } => { operands.extend(args.iter().map(|v| Part::Instruction((*v).into()))); opcode } InstructionData::BinaryImm64 { opcode: opcode @ Opcode::BandImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::BorImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::BxorImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::IaddImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::IfcmpImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::ImulImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::IrsubImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::IshlImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::RotlImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::RotrImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::SdivImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::SremImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::SshrImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::UdivImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::UremImm, imm, arg, } | InstructionData::BinaryImm64 { opcode: opcode @ Opcode::UshrImm, imm, arg, } => { operands.extend( iter::once(imm.into()).chain(iter::once(Part::Instruction(arg.into()))), ); opcode } InstructionData::Branch { opcode: opcode @ Opcode::Brnz, ref args, destination: _, } | InstructionData::Branch { opcode: opcode @ Opcode::Brz, ref args, destination: _, } => { operands.extend( args.as_slice(&pos.func.dfg.value_lists) .iter() .map(|v| Part::Instruction((*v).into())) // NB: Peepmatic only knows about the condition, not any // of the arguments to the block, which are special // cased elsewhere, if/when we actually replace the // instruction. .take(1), ); opcode } InstructionData::CondTrap { opcode: opcode @ Opcode::Trapnz, arg, code: _, } | InstructionData::CondTrap { opcode: opcode @ Opcode::Trapz, arg, code: _, } => { operands.extend(iter::once(Part::Instruction(arg.into()))); opcode } InstructionData::IntCompare { opcode: opcode @ Opcode::Icmp, cond, args, } => { operands.extend( iter::once(intcc_to_peepmatic(cond).into()) .chain(args.iter().map(|v| Part::Instruction((*v).into()))), ); opcode } InstructionData::IntCompareImm { opcode: opcode @ Opcode::IcmpImm, cond, imm, arg, } => { operands.extend( iter::once(intcc_to_peepmatic(cond).into()) .chain(iter::once(Part::Constant(imm.into()))) .chain(iter::once(Part::Instruction(arg.into()))), ); opcode } InstructionData::Ternary { opcode: opcode @ Opcode::Select, ref args, } => { operands.extend(args.iter().map(|v| Part::Instruction((*v).into()))); opcode } InstructionData::Unary { opcode: opcode @ Opcode::AdjustSpDown, arg, } | InstructionData::Unary { opcode: opcode @ Opcode::Bint, arg, } | InstructionData::Unary { opcode: opcode @ Opcode::Ireduce, arg, } | InstructionData::Unary { opcode: opcode @ Opcode::Sextend, arg, } | InstructionData::Unary { opcode: opcode @ Opcode::Uextend, arg, } => { operands.extend(iter::once(Part::Instruction(arg.into()))); opcode } InstructionData::UnaryBool { opcode, imm } => { operands.extend(iter::once(Part::Constant(Constant::Bool( imm, BitWidth::Polymorphic, )))); opcode } InstructionData::UnaryImm { opcode: opcode @ Opcode::AdjustSpDownImm, imm, } | InstructionData::UnaryImm { opcode: opcode @ Opcode::Iconst, imm, } => { operands.extend(iter::once(imm.into())); opcode } ref otherwise => { log::trace!("Not supported by Peepmatic: {:?}", otherwise); return None; } }) } fn make_inst_1( &self, pos: &mut FuncCursor<'b>, root: ValueOrInst, operator: Opcode, r#type: Type, a: Part, ) -> ValueOrInst { log::trace!("make_inst_1: {:?}({:?})", operator, a); let root = root.resolve_inst(&pos.func.dfg).unwrap(); match operator { Opcode::AdjustSpDown => { let a = part_to_value(pos, root, a).unwrap(); pos.ins().adjust_sp_down(a).into() } Opcode::AdjustSpDownImm => { let c = a.unwrap_constant(); let imm = Imm64::try_from(c).unwrap(); pos.ins().adjust_sp_down_imm(imm).into() } Opcode::Bconst => { let c = a.unwrap_constant(); let val = const_to_value(pos.ins(), c, root); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Bint => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); let val = pos.ins().bint(ty, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Bnot => { let a = part_to_value(pos, root, a).unwrap(); let val = pos.ins().bnot(a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Brnz => { let a = part_to_value(pos, root, a).unwrap(); // NB: branching instructions must be the root of an // optimization's right-hand side, so we get the destination // block and arguments from the left-hand side's root. Peepmatic // doesn't currently represent labels or varargs. let block = pos.func.dfg[root].branch_destination().unwrap(); let args = pos.func.dfg.inst_args(root)[1..].to_vec(); pos.ins().brnz(a, block, &args).into() } Opcode::Brz => { let a = part_to_value(pos, root, a).unwrap(); // See the comment in the `Opcode::Brnz` match argm. let block = pos.func.dfg[root].branch_destination().unwrap(); let args = pos.func.dfg.inst_args(root)[1..].to_vec(); pos.ins().brz(a, block, &args).into() } Opcode::Iconst => { let a = a.unwrap_constant(); let val = const_to_value(pos.ins(), a, root); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Ireduce => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); let val = pos.ins().ireduce(ty, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Sextend => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); let val = pos.ins().sextend(ty, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Trapnz => { let a = part_to_value(pos, root, a).unwrap(); // NB: similar to branching instructions (see comment in the // `Opcode::Brnz` match arm) trapping instructions must be the // root of an optimization's right-hand side, and we get the // trap code from the root of the left-hand side. Peepmatic // doesn't currently represent trap codes. let code = pos.func.dfg[root].trap_code().unwrap(); pos.ins().trapnz(a, code).into() } Opcode::Trapz => { let a = part_to_value(pos, root, a).unwrap(); // See comment in the `Opcode::Trapnz` match arm. let code = pos.func.dfg[root].trap_code().unwrap(); pos.ins().trapz(a, code).into() } Opcode::Uextend => { let a = part_to_value(pos, root, a).unwrap(); let ty = peepmatic_ty_to_ir_ty(r#type, &pos.func.dfg, root); let val = pos.ins().uextend(ty, a); pos.func.dfg.value_def(val).unwrap_inst().into() } _ => unreachable!(), } } fn make_inst_2( &self, pos: &mut FuncCursor<'b>, root: ValueOrInst, operator: Opcode, _: Type, a: Part, b: Part, ) -> ValueOrInst { log::trace!("make_inst_2: {:?}({:?}, {:?})", operator, a, b); let root = root.resolve_inst(&pos.func.dfg).unwrap(); match operator { Opcode::Band => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().band(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::BandImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().band_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Bor => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().bor(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::BorImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().bor_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Bxor => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().bxor(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::BxorImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().bxor_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Iadd => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().iadd(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::IaddImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().iadd_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Ifcmp => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ifcmp(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::IfcmpImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ifcmp_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Imul => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().imul(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::ImulImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().imul_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::IrsubImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().irsub_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Ishl => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ishl(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::IshlImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ishl_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Isub => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().isub(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Rotl => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().rotl(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::RotlImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().rotl_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Rotr => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().rotr(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::RotrImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().rotr_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Sdiv => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().sdiv(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::SdivImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().sdiv_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Srem => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().srem(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::SremImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().srem_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Sshr => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().sshr(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::SshrImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().sshr_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Udiv => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().udiv(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::UdivImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().udiv_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Urem => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().urem(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::UremImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().urem_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Ushr => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ushr(a, b); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::UshrImm => { let a = part_to_imm64(pos, a); let b = part_to_value(pos, root, b).unwrap(); let val = pos.ins().ushr_imm(b, a); pos.func.dfg.value_def(val).unwrap_inst().into() } _ => unreachable!(), } } fn make_inst_3( &self, pos: &mut FuncCursor<'b>, root: ValueOrInst, operator: Opcode, _: Type, a: Part, b: Part, c: Part, ) -> ValueOrInst { log::trace!("make_inst_3: {:?}({:?}, {:?}, {:?})", operator, a, b, c); let root = root.resolve_inst(&pos.func.dfg).unwrap(); match operator { Opcode::Icmp => { let cond = a.unwrap_condition_code(); let cond = peepmatic_to_intcc(cond); let b = part_to_value(pos, root, b).unwrap(); let c = part_to_value(pos, root, c).unwrap(); let val = pos.ins().icmp(cond, b, c); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::IcmpImm => { let cond = a.unwrap_condition_code(); let cond = peepmatic_to_intcc(cond); let imm = part_to_imm64(pos, b); let c = part_to_value(pos, root, c).unwrap(); let val = pos.ins().icmp_imm(cond, c, imm); pos.func.dfg.value_def(val).unwrap_inst().into() } Opcode::Select => { let a = part_to_value(pos, root, a).unwrap(); let b = part_to_value(pos, root, b).unwrap(); let c = part_to_value(pos, root, c).unwrap(); let val = pos.ins().select(a, b, c); pos.func.dfg.value_def(val).unwrap_inst().into() } _ => unreachable!(), } } fn instruction_to_constant( &self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst, ) -> Option { value_or_inst.to_constant(pos) } fn instruction_result_bit_width( &self, pos: &mut FuncCursor<'b>, value_or_inst: ValueOrInst, ) -> u8 { value_or_inst.result_bit_width(&pos.func.dfg) } fn native_word_size_in_bits(&self, _pos: &mut FuncCursor<'b>) -> u8 { self.pointer_bits() } } #[cfg(test)] #[cfg(feature = "x86")] mod tests { use super::*; use crate::isa::lookup; use crate::settings::{builder, Flags}; use std::str::FromStr; use target_lexicon::triple; #[test] fn get_peepmatic_preopt() { let isa = lookup(triple!("x86_64")) .expect("expect x86 ISA") .finish(Flags::new(builder())); let _ = preopt(&*isa); } }