diff --git a/.rustfmt.toml b/.rustfmt.toml index 0efed58c53..cbd89f50b4 100644 --- a/.rustfmt.toml +++ b/.rustfmt.toml @@ -1,3 +1,4 @@ imports_granularity = "Crate" imports_layout = "HorizontalVertical" edition = "2021" +match_arm_leading_pipes = "Preserve" diff --git a/crates/wasmi/src/engine/bytecode/mod.rs b/crates/wasmi/src/engine/bytecode/mod.rs index 244670a41d..44a5232b2a 100644 --- a/crates/wasmi/src/engine/bytecode/mod.rs +++ b/crates/wasmi/src/engine/bytecode/mod.rs @@ -1,6 +1,7 @@ mod construct; mod immediate; mod utils; +mod visit_regs; #[cfg(test)] mod tests; @@ -30,6 +31,7 @@ pub use self::{ Sign, Table, }, + visit_regs::VisitRegs, }; use crate::{core::TrapCode, engine::EngineFunc, Error}; use core::num::{NonZeroI32, NonZeroI64, NonZeroU32, NonZeroU64}; diff --git a/crates/wasmi/src/engine/bytecode/visit_regs.rs b/crates/wasmi/src/engine/bytecode/visit_regs.rs new file mode 100644 index 0000000000..20bdc7b532 --- /dev/null +++ b/crates/wasmi/src/engine/bytecode/visit_regs.rs @@ -0,0 +1,620 @@ +use super::{Instruction, Reg, RegSpan, RegSpanIter}; + +impl Instruction { + /// Visit [`Reg`]s of `self` via the `visitor`. + pub fn visit_regs(&mut self, visitor: &mut V) { + HostVisitor::host_visitor(self, visitor) + } +} + +/// Implemented by [`Reg`] visitors to visit [`Reg`]s of an [`Instruction`] via [`Instruction::visit_regs`]. +pub trait VisitRegs { + /// Visits a [`Reg`] storing the result of an [`Instruction`]. + fn visit_result_reg(&mut self, reg: &mut Reg); + /// Visits a [`RegSpan`] storing the results of an [`Instruction`]. + fn visit_result_regs(&mut self, reg: &mut RegSpan, len: Option); + /// Visits a [`Reg`] storing an input of an [`Instruction`]. + fn visit_input_reg(&mut self, reg: &mut Reg); + /// Visits a [`RegSpan`] storing inputs of an [`Instruction`]. + fn visit_input_regs(&mut self, regs: &mut RegSpan, len: Option); +} + +/// Internal trait used to dispatch to a [`VisitRegs`] visitor. +trait HostVisitor { + /// Host the [`VisitRegs`] visitor in the appropriate way. + fn host_visitor(self, visitor: &mut V); +} + +impl HostVisitor for &'_ mut Reg { + fn host_visitor(self, visitor: &mut V) { + visitor.visit_input_reg(self); + } +} + +impl HostVisitor for &'_ mut [Reg; N] { + fn host_visitor(self, visitor: &mut V) { + for reg in self { + visitor.visit_input_reg(reg); + } + } +} + +impl HostVisitor for &'_ mut RegSpan { + fn host_visitor(self, visitor: &mut V) { + visitor.visit_input_regs(self, None); + } +} + +impl HostVisitor for &'_ mut RegSpanIter { + fn host_visitor(self, visitor: &mut V) { + let len = self.len_as_u16(); + let mut span = self.span(); + visitor.visit_input_regs(&mut span, Some(len)); + *self = span.iter(len); + } +} + +/// Type-wrapper to signal that the wrapped [`Reg`], [`RegSpan`] (etc.) is a result. +pub struct Res(T); + +impl HostVisitor for Res<&'_ mut Reg> { + fn host_visitor(self, visitor: &mut V) { + visitor.visit_result_reg(self.0); + } +} + +impl HostVisitor for Res<&'_ mut RegSpan> { + fn host_visitor(self, visitor: &mut V) { + visitor.visit_result_regs(self.0, None); + } +} + +macro_rules! host_visitor { + ( $visitor:expr => $($r:expr),* $(,)? ) => {{ + $( HostVisitor::host_visitor($r, $visitor) );* + }}; +} + +impl HostVisitor for &'_ mut Instruction { + fn host_visitor(self, visitor: &mut V) { + use Instruction as Instr; + match self { + | Instr::Trap { .. } | Instr::ConsumeFuel { .. } | Instr::Return => {} + | Instr::ReturnReg { value } => host_visitor!(visitor => value), + | Instr::ReturnReg2 { values } => host_visitor!(visitor => values), + | Instr::ReturnReg3 { values } => host_visitor!(visitor => values), + | Instr::ReturnImm32 { .. } + | Instr::ReturnI64Imm32 { .. } + | Instr::ReturnF64Imm32 { .. } => {} + | Instr::ReturnSpan { values } => host_visitor!(visitor => values), + | Instr::ReturnMany { values } => host_visitor!(visitor => values), + | Instr::ReturnNez { condition } => host_visitor!(visitor => condition), + | Instr::ReturnNezReg { condition, value } => { + host_visitor!(visitor => condition, value) + } + | Instr::ReturnNezReg2 { condition, values } => { + host_visitor!(visitor => condition, values) + } + | Instr::ReturnNezImm32 { condition, .. } + | Instr::ReturnNezI64Imm32 { condition, .. } + | Instr::ReturnNezF64Imm32 { condition, .. } => host_visitor!(visitor => condition), + | Instr::ReturnNezSpan { condition, values } => { + host_visitor!(visitor => condition, values) + } + | Instr::ReturnNezMany { condition, values } => { + host_visitor!(visitor => condition, values) + } + | Instr::Branch { .. } => {} + | Instr::BranchCmpFallback { lhs, rhs, params } => { + host_visitor!(visitor => lhs, rhs, params) + } + | Instr::BranchI32And { lhs, rhs, .. } + | Instr::BranchI32Or { lhs, rhs, .. } + | Instr::BranchI32Xor { lhs, rhs, .. } + | Instr::BranchI32AndEqz { lhs, rhs, .. } + | Instr::BranchI32OrEqz { lhs, rhs, .. } + | Instr::BranchI32XorEqz { lhs, rhs, .. } + | Instr::BranchI32Eq { lhs, rhs, .. } + | Instr::BranchI32Ne { lhs, rhs, .. } + | Instr::BranchI32LtS { lhs, rhs, .. } + | Instr::BranchI32LtU { lhs, rhs, .. } + | Instr::BranchI32LeS { lhs, rhs, .. } + | Instr::BranchI32LeU { lhs, rhs, .. } + | Instr::BranchI32GtS { lhs, rhs, .. } + | Instr::BranchI32GtU { lhs, rhs, .. } + | Instr::BranchI32GeS { lhs, rhs, .. } + | Instr::BranchI32GeU { lhs, rhs, .. } + | Instr::BranchI64Eq { lhs, rhs, .. } + | Instr::BranchI64Ne { lhs, rhs, .. } + | Instr::BranchI64LtS { lhs, rhs, .. } + | Instr::BranchI64LtU { lhs, rhs, .. } + | Instr::BranchI64LeS { lhs, rhs, .. } + | Instr::BranchI64LeU { lhs, rhs, .. } + | Instr::BranchI64GtS { lhs, rhs, .. } + | Instr::BranchI64GtU { lhs, rhs, .. } + | Instr::BranchI64GeS { lhs, rhs, .. } + | Instr::BranchI64GeU { lhs, rhs, .. } + | Instr::BranchF32Eq { lhs, rhs, .. } + | Instr::BranchF32Ne { lhs, rhs, .. } + | Instr::BranchF32Lt { lhs, rhs, .. } + | Instr::BranchF32Le { lhs, rhs, .. } + | Instr::BranchF32Gt { lhs, rhs, .. } + | Instr::BranchF32Ge { lhs, rhs, .. } + | Instr::BranchF64Eq { lhs, rhs, .. } + | Instr::BranchF64Ne { lhs, rhs, .. } + | Instr::BranchF64Lt { lhs, rhs, .. } + | Instr::BranchF64Le { lhs, rhs, .. } + | Instr::BranchF64Gt { lhs, rhs, .. } + | Instr::BranchF64Ge { lhs, rhs, .. } => host_visitor!(visitor => lhs, rhs), + | Instr::BranchI32AndImm { lhs, .. } + | Instr::BranchI32OrImm { lhs, .. } + | Instr::BranchI32XorImm { lhs, .. } + | Instr::BranchI32AndEqzImm { lhs, .. } + | Instr::BranchI32OrEqzImm { lhs, .. } + | Instr::BranchI32XorEqzImm { lhs, .. } + | Instr::BranchI32EqImm { lhs, .. } + | Instr::BranchI32NeImm { lhs, .. } + | Instr::BranchI32LtSImm { lhs, .. } + | Instr::BranchI32LtUImm { lhs, .. } + | Instr::BranchI32LeSImm { lhs, .. } + | Instr::BranchI32LeUImm { lhs, .. } + | Instr::BranchI32GtSImm { lhs, .. } + | Instr::BranchI32GtUImm { lhs, .. } + | Instr::BranchI32GeSImm { lhs, .. } + | Instr::BranchI32GeUImm { lhs, .. } + | Instr::BranchI64EqImm { lhs, .. } + | Instr::BranchI64NeImm { lhs, .. } + | Instr::BranchI64LtSImm { lhs, .. } + | Instr::BranchI64LtUImm { lhs, .. } + | Instr::BranchI64LeSImm { lhs, .. } + | Instr::BranchI64LeUImm { lhs, .. } + | Instr::BranchI64GtSImm { lhs, .. } + | Instr::BranchI64GtUImm { lhs, .. } + | Instr::BranchI64GeSImm { lhs, .. } + | Instr::BranchI64GeUImm { lhs, .. } => host_visitor!(visitor => lhs), + | Instr::BranchTable0 { index, .. } + | Instr::BranchTable1 { index, .. } + | Instr::BranchTable2 { index, .. } + | Instr::BranchTable3 { index, .. } + | Instr::BranchTableSpan { index, .. } + | Instr::BranchTableMany { index, .. } => host_visitor!(visitor => index), + | Instr::Copy { result, value } => host_visitor!(visitor => Res(result), value), + | Instr::Copy2 { results, values } => host_visitor!(visitor => Res(results), values), + | Instr::CopyImm32 { result, .. } + | Instr::CopyI64Imm32 { result, .. } + | Instr::CopyF64Imm32 { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::CopySpan { + results, values, .. + } + | Instr::CopySpanNonOverlapping { + results, values, .. + } => host_visitor!(visitor => Res(results), values), + | Instr::CopyMany { results, values } + | Instr::CopyManyNonOverlapping { results, values } => { + host_visitor!(visitor => Res(results), values) + } + | Instr::ReturnCallInternal0 { .. } + | Instr::ReturnCallInternal { .. } + | Instr::ReturnCallImported0 { .. } + | Instr::ReturnCallImported { .. } + | Instr::ReturnCallIndirect0 { .. } + | Instr::ReturnCallIndirect0Imm16 { .. } + | Instr::ReturnCallIndirect { .. } + | Instr::ReturnCallIndirectImm16 { .. } => {} + | Instr::CallInternal0 { results, .. } + | Instr::CallInternal { results, .. } + | Instr::CallImported0 { results, .. } + | Instr::CallImported { results, .. } + | Instr::CallIndirect0 { results, .. } + | Instr::CallIndirect0Imm16 { results, .. } + | Instr::CallIndirect { results, .. } + | Instr::CallIndirectImm16 { results, .. } => host_visitor!(visitor => Res(results)), + | Instr::Select { result, lhs } | Instr::SelectImm32Rhs { result, lhs } => { + host_visitor!(visitor => Res(result), lhs) + } + | Instr::SelectImm32Lhs { result, .. } | Instr::SelectImm32 { result, .. } => { + host_visitor!(visitor => Res(result)) + } + | Instr::SelectI64Imm32Rhs { result, lhs } => { + host_visitor!(visitor => Res(result), lhs) + } + | Instr::SelectI64Imm32Lhs { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::SelectI64Imm32 { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::SelectF64Imm32Rhs { result, lhs } => { + host_visitor!(visitor => Res(result), lhs) + } + | Instr::SelectF64Imm32Lhs { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::SelectF64Imm32 { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::RefFunc { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::GlobalGet { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::GlobalSet { input, .. } => host_visitor!(visitor => input), + | Instr::GlobalSetI32Imm16 { .. } | Instr::GlobalSetI64Imm16 { .. } => {} + | Instr::I32Load { result, ptr } + | Instr::I64Load { result, ptr } + | Instr::F32Load { result, ptr } + | Instr::F64Load { result, ptr } + | Instr::I32Load8s { result, ptr } + | Instr::I32Load8u { result, ptr } + | Instr::I32Load16s { result, ptr } + | Instr::I32Load16u { result, ptr } + | Instr::I64Load8s { result, ptr } + | Instr::I64Load8u { result, ptr } + | Instr::I64Load16s { result, ptr } + | Instr::I64Load16u { result, ptr } + | Instr::I64Load32s { result, ptr } + | Instr::I64Load32u { result, ptr } => host_visitor!(visitor => Res(result), ptr), + | Instr::I32LoadAt { result, .. } + | Instr::I64LoadAt { result, .. } + | Instr::F32LoadAt { result, .. } + | Instr::F64LoadAt { result, .. } + | Instr::I32Load8sAt { result, .. } + | Instr::I32Load8uAt { result, .. } + | Instr::I32Load16sAt { result, .. } + | Instr::I32Load16uAt { result, .. } + | Instr::I64Load8sAt { result, .. } + | Instr::I64Load8uAt { result, .. } + | Instr::I64Load16sAt { result, .. } + | Instr::I64Load16uAt { result, .. } + | Instr::I64Load32sAt { result, .. } + | Instr::I64Load32uAt { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::I32LoadOffset16 { result, ptr, .. } + | Instr::I64LoadOffset16 { result, ptr, .. } + | Instr::F32LoadOffset16 { result, ptr, .. } + | Instr::F64LoadOffset16 { result, ptr, .. } + | Instr::I32Load8sOffset16 { result, ptr, .. } + | Instr::I32Load8uOffset16 { result, ptr, .. } + | Instr::I32Load16sOffset16 { result, ptr, .. } + | Instr::I32Load16uOffset16 { result, ptr, .. } + | Instr::I64Load8sOffset16 { result, ptr, .. } + | Instr::I64Load8uOffset16 { result, ptr, .. } + | Instr::I64Load16sOffset16 { result, ptr, .. } + | Instr::I64Load16uOffset16 { result, ptr, .. } + | Instr::I64Load32sOffset16 { result, ptr, .. } + | Instr::I64Load32uOffset16 { result, ptr, .. } => { + host_visitor!(visitor => Res(result), ptr) + } + | Instr::I32Store { ptr, .. } + | Instr::I32Store8 { ptr, .. } + | Instr::I32Store16 { ptr, .. } + | Instr::I64Store { ptr, .. } + | Instr::I64Store8 { ptr, .. } + | Instr::I64Store16 { ptr, .. } + | Instr::I64Store32 { ptr, .. } + | Instr::F32Store { ptr, .. } + | Instr::F64Store { ptr, .. } => host_visitor!(visitor => ptr), + | Instr::I32StoreOffset16 { ptr, value, .. } + | Instr::I32Store8Offset16 { ptr, value, .. } + | Instr::I32Store16Offset16 { ptr, value, .. } + | Instr::I64StoreOffset16 { ptr, value, .. } + | Instr::I64Store8Offset16 { ptr, value, .. } + | Instr::I64Store16Offset16 { ptr, value, .. } + | Instr::I64Store32Offset16 { ptr, value, .. } + | Instr::F32StoreOffset16 { ptr, value, .. } + | Instr::F64StoreOffset16 { ptr, value, .. } => host_visitor!(visitor => ptr, value), + | Instr::I32StoreAt { value, .. } + | Instr::I32Store8At { value, .. } + | Instr::I32Store16At { value, .. } + | Instr::I64StoreAt { value, .. } + | Instr::I64Store8At { value, .. } + | Instr::I64Store16At { value, .. } + | Instr::I64Store32At { value, .. } + | Instr::F32StoreAt { value, .. } + | Instr::F64StoreAt { value, .. } => host_visitor!(visitor => value), + | Instr::I32StoreOffset16Imm16 { ptr, .. } + | Instr::I32Store8Offset16Imm { ptr, .. } + | Instr::I32Store16Offset16Imm { ptr, .. } + | Instr::I64StoreOffset16Imm16 { ptr, .. } + | Instr::I64Store8Offset16Imm { ptr, .. } + | Instr::I64Store16Offset16Imm { ptr, .. } + | Instr::I64Store32Offset16Imm16 { ptr, .. } => host_visitor!(visitor => ptr), + | Instr::I32StoreAtImm16 { .. } + | Instr::I32Store8AtImm { .. } + | Instr::I32Store16AtImm { .. } + | Instr::I64StoreAtImm16 { .. } + | Instr::I64Store8AtImm { .. } + | Instr::I64Store16AtImm { .. } + | Instr::I64Store32AtImm16 { .. } => {} + | Instr::I32Eq { result, lhs, rhs } + | Instr::I32Ne { result, lhs, rhs } + | Instr::I32LtS { result, lhs, rhs } + | Instr::I32LtU { result, lhs, rhs } + | Instr::I32GtS { result, lhs, rhs } + | Instr::I32GtU { result, lhs, rhs } + | Instr::I32LeS { result, lhs, rhs } + | Instr::I32LeU { result, lhs, rhs } + | Instr::I32GeS { result, lhs, rhs } + | Instr::I32GeU { result, lhs, rhs } + | Instr::I64Eq { result, lhs, rhs } + | Instr::I64Ne { result, lhs, rhs } + | Instr::I64LtS { result, lhs, rhs } + | Instr::I64LtU { result, lhs, rhs } + | Instr::I64GtS { result, lhs, rhs } + | Instr::I64GtU { result, lhs, rhs } + | Instr::I64LeS { result, lhs, rhs } + | Instr::I64LeU { result, lhs, rhs } + | Instr::I64GeS { result, lhs, rhs } + | Instr::I64GeU { result, lhs, rhs } + | Instr::F32Eq { result, lhs, rhs } + | Instr::F32Ne { result, lhs, rhs } + | Instr::F32Lt { result, lhs, rhs } + | Instr::F32Le { result, lhs, rhs } + | Instr::F32Gt { result, lhs, rhs } + | Instr::F32Ge { result, lhs, rhs } + | Instr::F64Eq { result, lhs, rhs } + | Instr::F64Ne { result, lhs, rhs } + | Instr::F64Lt { result, lhs, rhs } + | Instr::F64Le { result, lhs, rhs } + | Instr::F64Gt { result, lhs, rhs } + | Instr::F64Ge { result, lhs, rhs } => host_visitor!(visitor => Res(result), lhs, rhs), + | Instr::I32EqImm16 { result, lhs, .. } + | Instr::I32NeImm16 { result, lhs, .. } + | Instr::I32LtSImm16 { result, lhs, .. } + | Instr::I32LtUImm16 { result, lhs, .. } + | Instr::I32GtSImm16 { result, lhs, .. } + | Instr::I32GtUImm16 { result, lhs, .. } + | Instr::I32LeSImm16 { result, lhs, .. } + | Instr::I32LeUImm16 { result, lhs, .. } + | Instr::I32GeSImm16 { result, lhs, .. } + | Instr::I32GeUImm16 { result, lhs, .. } + | Instr::I64EqImm16 { result, lhs, .. } + | Instr::I64NeImm16 { result, lhs, .. } + | Instr::I64LtSImm16 { result, lhs, .. } + | Instr::I64LtUImm16 { result, lhs, .. } + | Instr::I64GtSImm16 { result, lhs, .. } + | Instr::I64GtUImm16 { result, lhs, .. } + | Instr::I64LeSImm16 { result, lhs, .. } + | Instr::I64LeUImm16 { result, lhs, .. } + | Instr::I64GeSImm16 { result, lhs, .. } + | Instr::I64GeUImm16 { result, lhs, .. } => host_visitor!(visitor => Res(result), lhs), + | Instr::I32Clz { result, input } + | Instr::I32Ctz { result, input } + | Instr::I32Popcnt { result, input } => host_visitor!(visitor => Res(result), input), + | Instr::I32Add { result, lhs, rhs } + | Instr::I32Sub { result, lhs, rhs } + | Instr::I32Mul { result, lhs, rhs } + | Instr::I32DivS { result, lhs, rhs } + | Instr::I32DivU { result, lhs, rhs } + | Instr::I32RemS { result, lhs, rhs } + | Instr::I32RemU { result, lhs, rhs } + | Instr::I32And { result, lhs, rhs } + | Instr::I32AndEqz { result, lhs, rhs } + | Instr::I32Or { result, lhs, rhs } + | Instr::I32OrEqz { result, lhs, rhs } + | Instr::I32Xor { result, lhs, rhs } + | Instr::I32XorEqz { result, lhs, rhs } + | Instr::I32Shl { result, lhs, rhs } + | Instr::I32ShrU { result, lhs, rhs } + | Instr::I32ShrS { result, lhs, rhs } + | Instr::I32Rotl { result, lhs, rhs } + | Instr::I32Rotr { result, lhs, rhs } => { + host_visitor!(visitor => Res(result), lhs, rhs) + } + | Instr::I32AddImm16 { result, lhs, .. } + | Instr::I32MulImm16 { result, lhs, .. } + | Instr::I32DivSImm16 { result, lhs, .. } + | Instr::I32DivUImm16 { result, lhs, .. } + | Instr::I32RemSImm16 { result, lhs, .. } + | Instr::I32RemUImm16 { result, lhs, .. } + | Instr::I32AndEqzImm16 { result, lhs, .. } + | Instr::I32AndImm16 { result, lhs, .. } + | Instr::I32OrEqzImm16 { result, lhs, .. } + | Instr::I32OrImm16 { result, lhs, .. } + | Instr::I32XorEqzImm16 { result, lhs, .. } + | Instr::I32XorImm16 { result, lhs, .. } + | Instr::I32ShlImm { result, lhs, .. } + | Instr::I32ShrUImm { result, lhs, .. } + | Instr::I32ShrSImm { result, lhs, .. } + | Instr::I32RotlImm { result, lhs, .. } + | Instr::I32RotrImm { result, lhs, .. } => host_visitor!(visitor => Res(result), lhs), + | Instr::I32SubImm16Rev { result, rhs, .. } + | Instr::I32DivSImm16Rev { result, rhs, .. } + | Instr::I32DivUImm16Rev { result, rhs, .. } + | Instr::I32RemSImm16Rev { result, rhs, .. } + | Instr::I32RemUImm16Rev { result, rhs, .. } + | Instr::I32ShlImm16Rev { result, rhs, .. } + | Instr::I32ShrUImm16Rev { result, rhs, .. } + | Instr::I32ShrSImm16Rev { result, rhs, .. } + | Instr::I32RotlImm16Rev { result, rhs, .. } + | Instr::I32RotrImm16Rev { result, rhs, .. } => { + host_visitor!(visitor => Res(result), rhs) + } + | Instr::I64Clz { result, input } + | Instr::I64Ctz { result, input } + | Instr::I64Popcnt { result, input } => host_visitor!(visitor => Res(result), input), + | Instr::I64Add { result, lhs, rhs } + | Instr::I64Sub { result, lhs, rhs } + | Instr::I64Mul { result, lhs, rhs } + | Instr::I64DivS { result, lhs, rhs } + | Instr::I64DivU { result, lhs, rhs } + | Instr::I64RemS { result, lhs, rhs } + | Instr::I64RemU { result, lhs, rhs } + | Instr::I64And { result, lhs, rhs } + | Instr::I64Or { result, lhs, rhs } + | Instr::I64Xor { result, lhs, rhs } + | Instr::I64Shl { result, lhs, rhs } + | Instr::I64ShrU { result, lhs, rhs } + | Instr::I64ShrS { result, lhs, rhs } + | Instr::I64Rotl { result, lhs, rhs } + | Instr::I64Rotr { result, lhs, rhs } => { + host_visitor!(visitor => Res(result), lhs, rhs) + } + | Instr::I64AddImm16 { result, lhs, .. } + | Instr::I64MulImm16 { result, lhs, .. } + | Instr::I64DivSImm16 { result, lhs, .. } + | Instr::I64DivUImm16 { result, lhs, .. } + | Instr::I64RemSImm16 { result, lhs, .. } + | Instr::I64RemUImm16 { result, lhs, .. } + | Instr::I64AndImm16 { result, lhs, .. } + | Instr::I64OrImm16 { result, lhs, .. } + | Instr::I64XorImm16 { result, lhs, .. } + | Instr::I64ShlImm { result, lhs, .. } + | Instr::I64ShrUImm { result, lhs, .. } + | Instr::I64ShrSImm { result, lhs, .. } + | Instr::I64RotlImm { result, lhs, .. } + | Instr::I64RotrImm { result, lhs, .. } => host_visitor!(visitor => Res(result), lhs), + | Instr::I64SubImm16Rev { result, rhs, .. } + | Instr::I64DivSImm16Rev { result, rhs, .. } + | Instr::I64DivUImm16Rev { result, rhs, .. } + | Instr::I64RemSImm16Rev { result, rhs, .. } + | Instr::I64RemUImm16Rev { result, rhs, .. } + | Instr::I64ShlImm16Rev { result, rhs, .. } + | Instr::I64ShrUImm16Rev { result, rhs, .. } + | Instr::I64ShrSImm16Rev { result, rhs, .. } + | Instr::I64RotlImm16Rev { result, rhs, .. } + | Instr::I64RotrImm16Rev { result, rhs, .. } => { + host_visitor!(visitor => Res(result), rhs) + } + | Instr::I32WrapI64 { result, input } + | Instr::I32Extend8S { result, input } + | Instr::I32Extend16S { result, input } + | Instr::I64Extend8S { result, input } + | Instr::I64Extend16S { result, input } + | Instr::I64Extend32S { result, input } + | Instr::F32Abs { result, input } + | Instr::F32Neg { result, input } + | Instr::F32Ceil { result, input } + | Instr::F32Floor { result, input } + | Instr::F32Trunc { result, input } + | Instr::F32Nearest { result, input } + | Instr::F32Sqrt { result, input } => host_visitor!(visitor => Res(result), input), + | Instr::F32Add { result, lhs, rhs } + | Instr::F32Sub { result, lhs, rhs } + | Instr::F32Mul { result, lhs, rhs } + | Instr::F32Div { result, lhs, rhs } + | Instr::F32Min { result, lhs, rhs } + | Instr::F32Max { result, lhs, rhs } + | Instr::F32Copysign { result, lhs, rhs } => { + host_visitor!(visitor => Res(result), lhs, rhs) + } + | Instr::F32CopysignImm { result, lhs, .. } => { + host_visitor!(visitor => Res(result), lhs) + } + | Instr::F64Abs { result, input } + | Instr::F64Neg { result, input } + | Instr::F64Ceil { result, input } + | Instr::F64Floor { result, input } + | Instr::F64Trunc { result, input } + | Instr::F64Nearest { result, input } + | Instr::F64Sqrt { result, input } => host_visitor!(visitor => Res(result), input), + | Instr::F64Add { result, lhs, rhs } + | Instr::F64Sub { result, lhs, rhs } + | Instr::F64Mul { result, lhs, rhs } + | Instr::F64Div { result, lhs, rhs } + | Instr::F64Min { result, lhs, rhs } + | Instr::F64Max { result, lhs, rhs } + | Instr::F64Copysign { result, lhs, rhs } => { + host_visitor!(visitor => Res(result), lhs, rhs) + } + | Instr::F64CopysignImm { result, lhs, .. } => { + host_visitor!(visitor => Res(result), lhs) + } + | Instr::I32TruncF32S { result, input } + | Instr::I32TruncF32U { result, input } + | Instr::I32TruncF64S { result, input } + | Instr::I32TruncF64U { result, input } + | Instr::I64TruncF32S { result, input } + | Instr::I64TruncF32U { result, input } + | Instr::I64TruncF64S { result, input } + | Instr::I64TruncF64U { result, input } + | Instr::I32TruncSatF32S { result, input } + | Instr::I32TruncSatF32U { result, input } + | Instr::I32TruncSatF64S { result, input } + | Instr::I32TruncSatF64U { result, input } + | Instr::I64TruncSatF32S { result, input } + | Instr::I64TruncSatF32U { result, input } + | Instr::I64TruncSatF64S { result, input } + | Instr::I64TruncSatF64U { result, input } + | Instr::F32DemoteF64 { result, input } + | Instr::F64PromoteF32 { result, input } + | Instr::F32ConvertI32S { result, input } + | Instr::F32ConvertI32U { result, input } + | Instr::F32ConvertI64S { result, input } + | Instr::F32ConvertI64U { result, input } + | Instr::F64ConvertI32S { result, input } + | Instr::F64ConvertI32U { result, input } + | Instr::F64ConvertI64S { result, input } + | Instr::F64ConvertI64U { result, input } => { + host_visitor!(visitor => Res(result), input) + } + | Instr::TableGet { result, index } => host_visitor!(visitor => Res(result), index), + | Instr::TableGetImm { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::TableSize { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::TableSet { index, value } => host_visitor!(visitor => index, value), + | Instr::TableSetAt { value, .. } => host_visitor!(visitor => value), + | Instr::TableCopy { dst, src, len } => host_visitor!(visitor => dst, src, len), + | Instr::TableCopyTo { src, len, .. } => host_visitor!(visitor => src, len), + | Instr::TableCopyFrom { dst, len, .. } => host_visitor!(visitor => dst, len), + | Instr::TableCopyFromTo { len, .. } => host_visitor!(visitor => len), + | Instr::TableCopyExact { dst, src, .. } => host_visitor!(visitor => dst, src), + | Instr::TableCopyToExact { src, .. } => host_visitor!(visitor => src), + | Instr::TableCopyFromExact { dst, .. } => host_visitor!(visitor => dst), + | Instr::TableCopyFromToExact { .. } => {} + | Instr::TableInit { dst, src, len } => host_visitor!(visitor => dst, src, len), + | Instr::TableInitTo { src, len, .. } => host_visitor!(visitor => src, len), + | Instr::TableInitFrom { dst, len, .. } => host_visitor!(visitor => dst, len), + | Instr::TableInitFromTo { len, .. } => host_visitor!(visitor => len), + | Instr::TableInitExact { dst, src, .. } => host_visitor!(visitor => dst, src), + | Instr::TableInitToExact { src, .. } => host_visitor!(visitor => src), + | Instr::TableInitFromExact { dst, .. } => host_visitor!(visitor => dst), + | Instr::TableInitFromToExact { .. } => {} + | Instr::TableFill { dst, len, value } => host_visitor!(visitor => dst, len, value), + | Instr::TableFillAt { len, value, .. } => host_visitor!(visitor => len, value), + | Instr::TableFillExact { dst, value, .. } => host_visitor!(visitor => dst, value), + | Instr::TableFillAtExact { value, .. } => host_visitor!(visitor => value), + | Instr::TableGrow { + result, + delta, + value, + } => host_visitor!(visitor => Res(result), delta, value), + | Instr::TableGrowImm { result, value, .. } => { + host_visitor!(visitor => Res(result), value) + } + | Instr::ElemDrop(_) | Instr::DataDrop(_) => {} + | Instr::MemorySize { result } => host_visitor!(visitor => Res(result)), + | Instr::MemoryGrow { result, delta } => host_visitor!(visitor => Res(result), delta), + | Instr::MemoryGrowBy { result, .. } => host_visitor!(visitor => Res(result)), + | Instr::MemoryCopy { dst, src, len } => host_visitor!(visitor => dst, src, len), + | Instr::MemoryCopyTo { src, len, .. } => host_visitor!(visitor => src, len), + | Instr::MemoryCopyFrom { dst, len, .. } => host_visitor!(visitor => dst, len), + | Instr::MemoryCopyFromTo { len, .. } => host_visitor!(visitor => len), + | Instr::MemoryCopyExact { dst, src, .. } => host_visitor!(visitor => dst, src), + | Instr::MemoryCopyToExact { src, .. } => host_visitor!(visitor => src), + | Instr::MemoryCopyFromExact { dst, .. } => host_visitor!(visitor => dst), + | Instr::MemoryCopyFromToExact { .. } => {} + | Instr::MemoryFill { dst, value, len } => host_visitor!(visitor => dst, value, len), + | Instr::MemoryFillAt { value, len, .. } => host_visitor!(visitor => value, len), + | Instr::MemoryFillImm { dst, len, .. } => host_visitor!(visitor => dst, len), + | Instr::MemoryFillExact { dst, value, .. } => host_visitor!(visitor => dst, value), + | Instr::MemoryFillAtImm { len, .. } => host_visitor!(visitor => len), + | Instr::MemoryFillAtExact { value, .. } => host_visitor!(visitor => value), + | Instr::MemoryFillImmExact { dst, .. } => host_visitor!(visitor => dst), + | Instr::MemoryFillAtImmExact { .. } => {} + | Instr::MemoryInit { dst, src, len } => host_visitor!(visitor => dst, src, len), + | Instr::MemoryInitTo { src, len, .. } => host_visitor!(visitor => src, len), + | Instr::MemoryInitFrom { dst, len, .. } => host_visitor!(visitor => dst, len), + | Instr::MemoryInitFromTo { len, .. } => host_visitor!(visitor => len), + | Instr::MemoryInitExact { dst, src, .. } => host_visitor!(visitor => dst, src), + | Instr::MemoryInitToExact { src, .. } => host_visitor!(visitor => src), + | Instr::MemoryInitFromExact { dst, .. } => host_visitor!(visitor => dst), + | Instr::MemoryInitFromToExact { .. } => {} + | Instr::TableIndex { .. } + | Instr::DataIndex { .. } + | Instr::ElemIndex { .. } + | Instr::Const32 { .. } + | Instr::I64Const32 { .. } + | Instr::F64Const32 { .. } => {} + | Instr::BranchTableTarget { results, .. } + | Instr::BranchTableTargetNonOverlapping { results, .. } => { + host_visitor!(visitor => results) + } + | Instr::RegisterAndImm32 { reg, .. } => host_visitor!(visitor => reg), + | Instr::RegisterSpan { span } => host_visitor!(visitor => span), + | Instr::Register { reg } => host_visitor!(visitor => reg), + | Instr::Register2 { regs } => host_visitor!(visitor => regs), + | Instr::Register3 { regs } => host_visitor!(visitor => regs), + | Instr::RegisterList { regs } => host_visitor!(visitor => regs), + | Instr::CallIndirectParams { index, .. } => host_visitor!(visitor => index), + | Instr::CallIndirectParamsImm16 { .. } => {} + } + } +} diff --git a/crates/wasmi/src/engine/translator/visit_register.rs b/crates/wasmi/src/engine/translator/visit_register.rs index 8d3c802512..2d031e1ec8 100644 --- a/crates/wasmi/src/engine/translator/visit_register.rs +++ b/crates/wasmi/src/engine/translator/visit_register.rs @@ -1,583 +1,47 @@ -use crate::engine::bytecode::{Instruction, Reg, RegSpan, RegSpanIter}; +use crate::engine::bytecode::{Instruction, Reg, RegSpan, VisitRegs}; -macro_rules! visit_registers { - ( $f:expr, $($field:expr),* $(,)? ) => {{ - $( - $f($field) - );* - }}; -} - -/// Trait implemented by types that allow to visit their [`Reg`] fields. +/// Extension-trait for [`Instruction`] to only visit certain [`Reg`]s via closure. pub trait VisitInputRegisters { /// Calls `f` on all input [`Reg`]. fn visit_input_registers(&mut self, f: impl FnMut(&mut Reg)); } -impl VisitInputRegisters for Instruction { - #[rustfmt::skip] - fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Reg)) { - match self { - Instruction::TableIndex { .. } | - Instruction::DataIndex { .. } | - Instruction::ElemIndex { .. } | - Instruction::Const32 { .. } | - Instruction::I64Const32 { .. } | - Instruction::F64Const32 { .. } => {}, - Instruction::RegisterAndImm32 { reg, .. } => f(reg), - Instruction::Register { reg } => f(reg), - Instruction::Register2 { regs } => regs.visit_input_registers(f), - Instruction::Register3 { regs } | - Instruction::RegisterList { regs } => regs.visit_input_registers(f), - Instruction::RegisterSpan { span } => span.visit_input_registers(f), - Instruction::BranchTableTarget { .. } | - Instruction::BranchTableTargetNonOverlapping { .. } => {}, - Instruction::Trap { .. } | - Instruction::ConsumeFuel { .. } | - Instruction::Return => {}, - Instruction::ReturnReg { value } => f(value), - Instruction::ReturnReg2 { values } => values.visit_input_registers(f), - Instruction::ReturnReg3 { values } => values.visit_input_registers(f), - Instruction::ReturnImm32 { .. } | - Instruction::ReturnI64Imm32 { .. } | - Instruction::ReturnF64Imm32 { .. } => {}, - Instruction::ReturnSpan { values } => { - values.visit_input_registers(f); - } - Instruction::ReturnMany { values } => { - values.visit_input_registers(f); - } - Instruction::ReturnNez { condition } => f(condition), - Instruction::ReturnNezReg { condition, value } => visit_registers!(f, condition, value), - Instruction::ReturnNezReg2 { condition, values } => { - f(condition); - values.visit_input_registers(f); - } - Instruction::ReturnNezImm32 { condition, .. } => f(condition), - Instruction::ReturnNezI64Imm32 { condition, .. } => f(condition), - Instruction::ReturnNezF64Imm32 { condition, .. } => f(condition), - Instruction::ReturnNezSpan { condition, values } => { - f(condition); - values.visit_input_registers(f); - } - Instruction::ReturnNezMany { condition, values } => { - f(condition); - values.visit_input_registers(f); - } - Instruction::Branch { .. } => {}, - Instruction::BranchTable0 { index, .. } | - Instruction::BranchTable1 { index, .. } | - Instruction::BranchTable2 { index, .. } | - Instruction::BranchTable3 { index, .. } | - Instruction::BranchTableSpan { index, .. } | - Instruction::BranchTableMany { index, .. } => f(index), +/// A [`Reg`] visitor. +pub struct Visitor { + f: F, +} - Instruction::BranchCmpFallback { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32And { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32AndImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32Or { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32OrImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32Xor { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32XorImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32AndEqz { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32AndEqzImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32OrEqz { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32OrEqzImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32XorEqz { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32XorEqzImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32Eq { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32EqImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32Ne { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32NeImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32LtS { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32LtSImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32LtU { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32LtUImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32LeS { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32LeSImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32LeU { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32LeUImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32GtS { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32GtSImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32GtU { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32GtUImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32GeS { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32GeSImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI32GeU { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI32GeUImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64Eq { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64EqImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64Ne { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64NeImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64LtS { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64LtSImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64LtU { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64LtUImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64LeS { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64LeSImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64LeU { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64LeUImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64GtS { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64GtSImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64GtU { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64GtUImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64GeS { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64GeSImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchI64GeU { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::BranchI64GeUImm { lhs, .. } => visit_registers!(f, lhs), - Instruction::BranchF32Eq { lhs, rhs, .. } | - Instruction::BranchF32Ne { lhs, rhs, .. } | - Instruction::BranchF32Lt { lhs, rhs, .. } | - Instruction::BranchF32Le { lhs, rhs, .. } | - Instruction::BranchF32Gt { lhs, rhs, .. } | - Instruction::BranchF32Ge { lhs, rhs, .. } | - Instruction::BranchF64Eq { lhs, rhs, .. } | - Instruction::BranchF64Ne { lhs, rhs, .. } | - Instruction::BranchF64Lt { lhs, rhs, .. } | - Instruction::BranchF64Le { lhs, rhs, .. } | - Instruction::BranchF64Gt { lhs, rhs, .. } | - Instruction::BranchF64Ge { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), +impl VisitRegs for Visitor { + #[inline(always)] + fn visit_result_reg(&mut self, _reg: &mut Reg) {} - Instruction::Copy { result, value } => { - // Note: for copy instructions unlike all other instructions - // we need to also visit the result register since - // encoding of `local.set` or `local.tee` might actually - // result in a `copy` instruction with a `result` register - // allocated in the storage-space. - visit_registers!(f, result, value) - } - Instruction::Copy2 { results, values } => { - // Note: we need to visit the results of the `Copy2` instruction - // as well since it might have been generated while preserving locals - // on the compilation stack when entering a control flow `block` - // or `if`. - f(results.head_mut()); - values.visit_input_registers(f); - } - Instruction::CopyImm32 { result: _, value: _ } | - Instruction::CopyI64Imm32 { result: _, value: _ } | - Instruction::CopyF64Imm32 { result: _, value: _ } => {}, - Instruction::CopySpan { results: _, values, len: _ } => { - values.visit_input_registers(f); - } - Instruction::CopySpanNonOverlapping { results, values, len: _ } => { - // Note: we need to visit the results of the `CopySpanNonOverlapping` instruction - // as well since it might have been generated while preserving locals - // on the compilation stack when entering a control flow `block` - // or `if`. - f(results.head_mut()); - values.visit_input_registers(f); - } - Instruction::CopyMany { results: _, values } => { - values.visit_input_registers(f); - } - Instruction::CopyManyNonOverlapping { results, values } => { - // Note: we need to visit the results of the `CopyManyNonOverlapping` instruction - // as well since it might have been generated while preserving locals - // on the compilation stack when entering a control flow `block` - // or `if`. - f(results.head_mut()); - values.visit_input_registers(f); - } - Instruction::CallIndirectParams { index, .. } => f(index), - Instruction::CallIndirectParamsImm16 { .. } => {}, - Instruction::ReturnCallInternal0 { .. } | - Instruction::ReturnCallInternal { .. } | - Instruction::ReturnCallImported0 { .. } | - Instruction::ReturnCallImported { .. } | - Instruction::ReturnCallIndirect0 { .. } | - Instruction::ReturnCallIndirect0Imm16 { .. } | - Instruction::ReturnCallIndirect { .. } | - Instruction::ReturnCallIndirectImm16 { .. } => {}, - Instruction::CallInternal0 { .. } | - Instruction::CallInternal { .. } | - Instruction::CallImported0 { .. } | - Instruction::CallImported { .. } | - Instruction::CallIndirect0 { .. } | - Instruction::CallIndirect0Imm16 { .. } | - Instruction::CallIndirect { .. } | - Instruction::CallIndirectImm16 { .. } => {}, - Instruction::Select { lhs, .. } => f(lhs), - Instruction::SelectImm32Rhs { lhs, .. } => f(lhs), - Instruction::SelectImm32Lhs { .. } | - Instruction::SelectImm32 { .. } => {}, - Instruction::SelectI64Imm32Rhs { lhs, .. } => f(lhs), - Instruction::SelectI64Imm32Lhs { .. } | - Instruction::SelectI64Imm32 { .. } => {}, - Instruction::SelectF64Imm32Rhs { lhs, .. } => f(lhs), - Instruction::SelectF64Imm32Lhs { .. } | - Instruction::SelectF64Imm32 { .. } => {}, - Instruction::RefFunc { .. } | - Instruction::TableGet { .. } | - Instruction::TableGetImm { .. } | - Instruction::TableSize { .. } => {}, - Instruction::TableSet { index, value } => visit_registers!(f, index, value), - Instruction::TableSetAt { value, .. } => f(value), - Instruction::TableCopy { dst, src, len } => visit_registers!(f, dst, src, len), - Instruction::TableCopyTo { dst: _, src, len } => visit_registers!(f, src, len), - Instruction::TableCopyFrom { dst, src: _, len } => visit_registers!(f, dst, len), - Instruction::TableCopyFromTo { dst: _, src: _, len } => f(len), - Instruction::TableCopyExact { dst, src, len: _ } => visit_registers!(f, dst, src), - Instruction::TableCopyToExact { dst: _, src, len: _ } => f(src), - Instruction::TableCopyFromExact { dst, src: _, len: _ } => f(dst), - Instruction::TableCopyFromToExact { dst: _, src: _, len: _ } => {}, - Instruction::TableInit { dst, src, len } => visit_registers!(f, dst, src, len), - Instruction::TableInitTo { dst: _, src, len } => visit_registers!(f, src, len), - Instruction::TableInitFrom { dst, src: _, len } => visit_registers!(f, dst, len), - Instruction::TableInitFromTo { dst: _, src: _, len } => f(len), - Instruction::TableInitExact { dst, src, len: _ } => visit_registers!(f, dst, src), - Instruction::TableInitToExact { dst: _, src, len: _ } => f(src), - Instruction::TableInitFromExact { dst, src: _, len: _ } => f(dst), - Instruction::TableInitFromToExact { dst: _, src: _, len: _ } => {}, - Instruction::TableFill { dst, len, value } => visit_registers!(f, dst, len, value), - Instruction::TableFillAt { dst: _, len, value } => visit_registers!(f, len, value), - Instruction::TableFillExact { dst, len: _, value } => visit_registers!(f, dst, value), - Instruction::TableFillAtExact { dst: _, len: _, value } => f(value), - Instruction::TableGrow { result: _, delta, value } => visit_registers!(f, delta, value), - Instruction::TableGrowImm { result: _, delta: _, value } => f(value), - Instruction::ElemDrop(_) => {} - Instruction::DataDrop(_) => {} - Instruction::MemorySize { result: _ } => {}, - Instruction::MemoryGrow { result: _, delta } => f(delta), - Instruction::MemoryGrowBy { result: _, delta: _ } => {}, - Instruction::MemoryCopy { dst, src, len } => visit_registers!(f, dst, src, len), - Instruction::MemoryCopyTo { dst: _, src, len } => visit_registers!(f, src, len), - Instruction::MemoryCopyFrom { dst, src: _, len } => visit_registers!(f, dst, len), - Instruction::MemoryCopyFromTo { dst: _, src: _, len } => f(len), - Instruction::MemoryCopyExact { dst, src, len: _ } => visit_registers!(f, dst, src), - Instruction::MemoryCopyToExact { dst: _, src, len: _ } => f(src), - Instruction::MemoryCopyFromExact { dst, src: _, len: _ } => f(dst), - Instruction::MemoryCopyFromToExact { dst: _, src: _, len: _ } => {}, - Instruction::MemoryFill { dst, value, len } => visit_registers!(f, dst, value, len), - Instruction::MemoryFillAt { dst: _, value, len } => visit_registers!(f, value, len), - Instruction::MemoryFillImm { dst, value: _, len } => visit_registers!(f, dst, len), - Instruction::MemoryFillExact { dst, value, len: _ } => visit_registers!(f, dst, value), - Instruction::MemoryFillAtImm { dst: _, value: _, len } => f(len), - Instruction::MemoryFillAtExact { dst: _, value, len: _ } => f(value), - Instruction::MemoryFillImmExact { dst, value: _, len: _ } => f(dst), - Instruction::MemoryFillAtImmExact { dst: _, value: _, len: _ } => {}, - Instruction::MemoryInit { dst, src, len } => visit_registers!(f, dst, src, len), - Instruction::MemoryInitTo { dst: _, src, len } => visit_registers!(f, src, len), - Instruction::MemoryInitFrom { dst, src: _, len } => visit_registers!(f, dst, len), - Instruction::MemoryInitFromTo { dst: _, src: _, len } => f(len), - Instruction::MemoryInitExact { dst, src, len: _ } => visit_registers!(f, dst, src), - Instruction::MemoryInitToExact { dst: _, src, len: _ } => f(src), - Instruction::MemoryInitFromExact { dst, src: _, len: _ } => f(dst), - Instruction::MemoryInitFromToExact { dst: _, src: _, len: _ } => {}, - Instruction::GlobalGet { result: _, global: _ } => {}, - Instruction::GlobalSet { global: _, input } => f(input), - Instruction::GlobalSetI32Imm16 { global: _, input: _ } | - Instruction::GlobalSetI64Imm16 { global: _, input: _ } => {}, - Instruction::I32Load { ptr, .. } | - Instruction::I64Load { ptr, .. } | - Instruction::F32Load { ptr, .. } | - Instruction::F64Load { ptr, .. } | - Instruction::I32Load8s { ptr, .. } | - Instruction::I32Load8u { ptr, .. } | - Instruction::I32Load16s { ptr, .. } | - Instruction::I32Load16u { ptr, .. } | - Instruction::I64Load8s { ptr, .. } | - Instruction::I64Load8u { ptr, .. } | - Instruction::I64Load16s { ptr, .. } | - Instruction::I64Load16u { ptr, .. } | - Instruction::I64Load32s { ptr, .. } | - Instruction::I64Load32u { ptr, .. } | - Instruction::I32LoadOffset16 { ptr, .. } | - Instruction::I64LoadOffset16 { ptr, .. } | - Instruction::F32LoadOffset16 { ptr, .. } | - Instruction::F64LoadOffset16 { ptr, .. } | - Instruction::I32Load8sOffset16 { ptr, .. } | - Instruction::I32Load8uOffset16 { ptr, .. } | - Instruction::I32Load16sOffset16 { ptr, .. } | - Instruction::I32Load16uOffset16 { ptr, .. } | - Instruction::I64Load8sOffset16 { ptr, .. } | - Instruction::I64Load8uOffset16 { ptr, .. } | - Instruction::I64Load16sOffset16 { ptr, .. } | - Instruction::I64Load16uOffset16 { ptr, .. } | - Instruction::I64Load32sOffset16 { ptr, .. } | - Instruction::I64Load32uOffset16 { ptr, .. } => f(ptr), - Instruction::I32LoadAt { .. } => {}, - Instruction::I64LoadAt { .. } => {}, - Instruction::F32LoadAt { .. } => {}, - Instruction::F64LoadAt { .. } => {}, - Instruction::I32Load8sAt { .. } => {}, - Instruction::I32Load8uAt { .. } => {}, - Instruction::I32Load16sAt { .. } => {}, - Instruction::I32Load16uAt { .. } => {}, - Instruction::I64Load8sAt { .. } => {}, - Instruction::I64Load8uAt { .. } => {}, - Instruction::I64Load16sAt { .. } => {}, - Instruction::I64Load16uAt { .. } => {}, - Instruction::I64Load32sAt { .. } => {}, - Instruction::I64Load32uAt { .. } => {}, - Instruction::I32Store { ptr, .. } | - Instruction::I32StoreOffset16Imm16 { ptr, .. } | - Instruction::I32Store8 { ptr, .. } | - Instruction::I32Store8Offset16Imm { ptr, .. } | - Instruction::I32Store16 { ptr, .. } | - Instruction::I32Store16Offset16Imm { ptr, .. } | - Instruction::I64Store { ptr, .. } | - Instruction::I64StoreOffset16Imm16 { ptr, .. } | - Instruction::I64Store8 { ptr, .. } | - Instruction::I64Store8Offset16Imm { ptr, .. } | - Instruction::I64Store16 { ptr, .. } | - Instruction::I64Store16Offset16Imm { ptr, .. } | - Instruction::I64Store32 { ptr, .. } | - Instruction::I64Store32Offset16Imm16 { ptr, .. } | - Instruction::F32Store { ptr, .. } | - Instruction::F64Store { ptr, .. } => f(ptr), - Instruction::F32StoreAt { value, .. } | - Instruction::F64StoreAt { value, .. } | - Instruction::I32StoreAt { value, .. } | - Instruction::I32Store8At { value, .. } | - Instruction::I32Store16At { value, .. } | - Instruction::I64StoreAt { value, .. } | - Instruction::I64Store8At { value, .. } | - Instruction::I64Store16At { value, .. } | - Instruction::I64Store32At { value, .. } => f(value), - Instruction::I32StoreAtImm16 { .. } | - Instruction::I32Store8AtImm { .. } | - Instruction::I32Store16AtImm { .. } | - Instruction::I64StoreAtImm16 { .. } | - Instruction::I64Store8AtImm { .. } | - Instruction::I64Store16AtImm { .. } | - Instruction::I64Store32AtImm16 { .. } => {} - Instruction::I32StoreOffset16 { ptr, value, ..} | - Instruction::I32Store8Offset16 { ptr, value, ..} | - Instruction::I32Store16Offset16 { ptr, value, ..} | - Instruction::I64StoreOffset16 { ptr, value, ..} | - Instruction::I64Store8Offset16 { ptr, value, ..} | - Instruction::I64Store16Offset16 { ptr, value, ..} | - Instruction::I64Store32Offset16 { ptr, value, ..} | - Instruction::F32StoreOffset16 { ptr, value, .. } | - Instruction::F64StoreOffset16 { ptr, value, .. } => visit_registers!(f, ptr, value), - Instruction::I32Eq { lhs, rhs, .. } | - Instruction::I64Eq { lhs, rhs, .. } | - Instruction::I32Ne { lhs, rhs, .. } | - Instruction::I64Ne { lhs, rhs, .. } | - Instruction::I32LtS { lhs, rhs, .. } | - Instruction::I32LtU { lhs, rhs, .. } | - Instruction::I64LtS { lhs, rhs, .. } | - Instruction::I64LtU { lhs, rhs, .. } | - Instruction::I32GtS { lhs, rhs, .. } | - Instruction::I32GtU { lhs, rhs, .. } | - Instruction::I64GtS { lhs, rhs, .. } | - Instruction::I64GtU { lhs, rhs, .. } | - Instruction::I32LeS { lhs, rhs, .. } | - Instruction::I32LeU { lhs, rhs, .. } | - Instruction::I64LeS { lhs, rhs, .. } | - Instruction::I64LeU { lhs, rhs, .. } | - Instruction::I32GeS { lhs, rhs, .. } | - Instruction::I32GeU { lhs, rhs, .. } | - Instruction::I64GeS { lhs, rhs, .. } | - Instruction::I64GeU { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::I32EqImm16 { lhs, .. } | - Instruction::I64EqImm16 { lhs, .. } | - Instruction::I32NeImm16 { lhs, .. } | - Instruction::I64NeImm16 { lhs, .. } | - Instruction::I32LtSImm16 { lhs, .. } | - Instruction::I32LtUImm16 { lhs, .. } | - Instruction::I64LtSImm16 { lhs, .. } | - Instruction::I64LtUImm16 { lhs, .. } | - Instruction::I32GtSImm16 { lhs, .. } | - Instruction::I32GtUImm16 { lhs, .. } | - Instruction::I64GtSImm16 { lhs, .. } | - Instruction::I64GtUImm16 { lhs, .. } | - Instruction::I32LeSImm16 { lhs, .. } | - Instruction::I32LeUImm16 { lhs, .. } | - Instruction::I64LeSImm16 { lhs, .. } | - Instruction::I64LeUImm16 { lhs, .. } | - Instruction::I32GeSImm16 { lhs, .. } | - Instruction::I32GeUImm16 { lhs, .. } | - Instruction::I64GeSImm16 { lhs, .. } | - Instruction::I64GeUImm16 { lhs, .. } => f(lhs), - Instruction::F32Eq { lhs, rhs, .. } | - Instruction::F64Eq { lhs, rhs, .. } | - Instruction::F32Ne { lhs, rhs, .. } | - Instruction::F64Ne { lhs, rhs, .. } | - Instruction::F32Lt { lhs, rhs, .. } | - Instruction::F64Lt { lhs, rhs, .. } | - Instruction::F32Le { lhs, rhs, .. } | - Instruction::F64Le { lhs, rhs, .. } | - Instruction::F32Gt { lhs, rhs, .. } | - Instruction::F64Gt { lhs, rhs, .. } | - Instruction::F32Ge { lhs, rhs, .. } | - Instruction::F64Ge { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::I32Clz { input, .. } | - Instruction::I64Clz { input, .. } | - Instruction::I32Ctz { input, .. } | - Instruction::I64Ctz { input, .. } | - Instruction::I32Popcnt { input, .. } | - Instruction::I64Popcnt { input, .. } => f(input), - Instruction::I32Add { lhs, rhs, .. } | - Instruction::I64Add { lhs, rhs, .. } | - Instruction::I32Sub { lhs, rhs, .. } | - Instruction::I64Sub { lhs, rhs, .. } | - Instruction::I32Mul { lhs, rhs, .. } | - Instruction::I64Mul { lhs, rhs, .. } | - Instruction::I32DivS { lhs, rhs, .. } | - Instruction::I64DivS { lhs, rhs, .. } | - Instruction::I32DivU { lhs, rhs, .. } | - Instruction::I64DivU { lhs, rhs, .. } | - Instruction::I32RemS { lhs, rhs, .. } | - Instruction::I64RemS { lhs, rhs, .. } | - Instruction::I32RemU { lhs, rhs, .. } | - Instruction::I64RemU { lhs, rhs, .. } | - Instruction::I32And { lhs, rhs, .. } | - Instruction::I32AndEqz { lhs, rhs, .. } | - Instruction::I64And { lhs, rhs, .. } | - Instruction::I32Or { lhs, rhs, .. } | - Instruction::I32OrEqz { lhs, rhs, .. } | - Instruction::I64Or { lhs, rhs, .. } | - Instruction::I32Xor { lhs, rhs, .. } | - Instruction::I32XorEqz { lhs, rhs, .. } | - Instruction::I64Xor { lhs, rhs, .. } | - Instruction::I32Shl { lhs, rhs, .. } | - Instruction::I64Shl { lhs, rhs, .. } | - Instruction::I32ShrU { lhs, rhs, .. } | - Instruction::I64ShrU { lhs, rhs, .. } | - Instruction::I32ShrS { lhs, rhs, .. } | - Instruction::I64ShrS { lhs, rhs, .. } | - Instruction::I32Rotl { lhs, rhs, .. } | - Instruction::I64Rotl { lhs, rhs, .. } | - Instruction::I32Rotr { lhs, rhs, .. } | - Instruction::I64Rotr { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::I32AddImm16 { lhs, .. } | - Instruction::I64AddImm16 { lhs, .. } | - Instruction::I32MulImm16 { lhs, .. } | - Instruction::I64MulImm16 { lhs, .. } | - Instruction::I32DivSImm16 { lhs, .. } | - Instruction::I64DivSImm16 { lhs, .. } | - Instruction::I32DivUImm16 { lhs, .. } | - Instruction::I64DivUImm16 { lhs, .. } | - Instruction::I32RemSImm16 { lhs, .. } | - Instruction::I64RemSImm16 { lhs, .. } | - Instruction::I32RemUImm16 { lhs, .. } | - Instruction::I64RemUImm16 { lhs, .. } | - Instruction::I32AndEqzImm16 { lhs, .. } | - Instruction::I32AndImm16 { lhs, .. } | - Instruction::I64AndImm16 { lhs, .. } | - Instruction::I32OrEqzImm16 { lhs, .. } | - Instruction::I32OrImm16 { lhs, .. } | - Instruction::I64OrImm16 { lhs, .. } | - Instruction::I32XorEqzImm16 { lhs, .. } | - Instruction::I32XorImm16 { lhs, .. } | - Instruction::I64XorImm16 { lhs, .. } | - Instruction::I32ShlImm { lhs, .. } | - Instruction::I64ShlImm { lhs, .. } | - Instruction::I32ShrUImm { lhs, .. } | - Instruction::I64ShrUImm { lhs, .. } | - Instruction::I32ShrSImm { lhs, .. } | - Instruction::I64ShrSImm { lhs, .. } | - Instruction::I32RotlImm { lhs, .. } | - Instruction::I64RotlImm { lhs, .. } | - Instruction::I32RotrImm { lhs, .. } | - Instruction::I64RotrImm { lhs, .. } => f(lhs), - Instruction::I32SubImm16Rev { rhs, .. } | - Instruction::I64SubImm16Rev { rhs, .. } | - Instruction::I32DivSImm16Rev { rhs, .. } | - Instruction::I64DivSImm16Rev { rhs, .. } | - Instruction::I32DivUImm16Rev { rhs, .. } | - Instruction::I64DivUImm16Rev { rhs, .. } | - Instruction::I32RemSImm16Rev { rhs, .. } | - Instruction::I64RemSImm16Rev { rhs, .. } | - Instruction::I32RemUImm16Rev { rhs, .. } | - Instruction::I64RemUImm16Rev { rhs, .. } | - Instruction::I32ShlImm16Rev { rhs, .. } | - Instruction::I64ShlImm16Rev { rhs, .. } | - Instruction::I32ShrUImm16Rev { rhs, .. } | - Instruction::I64ShrUImm16Rev { rhs, .. } | - Instruction::I32ShrSImm16Rev { rhs, .. } | - Instruction::I64ShrSImm16Rev { rhs, .. } | - Instruction::I32RotlImm16Rev { rhs, .. } | - Instruction::I64RotlImm16Rev { rhs, .. } | - Instruction::I32RotrImm16Rev { rhs, .. } | - Instruction::I64RotrImm16Rev { rhs, .. } => f(rhs), - Instruction::F32Abs { input, .. } | - Instruction::F64Abs { input, .. } | - Instruction::F32Neg { input, .. } | - Instruction::F64Neg { input, .. } | - Instruction::F32Ceil { input, .. } | - Instruction::F64Ceil { input, .. } | - Instruction::F32Floor { input, .. } | - Instruction::F64Floor { input, .. } | - Instruction::F32Trunc { input, .. } | - Instruction::F64Trunc { input, .. } | - Instruction::F32Nearest { input, .. } | - Instruction::F64Nearest { input, .. } | - Instruction::F32Sqrt { input, .. } | - Instruction::F64Sqrt { input, .. } => f(input), - Instruction::F32Add { lhs, rhs, .. } | - Instruction::F64Add { lhs, rhs, .. } | - Instruction::F32Sub { lhs, rhs, .. } | - Instruction::F64Sub { lhs, rhs, .. } | - Instruction::F32Mul { lhs, rhs, .. } | - Instruction::F64Mul { lhs, rhs, .. } | - Instruction::F32Div { lhs, rhs, .. } | - Instruction::F64Div { lhs, rhs, .. } | - Instruction::F32Min { lhs, rhs, .. } | - Instruction::F64Min { lhs, rhs, .. } | - Instruction::F32Max { lhs, rhs, .. } | - Instruction::F64Max { lhs, rhs, .. } | - Instruction::F32Copysign { lhs, rhs, .. } | - Instruction::F64Copysign { lhs, rhs, .. } => visit_registers!(f, lhs, rhs), - Instruction::F32CopysignImm { lhs, .. } | - Instruction::F64CopysignImm { lhs, .. } => f(lhs), - Instruction::I32WrapI64 { input, .. } | - Instruction::I32TruncF32S { input, .. } | - Instruction::I32TruncF32U { input, .. } | - Instruction::I32TruncF64S { input, .. } | - Instruction::I32TruncF64U { input, .. } | - Instruction::I64TruncF32S { input, .. } | - Instruction::I64TruncF32U { input, .. } | - Instruction::I64TruncF64S { input, .. } | - Instruction::I64TruncF64U { input, .. } | - Instruction::I32TruncSatF32S { input, .. } | - Instruction::I32TruncSatF32U { input, .. } | - Instruction::I32TruncSatF64S { input, .. } | - Instruction::I32TruncSatF64U { input, .. } | - Instruction::I64TruncSatF32S { input, .. } | - Instruction::I64TruncSatF32U { input, .. } | - Instruction::I64TruncSatF64S { input, .. } | - Instruction::I64TruncSatF64U { input, .. } | - Instruction::I32Extend8S { input, .. } | - Instruction::I32Extend16S { input, .. } | - Instruction::I64Extend8S { input, .. } | - Instruction::I64Extend16S { input, .. } | - Instruction::I64Extend32S { input, .. } | - Instruction::F32DemoteF64 { input, .. } | - Instruction::F64PromoteF32 { input, .. } | - Instruction::F32ConvertI32S { input, .. } | - Instruction::F32ConvertI32U { input, .. } | - Instruction::F32ConvertI64S { input, .. } | - Instruction::F32ConvertI64U { input, .. } | - Instruction::F64ConvertI32S { input, .. } | - Instruction::F64ConvertI32U { input, .. } | - Instruction::F64ConvertI64S { input, .. } | - Instruction::F64ConvertI64U { input, .. } => f(input), - } - } -} + #[inline(always)] + fn visit_result_regs(&mut self, _reg: &mut RegSpan, _len: Option) {} -impl VisitInputRegisters for [Reg; N] { - fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Reg)) { - for register in self { - f(register); - } + #[inline] + fn visit_input_reg(&mut self, reg: &mut Reg) { + (self.f)(reg); } -} -impl VisitInputRegisters for RegSpan { - fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Reg)) { - f(self.head_mut()) + #[inline] + fn visit_input_regs(&mut self, regs: &mut RegSpan, _len: Option) { + (self.f)(regs.head_mut()); } } -impl VisitInputRegisters for RegSpanIter { +impl VisitInputRegisters for Instruction { fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Reg)) { - let len = self.len_as_u16(); - let mut span = self.span(); - f(span.head_mut()); - *self = span.iter(len); + // Note: for copy instructions that copy register values we also need to visit + // their results because preserved registers might be populating them. + match self { + | Self::Copy { result, .. } => f(result), + | Self::Copy2 { results, .. } + | Self::CopySpan { results, .. } + | Self::CopySpanNonOverlapping { results, .. } + | Self::CopyMany { results, .. } + | Self::CopyManyNonOverlapping { results, .. } => f(results.head_mut()), + _ => {} + } + self.visit_regs(&mut Visitor { f }); } }