From 6d72480f05fd2257ee58d4b7aa2e14bf63f4a59b Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 22 Nov 2023 12:22:55 +0100 Subject: [PATCH 01/17] improve Debug impl for Const16 --- .../src/engine/regmach/bytecode/immediate.rs | 12 ++++++- .../src/engine/regmach/bytecode/utils.rs | 31 +++++++++++++++++-- 2 files changed, 40 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/bytecode/immediate.rs b/crates/wasmi/src/engine/regmach/bytecode/immediate.rs index 06dbf40c08..dfaf338673 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/immediate.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/immediate.rs @@ -6,7 +6,6 @@ use wasmi_core::{F32, F64}; pub struct OutOfBoundsConst; /// A typed 16-bit encoded constant value. -#[derive(Debug)] pub struct Const16 { /// The underlying untyped value. inner: AnyConst16, @@ -14,6 +13,17 @@ pub struct Const16 { marker: PhantomData T>, } +impl Debug for Const16 +where + Self: Into, + T: Debug, +{ + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let inner: T = (*self).into(); + inner.fmt(f) + } +} + impl Const16 { /// Returns `true` if the [`Const16`]`` is equal to zero. pub fn is_zero(&self) -> bool { diff --git a/crates/wasmi/src/engine/regmach/bytecode/utils.rs b/crates/wasmi/src/engine/regmach/bytecode/utils.rs index 40ffc642fc..bdb3724af0 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/utils.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/utils.rs @@ -4,6 +4,7 @@ use crate::engine::{ func_builder::TranslationErrorInner, TranslationError, }; +use core::fmt; #[cfg(doc)] use super::Instruction; @@ -253,7 +254,7 @@ impl BinInstr { /// # Note /// /// Optimized for small constant values that fit into 16-bit. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq)] pub struct BinInstrImm16 { /// The register storing the result of the computation. pub result: Register, @@ -273,6 +274,19 @@ pub struct BinInstrImm16 { pub imm_in: Const16, } +impl fmt::Debug for BinInstrImm16 +where + T: fmt::Debug + From>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BinInstrImm16") + .field("result", &self.result) + .field("reg_in", &self.reg_in) + .field("imm_in", &self.imm_in) + .finish() + } +} + impl BinInstrImm16 { /// Creates a new [`BinInstrImm16`]. pub fn new(result: Register, reg_in: Register, imm_in: Const16) -> Self { @@ -542,7 +556,7 @@ impl BranchBinOpInstr { } /// A generic fused comparison and conditional branch [`Instruction`]. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone, PartialEq, Eq)] pub struct BranchBinOpInstrImm { /// The left-hand side operand to the conditional operator. pub lhs: Register, @@ -552,6 +566,19 @@ pub struct BranchBinOpInstrImm { pub offset: BranchOffset16, } +impl fmt::Debug for BranchBinOpInstrImm +where + T: fmt::Debug + From>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BranchBinOpInstrImm") + .field("lhs", &self.lhs) + .field("rhs", &self.rhs) + .field("offset", &self.offset) + .finish() + } +} + impl BranchBinOpInstrImm { /// Creates a new [`BranchBinOpInstr`]. pub fn new(lhs: Register, rhs: Const16, offset: BranchOffset16) -> Self { From e65ac71e4a3fc3e2f1ec9526a1a5b3c5b3770859 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 22 Nov 2023 12:23:22 +0100 Subject: [PATCH 02/17] add BinAssignInstr and BinAssignInstrImm32 types --- .../src/engine/regmach/bytecode/utils.rs | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/crates/wasmi/src/engine/regmach/bytecode/utils.rs b/crates/wasmi/src/engine/regmach/bytecode/utils.rs index bdb3724af0..f1133784f4 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/utils.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/utils.rs @@ -298,6 +298,50 @@ impl BinInstrImm16 { } } +/// A binary-assign [`Register`] based instruction. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct BinAssignInstr { + /// The register storing both the result and left-hand side value. + pub inout: Register, + /// The register holding the right-hand side value. + pub rhs: Register, +} + +impl BinAssignInstr { + /// Creates a new [`BinAssignInstr`]. + pub fn new(inout: Register, rhs: Register) -> Self { + Self { inout, rhs } + } +} + +/// A binary-assign [`Register`] based instruction. +#[derive(Copy, Clone, PartialEq, Eq)] +pub struct BinAssignInstrImm32 { + /// The register storing both the result and left-hand side value. + pub inout: Register, + /// The constant right-hand side value. + pub rhs: Const32, +} + +impl fmt::Debug for BinAssignInstrImm32 +where + T: fmt::Debug + From>, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("BinAssignInstrImm32") + .field("inout", &self.inout) + .field("rhs", &self.rhs) + .finish() + } +} + +impl BinAssignInstrImm32 { + /// Creates a new [`BinAssignInstr`]. + pub fn new(inout: Register, rhs: Const32) -> Self { + Self { inout, rhs } + } +} + /// A unary instruction. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct UnaryInstr { From e1fcac47d83bb4670d34d76c86e6e91a22f6712a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 22 Nov 2023 14:07:19 +0100 Subject: [PATCH 03/17] add new op-assign[_imm] instructions This commit still misses: - docs for the new instructions - execution of the new instructions - translation of the new instructions - optimization involving the new instructions --- .../src/engine/regmach/bytecode/construct.rs | 198 ++++++++++++++++++ .../wasmi/src/engine/regmach/bytecode/mod.rs | 162 ++++++++++++++ .../src/engine/regmach/bytecode/utils.rs | 10 +- .../src/engine/regmach/executor/instrs.rs | 1 + .../engine/regmach/translator/result_mut.rs | 168 +++++++++++++++ .../regmach/translator/visit_register.rs | 170 ++++++++++++++- 6 files changed, 706 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/bytecode/construct.rs b/crates/wasmi/src/engine/regmach/bytecode/construct.rs index 5ba856b30f..f02afe8ad3 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/construct.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/construct.rs @@ -1,6 +1,8 @@ use super::{ utils::{BranchOffset16, CopysignImmInstr, Sign}, AnyConst32, + BinAssignInstr, + BinAssignInstrImm32, BinInstr, BinInstrImm16, BranchBinOpInstr, @@ -248,6 +250,202 @@ constructor_for_branch_binop_imm! { fn branch_i64_ge_u_imm(u64) -> Self::BranchI64GeUImm; } +macro_rules! constructor_for_op_assign { + ( $( fn $name:ident() -> Self::$op_code:ident; )* ) => { + impl Instruction { + $( + #[doc = concat!("Creates a new [`Instruction::", stringify!($op_code), "`].")] + pub fn $name(inout: Register, rhs: Register) -> Self { + Self::$op_code(BinAssignInstr::new(inout, rhs)) + } + )* + } + } +} +constructor_for_op_assign! { + fn i32_eq_assign() -> Self::I32EqAssign; + fn i32_ne_assign() -> Self::I32NeAssign; + fn i32_lt_s_assign() -> Self::I32LtSAssign; + fn i32_lt_u_assign() -> Self::I32LtUAssign; + fn i32_le_s_assign() -> Self::I32LtSAssign; + fn i32_le_u_assign() -> Self::I32LtUAssign; + fn i32_gt_s_assign() -> Self::I32LtSAssign; + fn i32_gt_u_assign() -> Self::I32LtUAssign; + fn i32_ge_s_assign() -> Self::I32LtSAssign; + fn i32_ge_u_assign() -> Self::I32LtUAssign; + + fn i64_eq_assign() -> Self::I64EqAssign; + fn i64_ne_assign() -> Self::I64NeAssign; + fn i64_lt_s_assign() -> Self::I64LtSAssign; + fn i64_lt_u_assign() -> Self::I64LtUAssign; + fn i64_le_s_assign() -> Self::I64LtSAssign; + fn i64_le_u_assign() -> Self::I64LtUAssign; + fn i64_gt_s_assign() -> Self::I64LtSAssign; + fn i64_gt_u_assign() -> Self::I64LtUAssign; + fn i64_ge_s_assign() -> Self::I64LtSAssign; + fn i64_ge_u_assign() -> Self::I64LtUAssign; + + fn f32_eq_assign() -> Self::F32EqAssign; + fn f32_ne_assign() -> Self::F32NeAssign; + fn f32_lt_assign() -> Self::F32LtAssign; + fn f32_le_assign() -> Self::F32LeAssign; + fn f32_gt_assign() -> Self::F32GtAssign; + fn f32_ge_assign() -> Self::F32GeAssign; + + fn f64_eq_assign() -> Self::F64EqAssign; + fn f64_ne_assign() -> Self::F64NeAssign; + fn f64_lt_assign() -> Self::F64LtAssign; + fn f64_le_assign() -> Self::F64LeAssign; + fn f64_gt_assign() -> Self::F64GtAssign; + fn f64_ge_assign() -> Self::F64GeAssign; + + fn i32_add_assign() -> Self::I32AddAssign; + fn i32_sub_assign() -> Self::I32SubAssign; + fn i32_mul_assign() -> Self::I32MulAssign; + fn i32_div_s_assign() -> Self::I32DivSAssign; + fn i32_div_u_assign() -> Self::I32DivUAssign; + fn i32_rem_s_assign() -> Self::I32RemSAssign; + fn i32_rem_u_assign() -> Self::I32RemUAssign; + fn i32_and_assign() -> Self::I32AndAssign; + fn i32_or_assign() -> Self::I32OrAssign; + fn i32_xor_assign() -> Self::I32XorAssign; + fn i32_shl_assign() -> Self::I32ShlAssign; + fn i32_shr_s_assign() -> Self::I32ShrSAssign; + fn i32_shr_u_assign() -> Self::I32ShrUAssign; + fn i32_rotl_assign() -> Self::I32RotlAssign; + fn i32_rotr_assign() -> Self::I32RotrAssign; + + fn i64_add_assign() -> Self::I64AddAssign; + fn i64_sub_assign() -> Self::I64SubAssign; + fn i64_mul_assign() -> Self::I64MulAssign; + fn i64_div_s_assign() -> Self::I64DivSAssign; + fn i64_div_u_assign() -> Self::I64DivUAssign; + fn i64_rem_s_assign() -> Self::I64RemSAssign; + fn i64_rem_u_assign() -> Self::I64RemUAssign; + fn i64_and_assign() -> Self::I64AndAssign; + fn i64_or_assign() -> Self::I64OrAssign; + fn i64_xor_assign() -> Self::I64XorAssign; + fn i64_shl_assign() -> Self::I64ShlAssign; + fn i64_shr_s_assign() -> Self::I64ShrSAssign; + fn i64_shr_u_assign() -> Self::I64ShrUAssign; + fn i64_rotl_assign() -> Self::I64RotlAssign; + fn i64_rotr_assign() -> Self::I64RotrAssign; + + fn f32_add_assign() -> Self::F32AddAssign; + fn f32_sub_assign() -> Self::F32SubAssign; + fn f32_mul_assign() -> Self::F32MulAssign; + fn f32_div_assign() -> Self::F32DivAssign; + fn f32_min_assign() -> Self::F32MinAssign; + fn f32_max_assign() -> Self::F32MaxAssign; + fn f32_copysign_assign() -> Self::F32CopysignAssign; + + fn f64_add_assign() -> Self::F64AddAssign; + fn f64_sub_assign() -> Self::F64SubAssign; + fn f64_mul_assign() -> Self::F64MulAssign; + fn f64_div_assign() -> Self::F64DivAssign; + fn f64_min_assign() -> Self::F64MinAssign; + fn f64_max_assign() -> Self::F64MaxAssign; + fn f64_copysign_assign() -> Self::F64CopysignAssign; +} + +macro_rules! constructor_for_op_assign_imm32 { + ( $( fn $name:ident($ty:ty) -> Self::$op_code:ident; )* ) => { + impl Instruction { + $( + #[doc = concat!("Creates a new [`Instruction::", stringify!($op_code), "`].")] + pub fn $name(inout: Register, rhs: impl Into>) -> Self { + Self::$op_code(BinAssignInstrImm32::new(inout, rhs.into())) + } + )* + } + } +} +constructor_for_op_assign_imm32! { + fn i32_eq_assign_imm(i32) -> Self::I32EqAssignImm; + fn i32_ne_assign_imm(i32) -> Self::I32NeAssignImm; + fn i32_lt_s_assign_imm(i32) -> Self::I32LtSAssignImm; + fn i32_lt_u_assign_imm(u32) -> Self::I32LtUAssignImm; + fn i32_le_s_assign_imm(i32) -> Self::I32LtSAssignImm; + fn i32_le_u_assign_imm(u32) -> Self::I32LtUAssignImm; + fn i32_gt_s_assign_imm(i32) -> Self::I32LtSAssignImm; + fn i32_gt_u_assign_imm(u32) -> Self::I32LtUAssignImm; + fn i32_ge_s_assign_imm(i32) -> Self::I32LtSAssignImm; + fn i32_ge_u_assign_imm(u32) -> Self::I32LtUAssignImm; + + fn i64_eq_assign_imm32(i64) -> Self::I64EqAssignImm32; + fn i64_ne_assign_imm32(i64) -> Self::I64NeAssignImm32; + fn i64_lt_s_assign_imm32(i64) -> Self::I64LtSAssignImm32; + fn i64_lt_u_assign_imm32(u64) -> Self::I64LtUAssignImm32; + fn i64_le_s_assign_imm32(i64) -> Self::I64LtSAssignImm32; + fn i64_le_u_assign_imm32(u64) -> Self::I64LtUAssignImm32; + fn i64_gt_s_assign_imm32(i64) -> Self::I64LtSAssignImm32; + fn i64_gt_u_assign_imm32(u64) -> Self::I64LtUAssignImm32; + fn i64_ge_s_assign_imm32(i64) -> Self::I64LtSAssignImm32; + fn i64_ge_u_assign_imm32(u64) -> Self::I64LtUAssignImm32; + + fn f32_eq_assign_imm(f32) -> Self::F32EqAssignImm; + fn f32_ne_assign_imm(f32) -> Self::F32NeAssignImm; + fn f32_lt_assign_imm(f32) -> Self::F32LtAssignImm; + fn f32_le_assign_imm(f32) -> Self::F32LeAssignImm; + fn f32_gt_assign_imm(f32) -> Self::F32GtAssignImm; + fn f32_ge_assign_imm(f32) -> Self::F32GeAssignImm; + + fn f64_eq_assign_imm32(f64) -> Self::F64EqAssignImm32; + fn f64_ne_assign_imm32(f64) -> Self::F64NeAssignImm32; + fn f64_lt_assign_imm32(f64) -> Self::F64LtAssignImm32; + fn f64_le_assign_imm32(f64) -> Self::F64LeAssignImm32; + fn f64_gt_assign_imm32(f64) -> Self::F64GtAssignImm32; + fn f64_ge_assign_imm32(f64) -> Self::F64GeAssignImm32; + + fn i32_add_assign_imm(i32) -> Self::I32AddAssignImm; + fn i32_sub_assign_imm(i32) -> Self::I32SubAssignImm; + fn i32_mul_assign_imm(i32) -> Self::I32MulAssignImm; + fn i32_div_s_assign_imm(i32) -> Self::I32DivSAssignImm; + fn i32_div_u_assign_imm(u32) -> Self::I32DivUAssignImm; + fn i32_rem_s_assign_imm(i32) -> Self::I32RemSAssignImm; + fn i32_rem_u_assign_imm(u32) -> Self::I32RemUAssignImm; + fn i32_and_assign_imm(i32) -> Self::I32AndAssignImm; + fn i32_or_assign_imm(i32) -> Self::I32OrAssignImm; + fn i32_xor_assign_imm(i32) -> Self::I32XorAssignImm; + fn i32_shl_assign_imm(i32) -> Self::I32ShlAssignImm; + fn i32_shr_s_assign_imm(i32) -> Self::I32ShrSAssignImm; + fn i32_shr_u_assign_imm(u32) -> Self::I32ShrUAssignImm; + fn i32_rotl_assign_imm(i32) -> Self::I32RotlAssignImm; + fn i32_rotr_assign_imm(i32) -> Self::I32RotrAssignImm; + + fn i64_add_assign_imm32(i64) -> Self::I64AddAssignImm32; + fn i64_sub_assign_imm32(i64) -> Self::I64SubAssignImm32; + fn i64_mul_assign_imm32(i64) -> Self::I64MulAssignImm32; + fn i64_div_s_assign_imm32(i64) -> Self::I64DivSAssignImm32; + fn i64_div_u_assign_imm32(u64) -> Self::I64DivUAssignImm32; + fn i64_rem_s_assign_imm32(i64) -> Self::I64RemSAssignImm32; + fn i64_rem_u_assign_imm32(u64) -> Self::I64RemUAssignImm32; + fn i64_and_assign_imm32(i64) -> Self::I64AndAssignImm32; + fn i64_or_assign_imm32(i64) -> Self::I64OrAssignImm32; + fn i64_xor_assign_imm32(i64) -> Self::I64XorAssignImm32; + fn i64_shl_assign_imm32(i64) -> Self::I64ShlAssignImm32; + fn i64_shr_s_assign_imm32(i64) -> Self::I64ShrSAssignImm32; + fn i64_shr_u_assign_imm32(u64) -> Self::I64ShrUAssignImm32; + fn i64_rotl_assign_imm32(i64) -> Self::I64RotlAssignImm32; + fn i64_rotr_assign_imm32(i64) -> Self::I64RotrAssignImm32; + + fn f32_add_assign_imm(f32) -> Self::F32AddAssignImm; + fn f32_sub_assign_imm(f32) -> Self::F32SubAssignImm; + fn f32_mul_assign_imm(f32) -> Self::F32MulAssignImm; + fn f32_div_assign_imm(f32) -> Self::F32DivAssignImm; + fn f32_min_assign_imm(f32) -> Self::F32MinAssignImm; + fn f32_max_assign_imm(f32) -> Self::F32MaxAssignImm; + fn f32_copysign_assign_imm(f32) -> Self::F32CopysignAssignImm; + + fn f64_add_assign_imm32(f64) -> Self::F64AddAssignImm32; + fn f64_sub_assign_imm32(f64) -> Self::F64SubAssignImm32; + fn f64_mul_assign_imm32(f64) -> Self::F64MulAssignImm32; + fn f64_div_assign_imm32(f64) -> Self::F64DivAssignImm32; + fn f64_min_assign_imm32(f64) -> Self::F64MinAssignImm32; + fn f64_max_assign_imm32(f64) -> Self::F64MaxAssignImm32; + fn f64_copysign_assign_imm32(f64) -> Self::F64CopysignAssignImm32; +} + impl Instruction { /// Creates a new [`Instruction::Const32`] from the given `value`. pub fn const32(value: impl Into) -> Self { diff --git a/crates/wasmi/src/engine/regmach/bytecode/mod.rs b/crates/wasmi/src/engine/regmach/bytecode/mod.rs index 8ea97232b7..f644026b32 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/mod.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/mod.rs @@ -10,6 +10,8 @@ pub(crate) use self::{ immediate::{AnyConst16, AnyConst32, Const16, Const32}, provider::{Provider, ProviderSliceStack, UntypedProvider}, utils::{ + BinAssignInstr, + BinAssignInstrImm32, BinInstr, BinInstrImm16, BranchBinOpInstr, @@ -3084,6 +3086,166 @@ pub enum Instruction { F64ConvertI64S(UnaryInstr), /// Wasm `f64.convert_i64_u` instruction. F64ConvertI64U(UnaryInstr), + + I32EqAssign(BinAssignInstr), + I32EqAssignImm(BinAssignInstrImm32), + I32NeAssign(BinAssignInstr), + I32NeAssignImm(BinAssignInstrImm32), + I32LtSAssign(BinAssignInstr), + I32LtSAssignImm(BinAssignInstrImm32), + I32LtUAssign(BinAssignInstr), + I32LtUAssignImm(BinAssignInstrImm32), + I32GtSAssign(BinAssignInstr), + I32GtSAssignImm(BinAssignInstrImm32), + I32GtUAssign(BinAssignInstr), + I32GtUAssignImm(BinAssignInstrImm32), + I32LeSAssign(BinAssignInstr), + I32LeSAssignImm(BinAssignInstrImm32), + I32LeUAssign(BinAssignInstr), + I32LeUAssignImm(BinAssignInstrImm32), + I32GeSAssign(BinAssignInstr), + I32GeSAssignImm(BinAssignInstrImm32), + I32GeUAssign(BinAssignInstr), + I32GeUAssignImm(BinAssignInstrImm32), + + I64EqAssign(BinAssignInstr), + I64EqAssignImm32(BinAssignInstrImm32), + I64NeAssign(BinAssignInstr), + I64NeAssignImm32(BinAssignInstrImm32), + I64LtSAssign(BinAssignInstr), + I64LtSAssignImm32(BinAssignInstrImm32), + I64LtUAssign(BinAssignInstr), + I64LtUAssignImm32(BinAssignInstrImm32), + I64GtSAssign(BinAssignInstr), + I64GtSAssignImm32(BinAssignInstrImm32), + I64GtUAssign(BinAssignInstr), + I64GtUAssignImm32(BinAssignInstrImm32), + I64LeSAssign(BinAssignInstr), + I64LeSAssignImm32(BinAssignInstrImm32), + I64LeUAssign(BinAssignInstr), + I64LeUAssignImm32(BinAssignInstrImm32), + I64GeSAssign(BinAssignInstr), + I64GeSAssignImm32(BinAssignInstrImm32), + I64GeUAssign(BinAssignInstr), + I64GeUAssignImm32(BinAssignInstrImm32), + + F32EqAssign(BinAssignInstr), + F32EqAssignImm(BinAssignInstrImm32), + F32NeAssign(BinAssignInstr), + F32NeAssignImm(BinAssignInstrImm32), + F32LtAssign(BinAssignInstr), + F32LtAssignImm(BinAssignInstrImm32), + F32LeAssign(BinAssignInstr), + F32LeAssignImm(BinAssignInstrImm32), + F32GtAssign(BinAssignInstr), + F32GtAssignImm(BinAssignInstrImm32), + F32GeAssign(BinAssignInstr), + F32GeAssignImm(BinAssignInstrImm32), + + F64EqAssign(BinAssignInstr), + F64EqAssignImm32(BinAssignInstrImm32), + F64NeAssign(BinAssignInstr), + F64NeAssignImm32(BinAssignInstrImm32), + F64LtAssign(BinAssignInstr), + F64LtAssignImm32(BinAssignInstrImm32), + F64LeAssign(BinAssignInstr), + F64LeAssignImm32(BinAssignInstrImm32), + F64GtAssign(BinAssignInstr), + F64GtAssignImm32(BinAssignInstrImm32), + F64GeAssign(BinAssignInstr), + F64GeAssignImm32(BinAssignInstrImm32), + + I32AddAssign(BinAssignInstr), + I32AddAssignImm(BinAssignInstrImm32), + I32SubAssign(BinAssignInstr), + I32SubAssignImm(BinAssignInstrImm32), + I32MulAssign(BinAssignInstr), + I32MulAssignImm(BinAssignInstrImm32), + I32DivSAssign(BinAssignInstr), + I32DivSAssignImm(BinAssignInstrImm32), + I32DivUAssign(BinAssignInstr), + I32DivUAssignImm(BinAssignInstrImm32), + I32RemSAssign(BinAssignInstr), + I32RemSAssignImm(BinAssignInstrImm32), + I32RemUAssign(BinAssignInstr), + I32RemUAssignImm(BinAssignInstrImm32), + I32AndAssign(BinAssignInstr), + I32AndAssignImm(BinAssignInstrImm32), + I32OrAssign(BinAssignInstr), + I32OrAssignImm(BinAssignInstrImm32), + I32XorAssign(BinAssignInstr), + I32XorAssignImm(BinAssignInstrImm32), + I32ShlAssign(BinAssignInstr), + I32ShlAssignImm(BinAssignInstrImm32), + I32ShrUAssign(BinAssignInstr), + I32ShrUAssignImm(BinAssignInstrImm32), + I32ShrSAssign(BinAssignInstr), + I32ShrSAssignImm(BinAssignInstrImm32), + I32RotlAssign(BinAssignInstr), + I32RotlAssignImm(BinAssignInstrImm32), + I32RotrAssign(BinAssignInstr), + I32RotrAssignImm(BinAssignInstrImm32), + + I64AddAssign(BinAssignInstr), + I64AddAssignImm32(BinAssignInstrImm32), + I64SubAssign(BinAssignInstr), + I64SubAssignImm32(BinAssignInstrImm32), + I64MulAssign(BinAssignInstr), + I64MulAssignImm32(BinAssignInstrImm32), + I64DivSAssign(BinAssignInstr), + I64DivSAssignImm32(BinAssignInstrImm32), + I64DivUAssign(BinAssignInstr), + I64DivUAssignImm32(BinAssignInstrImm32), + I64RemSAssign(BinAssignInstr), + I64RemSAssignImm32(BinAssignInstrImm32), + I64RemUAssign(BinAssignInstr), + I64RemUAssignImm32(BinAssignInstrImm32), + I64AndAssign(BinAssignInstr), + I64AndAssignImm32(BinAssignInstrImm32), + I64OrAssign(BinAssignInstr), + I64OrAssignImm32(BinAssignInstrImm32), + I64XorAssign(BinAssignInstr), + I64XorAssignImm32(BinAssignInstrImm32), + I64ShlAssign(BinAssignInstr), + I64ShlAssignImm32(BinAssignInstrImm32), + I64ShrUAssign(BinAssignInstr), + I64ShrUAssignImm32(BinAssignInstrImm32), + I64ShrSAssign(BinAssignInstr), + I64ShrSAssignImm32(BinAssignInstrImm32), + I64RotlAssign(BinAssignInstr), + I64RotlAssignImm32(BinAssignInstrImm32), + I64RotrAssign(BinAssignInstr), + I64RotrAssignImm32(BinAssignInstrImm32), + + F32AddAssign(BinAssignInstr), + F32AddAssignImm(BinAssignInstrImm32), + F32SubAssign(BinAssignInstr), + F32SubAssignImm(BinAssignInstrImm32), + F32MulAssign(BinAssignInstr), + F32MulAssignImm(BinAssignInstrImm32), + F32DivAssign(BinAssignInstr), + F32DivAssignImm(BinAssignInstrImm32), + F32MinAssign(BinAssignInstr), + F32MinAssignImm(BinAssignInstrImm32), + F32MaxAssign(BinAssignInstr), + F32MaxAssignImm(BinAssignInstrImm32), + F32CopysignAssign(BinAssignInstr), + F32CopysignAssignImm(BinAssignInstrImm32), + + F64AddAssign(BinAssignInstr), + F64AddAssignImm32(BinAssignInstrImm32), + F64SubAssign(BinAssignInstr), + F64SubAssignImm32(BinAssignInstrImm32), + F64MulAssign(BinAssignInstr), + F64MulAssignImm32(BinAssignInstrImm32), + F64DivAssign(BinAssignInstr), + F64DivAssignImm32(BinAssignInstrImm32), + F64MinAssign(BinAssignInstr), + F64MinAssignImm32(BinAssignInstrImm32), + F64MaxAssign(BinAssignInstr), + F64MaxAssignImm32(BinAssignInstrImm32), + F64CopysignAssign(BinAssignInstr), + F64CopysignAssignImm32(BinAssignInstrImm32), } impl Instruction { diff --git a/crates/wasmi/src/engine/regmach/bytecode/utils.rs b/crates/wasmi/src/engine/regmach/bytecode/utils.rs index f1133784f4..0475c9f5b0 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/utils.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/utils.rs @@ -315,7 +315,7 @@ impl BinAssignInstr { } /// A binary-assign [`Register`] based instruction. -#[derive(Copy, Clone, PartialEq, Eq)] +#[derive(Copy, Clone)] pub struct BinAssignInstrImm32 { /// The register storing both the result and left-hand side value. pub inout: Register, @@ -323,6 +323,14 @@ pub struct BinAssignInstrImm32 { pub rhs: Const32, } +impl PartialEq for BinAssignInstrImm32 { + fn eq(&self, other: &Self) -> bool { + self.inout == other.inout && self.rhs == other.rhs + } +} + +impl Eq for BinAssignInstrImm32 {} + impl fmt::Debug for BinAssignInstrImm32 where T: fmt::Debug + From>, diff --git a/crates/wasmi/src/engine/regmach/executor/instrs.rs b/crates/wasmi/src/engine/regmach/executor/instrs.rs index eec6196df6..328cd79ce0 100644 --- a/crates/wasmi/src/engine/regmach/executor/instrs.rs +++ b/crates/wasmi/src/engine/regmach/executor/instrs.rs @@ -835,6 +835,7 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { Instr::F64ConvertI32U(instr) => self.execute_f64_convert_i32_u(instr), Instr::F64ConvertI64S(instr) => self.execute_f64_convert_i64_s(instr), Instr::F64ConvertI64U(instr) => self.execute_f64_convert_i64_u(instr), + _ => todo!(), } } } diff --git a/crates/wasmi/src/engine/regmach/translator/result_mut.rs b/crates/wasmi/src/engine/regmach/translator/result_mut.rs index f3d4f9c525..93c6f69c61 100644 --- a/crates/wasmi/src/engine/regmach/translator/result_mut.rs +++ b/crates/wasmi/src/engine/regmach/translator/result_mut.rs @@ -9,6 +9,8 @@ use crate::{ bytecode::{FuncIdx, SignatureIdx}, regmach::{ bytecode::{ + BinAssignInstr, + BinAssignInstrImm32, BinInstr, BinInstrImm16, CopysignImmInstr, @@ -501,6 +503,158 @@ impl Instruction { Instruction::F64ConvertI32U(instr) | Instruction::F64ConvertI64S(instr) | Instruction::F64ConvertI64U(instr) => instr.result_mut(), + Instruction::I32EqAssign(instr) | + Instruction::I32NeAssign(instr) | + Instruction::I32LtSAssign(instr) | + Instruction::I32LtUAssign(instr) | + Instruction::I32GtSAssign(instr) | + Instruction::I32GtUAssign(instr) | + Instruction::I32LeSAssign(instr) | + Instruction::I32LeUAssign(instr) | + Instruction::I32GeSAssign(instr) | + Instruction::I32GeUAssign(instr) | + Instruction::I64EqAssign(instr) | + Instruction::I64NeAssign(instr) | + Instruction::I64LtSAssign(instr) | + Instruction::I64LtUAssign(instr) | + Instruction::I64GtSAssign(instr) | + Instruction::I64GtUAssign(instr) | + Instruction::I64LeSAssign(instr) | + Instruction::I64LeUAssign(instr) | + Instruction::I64GeSAssign(instr) | + Instruction::I64GeUAssign(instr) | + Instruction::F32EqAssign(instr) | + Instruction::F32NeAssign(instr) | + Instruction::F32LtAssign(instr) | + Instruction::F32LeAssign(instr) | + Instruction::F32GtAssign(instr) | + Instruction::F32GeAssign(instr) | + Instruction::F64EqAssign(instr) | + Instruction::F64NeAssign(instr) | + Instruction::F64LtAssign(instr) | + Instruction::F64LeAssign(instr) | + Instruction::F64GtAssign(instr) | + Instruction::F64GeAssign(instr) | + Instruction::I32AddAssign(instr) | + Instruction::I32SubAssign(instr) | + Instruction::I32MulAssign(instr) | + Instruction::I32DivSAssign(instr) | + Instruction::I32DivUAssign(instr) | + Instruction::I32RemSAssign(instr) | + Instruction::I32RemUAssign(instr) | + Instruction::I32AndAssign(instr) | + Instruction::I32OrAssign(instr) | + Instruction::I32XorAssign(instr) | + Instruction::I32ShlAssign(instr) | + Instruction::I32ShrUAssign(instr) | + Instruction::I32ShrSAssign(instr) | + Instruction::I32RotlAssign(instr) | + Instruction::I32RotrAssign(instr) | + Instruction::I64AddAssign(instr) | + Instruction::I64SubAssign(instr) | + Instruction::I64MulAssign(instr) | + Instruction::I64DivSAssign(instr) | + Instruction::I64DivUAssign(instr) | + Instruction::I64RemSAssign(instr) | + Instruction::I64RemUAssign(instr) | + Instruction::I64AndAssign(instr) | + Instruction::I64OrAssign(instr) | + Instruction::I64XorAssign(instr) | + Instruction::I64ShlAssign(instr) | + Instruction::I64ShrUAssign(instr) | + Instruction::I64ShrSAssign(instr) | + Instruction::I64RotlAssign(instr) | + Instruction::I64RotrAssign(instr) | + Instruction::F32AddAssign(instr) | + Instruction::F32SubAssign(instr) | + Instruction::F32MulAssign(instr) | + Instruction::F32DivAssign(instr) | + Instruction::F32MinAssign(instr) | + Instruction::F32MaxAssign(instr) | + Instruction::F32CopysignAssign(instr) | + Instruction::F64AddAssign(instr) | + Instruction::F64SubAssign(instr) | + Instruction::F64MulAssign(instr) | + Instruction::F64DivAssign(instr) | + Instruction::F64MinAssign(instr) | + Instruction::F64MaxAssign(instr) | + Instruction::F64CopysignAssign(instr) => instr.result_mut(), + Instruction::I32EqAssignImm(instr) | + Instruction::I32NeAssignImm(instr) | + Instruction::I32LtSAssignImm(instr) | + Instruction::I32GtSAssignImm(instr) | + Instruction::I32LeSAssignImm(instr) | + Instruction::I32GeSAssignImm(instr) => instr.result_mut(), + Instruction::I32LtUAssignImm(instr) | + Instruction::I32GtUAssignImm(instr) | + Instruction::I32LeUAssignImm(instr) | + Instruction::I32GeUAssignImm(instr) => instr.result_mut(), + Instruction::I64EqAssignImm32(instr) | + Instruction::I64NeAssignImm32(instr) | + Instruction::I64LtSAssignImm32(instr) | + Instruction::I64LeSAssignImm32(instr) | + Instruction::I64GtSAssignImm32(instr) | + Instruction::I64GeSAssignImm32(instr) => instr.result_mut(), + Instruction::I64LtUAssignImm32(instr) | + Instruction::I64GtUAssignImm32(instr) | + Instruction::I64LeUAssignImm32(instr) | + Instruction::I64GeUAssignImm32(instr) => instr.result_mut(), + Instruction::F32EqAssignImm(instr) | + Instruction::F32NeAssignImm(instr) | + Instruction::F32LtAssignImm(instr) | + Instruction::F32LeAssignImm(instr) | + Instruction::F32GtAssignImm(instr) | + Instruction::F32GeAssignImm(instr) => instr.result_mut(), + Instruction::F64EqAssignImm32(instr) | + Instruction::F64NeAssignImm32(instr) | + Instruction::F64LtAssignImm32(instr) | + Instruction::F64LeAssignImm32(instr) | + Instruction::F64GtAssignImm32(instr) | + Instruction::F64GeAssignImm32(instr) => instr.result_mut(), + Instruction::I32AddAssignImm(instr) | + Instruction::I32SubAssignImm(instr) | + Instruction::I32MulAssignImm(instr) | + Instruction::I32DivSAssignImm(instr) | + Instruction::I32RemSAssignImm(instr) | + Instruction::I32AndAssignImm(instr) | + Instruction::I32OrAssignImm(instr) | + Instruction::I32XorAssignImm(instr) | + Instruction::I32ShlAssignImm(instr) | + Instruction::I32ShrSAssignImm(instr) | + Instruction::I32RotlAssignImm(instr) | + Instruction::I32RotrAssignImm(instr) => instr.result_mut(), + Instruction::I32DivUAssignImm(instr) | + Instruction::I32RemUAssignImm(instr) | + Instruction::I32ShrUAssignImm(instr) => instr.result_mut(), + Instruction::I64AddAssignImm32(instr) | + Instruction::I64SubAssignImm32(instr) | + Instruction::I64MulAssignImm32(instr) | + Instruction::I64DivSAssignImm32(instr) | + Instruction::I64RemSAssignImm32(instr) | + Instruction::I64AndAssignImm32(instr) | + Instruction::I64OrAssignImm32(instr) | + Instruction::I64XorAssignImm32(instr) | + Instruction::I64ShlAssignImm32(instr) | + Instruction::I64ShrSAssignImm32(instr) | + Instruction::I64RotlAssignImm32(instr) | + Instruction::I64RotrAssignImm32(instr) => instr.result_mut(), + Instruction::I64DivUAssignImm32(instr) | + Instruction::I64RemUAssignImm32(instr) | + Instruction::I64ShrUAssignImm32(instr) => instr.result_mut(), + Instruction::F32AddAssignImm(instr) | + Instruction::F32SubAssignImm(instr) | + Instruction::F32MulAssignImm(instr) | + Instruction::F32DivAssignImm(instr) | + Instruction::F32MinAssignImm(instr) | + Instruction::F32MaxAssignImm(instr) | + Instruction::F32CopysignAssignImm(instr) => instr.result_mut(), + Instruction::F64AddAssignImm32(instr) | + Instruction::F64SubAssignImm32(instr) | + Instruction::F64MulAssignImm32(instr) | + Instruction::F64DivAssignImm32(instr) | + Instruction::F64MinAssignImm32(instr) | + Instruction::F64MaxAssignImm32(instr) | + Instruction::F64CopysignAssignImm32(instr) => instr.result_mut(), } } } @@ -560,6 +714,20 @@ fn call_indirect_result_mut<'a>( None } +impl BinAssignInstr { + /// Returns the single `result` [`Register`] of the [`BinAssignInstr`] if any. + pub fn result_mut(&mut self) -> Option<&mut Register> { + Some(&mut self.inout) + } +} + +impl BinAssignInstrImm32 { + /// Returns the single `result` [`Register`] of the [`BinAssignInstrImm32`] if any. + pub fn result_mut(&mut self) -> Option<&mut Register> { + Some(&mut self.inout) + } +} + impl LoadInstr { /// Returns the single `result` [`Register`] of the [`LoadInstr`] if any. pub fn result_mut(&mut self) -> Option<&mut Register> { diff --git a/crates/wasmi/src/engine/regmach/translator/visit_register.rs b/crates/wasmi/src/engine/regmach/translator/visit_register.rs index c8dd1ea489..c70b35958a 100644 --- a/crates/wasmi/src/engine/regmach/translator/visit_register.rs +++ b/crates/wasmi/src/engine/regmach/translator/visit_register.rs @@ -1,4 +1,6 @@ use crate::engine::regmach::bytecode::{ + BinAssignInstr, + BinAssignInstrImm32, BinInstr, BinInstrImm16, BranchBinOpInstr, @@ -82,7 +84,6 @@ impl VisitInputRegisters for Instruction { Instruction::BranchEqz { condition, .. } | Instruction::BranchNez { condition, .. } => f(condition), Instruction::BranchTable { index, .. } => f(index), - Instruction::BranchI32Eq(instr) => instr.visit_input_registers(f), Instruction::BranchI32EqImm(instr) => instr.visit_input_registers(f), Instruction::BranchI32Ne(instr) => instr.visit_input_registers(f), @@ -135,7 +136,6 @@ impl VisitInputRegisters for Instruction { Instruction::BranchF64Le(instr) => instr.visit_input_registers(f), Instruction::BranchF64Gt(instr) => instr.visit_input_registers(f), Instruction::BranchF64Ge(instr) => instr.visit_input_registers(f), - Instruction::Copy { result, value } => { // Note: for copy instruction unlike all other instructions // we need to also visit the result register since @@ -530,6 +530,160 @@ impl VisitInputRegisters for Instruction { Instruction::F64ConvertI32U(instr) => instr.visit_input_registers(f), Instruction::F64ConvertI64S(instr) => instr.visit_input_registers(f), Instruction::F64ConvertI64U(instr) => instr.visit_input_registers(f), + Instruction::I32EqAssign(instr) => instr.visit_input_registers(f), + Instruction::I32EqAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32NeAssign(instr) => instr.visit_input_registers(f), + Instruction::I32NeAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32LtSAssign(instr) => instr.visit_input_registers(f), + Instruction::I32LtSAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32LtUAssign(instr) => instr.visit_input_registers(f), + Instruction::I32LtUAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32GtSAssign(instr) => instr.visit_input_registers(f), + Instruction::I32GtSAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32GtUAssign(instr) => instr.visit_input_registers(f), + Instruction::I32GtUAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32LeSAssign(instr) => instr.visit_input_registers(f), + Instruction::I32LeSAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32LeUAssign(instr) => instr.visit_input_registers(f), + Instruction::I32LeUAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32GeSAssign(instr) => instr.visit_input_registers(f), + Instruction::I32GeSAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32GeUAssign(instr) => instr.visit_input_registers(f), + Instruction::I32GeUAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I64EqAssign(instr) => instr.visit_input_registers(f), + Instruction::I64EqAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64NeAssign(instr) => instr.visit_input_registers(f), + Instruction::I64NeAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64LtSAssign(instr) => instr.visit_input_registers(f), + Instruction::I64LtSAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64LtUAssign(instr) => instr.visit_input_registers(f), + Instruction::I64LtUAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64GtSAssign(instr) => instr.visit_input_registers(f), + Instruction::I64GtSAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64GtUAssign(instr) => instr.visit_input_registers(f), + Instruction::I64GtUAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64LeSAssign(instr) => instr.visit_input_registers(f), + Instruction::I64LeSAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64LeUAssign(instr) => instr.visit_input_registers(f), + Instruction::I64LeUAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64GeSAssign(instr) => instr.visit_input_registers(f), + Instruction::I64GeSAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64GeUAssign(instr) => instr.visit_input_registers(f), + Instruction::I64GeUAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F32EqAssign(instr) => instr.visit_input_registers(f), + Instruction::F32EqAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32NeAssign(instr) => instr.visit_input_registers(f), + Instruction::F32NeAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32LtAssign(instr) => instr.visit_input_registers(f), + Instruction::F32LtAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32LeAssign(instr) => instr.visit_input_registers(f), + Instruction::F32LeAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32GtAssign(instr) => instr.visit_input_registers(f), + Instruction::F32GtAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32GeAssign(instr) => instr.visit_input_registers(f), + Instruction::F32GeAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F64EqAssign(instr) => instr.visit_input_registers(f), + Instruction::F64EqAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64NeAssign(instr) => instr.visit_input_registers(f), + Instruction::F64NeAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64LtAssign(instr) => instr.visit_input_registers(f), + Instruction::F64LtAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64LeAssign(instr) => instr.visit_input_registers(f), + Instruction::F64LeAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64GtAssign(instr) => instr.visit_input_registers(f), + Instruction::F64GtAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64GeAssign(instr) => instr.visit_input_registers(f), + Instruction::F64GeAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I32AddAssign(instr) => instr.visit_input_registers(f), + Instruction::I32AddAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32SubAssign(instr) => instr.visit_input_registers(f), + Instruction::I32SubAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32MulAssign(instr) => instr.visit_input_registers(f), + Instruction::I32MulAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32DivSAssign(instr) => instr.visit_input_registers(f), + Instruction::I32DivSAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32DivUAssign(instr) => instr.visit_input_registers(f), + Instruction::I32DivUAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32RemSAssign(instr) => instr.visit_input_registers(f), + Instruction::I32RemSAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32RemUAssign(instr) => instr.visit_input_registers(f), + Instruction::I32RemUAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32AndAssign(instr) => instr.visit_input_registers(f), + Instruction::I32AndAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32OrAssign(instr) => instr.visit_input_registers(f), + Instruction::I32OrAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32XorAssign(instr) => instr.visit_input_registers(f), + Instruction::I32XorAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32ShlAssign(instr) => instr.visit_input_registers(f), + Instruction::I32ShlAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32ShrUAssign(instr) => instr.visit_input_registers(f), + Instruction::I32ShrUAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32ShrSAssign(instr) => instr.visit_input_registers(f), + Instruction::I32ShrSAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32RotlAssign(instr) => instr.visit_input_registers(f), + Instruction::I32RotlAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I32RotrAssign(instr) => instr.visit_input_registers(f), + Instruction::I32RotrAssignImm(instr) => instr.visit_input_registers(f), + Instruction::I64AddAssign(instr) => instr.visit_input_registers(f), + Instruction::I64AddAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64SubAssign(instr) => instr.visit_input_registers(f), + Instruction::I64SubAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64MulAssign(instr) => instr.visit_input_registers(f), + Instruction::I64MulAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64DivSAssign(instr) => instr.visit_input_registers(f), + Instruction::I64DivSAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64DivUAssign(instr) => instr.visit_input_registers(f), + Instruction::I64DivUAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64RemSAssign(instr) => instr.visit_input_registers(f), + Instruction::I64RemSAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64RemUAssign(instr) => instr.visit_input_registers(f), + Instruction::I64RemUAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64AndAssign(instr) => instr.visit_input_registers(f), + Instruction::I64AndAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64OrAssign(instr) => instr.visit_input_registers(f), + Instruction::I64OrAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64XorAssign(instr) => instr.visit_input_registers(f), + Instruction::I64XorAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64ShlAssign(instr) => instr.visit_input_registers(f), + Instruction::I64ShlAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64ShrUAssign(instr) => instr.visit_input_registers(f), + Instruction::I64ShrUAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64ShrSAssign(instr) => instr.visit_input_registers(f), + Instruction::I64ShrSAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64RotlAssign(instr) => instr.visit_input_registers(f), + Instruction::I64RotlAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::I64RotrAssign(instr) => instr.visit_input_registers(f), + Instruction::I64RotrAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F32AddAssign(instr) => instr.visit_input_registers(f), + Instruction::F32AddAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32SubAssign(instr) => instr.visit_input_registers(f), + Instruction::F32SubAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32MulAssign(instr) => instr.visit_input_registers(f), + Instruction::F32MulAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32DivAssign(instr) => instr.visit_input_registers(f), + Instruction::F32DivAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32MinAssign(instr) => instr.visit_input_registers(f), + Instruction::F32MinAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32MaxAssign(instr) => instr.visit_input_registers(f), + Instruction::F32MaxAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F32CopysignAssign(instr) => instr.visit_input_registers(f), + Instruction::F32CopysignAssignImm(instr) => instr.visit_input_registers(f), + Instruction::F64AddAssign(instr) => instr.visit_input_registers(f), + Instruction::F64AddAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64SubAssign(instr) => instr.visit_input_registers(f), + Instruction::F64SubAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64MulAssign(instr) => instr.visit_input_registers(f), + Instruction::F64MulAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64DivAssign(instr) => instr.visit_input_registers(f), + Instruction::F64DivAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64MinAssign(instr) => instr.visit_input_registers(f), + Instruction::F64MinAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64MaxAssign(instr) => instr.visit_input_registers(f), + Instruction::F64MaxAssignImm32(instr) => instr.visit_input_registers(f), + Instruction::F64CopysignAssign(instr) => instr.visit_input_registers(f), + Instruction::F64CopysignAssignImm32(instr) => instr.visit_input_registers(f), + + } } } @@ -542,6 +696,18 @@ impl VisitInputRegisters for [Register; N] { } } +impl BinAssignInstr { + fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Register)) { + f(&mut self.rhs); + } +} + +impl BinAssignInstrImm32 { + fn visit_input_registers(&mut self, _f: impl FnMut(&mut Register)) { + // Nothing to do. + } +} + impl LoadInstr { fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Register)) { f(&mut self.ptr); From f15971cb186ac24e89aa2a5e5d054b7618967a6f Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 22 Nov 2023 14:20:54 +0100 Subject: [PATCH 04/17] simplify where bounds --- crates/wasmi/src/engine/regmach/bytecode/utils.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/bytecode/utils.rs b/crates/wasmi/src/engine/regmach/bytecode/utils.rs index 0475c9f5b0..02b6e92a34 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/utils.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/utils.rs @@ -276,7 +276,7 @@ pub struct BinInstrImm16 { impl fmt::Debug for BinInstrImm16 where - T: fmt::Debug + From>, + Const16: fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("BinInstrImm16") @@ -333,7 +333,7 @@ impl Eq for BinAssignInstrImm32 {} impl fmt::Debug for BinAssignInstrImm32 where - T: fmt::Debug + From>, + Const32: fmt::Debug, { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { f.debug_struct("BinAssignInstrImm32") From 2c3d429af968e8fcd6a06ccaf7bf6b0aa1b4ddea Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Wed, 22 Nov 2023 15:10:31 +0100 Subject: [PATCH 05/17] add execution implementation for new instructions --- .../src/engine/regmach/executor/instrs.rs | 213 +++++++++++++++- .../engine/regmach/executor/instrs/binary.rs | 241 +++++++++++++++++- 2 files changed, 452 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/executor/instrs.rs b/crates/wasmi/src/engine/regmach/executor/instrs.rs index 328cd79ce0..9639c841d8 100644 --- a/crates/wasmi/src/engine/regmach/executor/instrs.rs +++ b/crates/wasmi/src/engine/regmach/executor/instrs.rs @@ -9,9 +9,12 @@ use crate::{ regmach::{ bytecode::{ AnyConst32, + BinAssignInstr, + BinAssignInstrImm32, BinInstr, BinInstrImm16, Const16, + Const32, Instruction, Register, RegisterSpan, @@ -835,7 +838,160 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { Instr::F64ConvertI32U(instr) => self.execute_f64_convert_i32_u(instr), Instr::F64ConvertI64S(instr) => self.execute_f64_convert_i64_s(instr), Instr::F64ConvertI64U(instr) => self.execute_f64_convert_i64_u(instr), - _ => todo!(), + Instr::I32EqAssign(instr) => self.execute_i32_eq_assign(instr), + Instr::I32NeAssign(instr) => self.execute_i32_ne_assign(instr), + Instr::I32LtSAssign(instr) => self.execute_i32_lt_s_assign(instr), + Instr::I32LtUAssign(instr) => self.execute_i32_lt_u_assign(instr), + Instr::I32LeSAssign(instr) => self.execute_i32_le_s_assign(instr), + Instr::I32LeUAssign(instr) => self.execute_i32_le_u_assign(instr), + Instr::I32GtSAssign(instr) => self.execute_i32_gt_s_assign(instr), + Instr::I32GtUAssign(instr) => self.execute_i32_gt_u_assign(instr), + Instr::I32GeSAssign(instr) => self.execute_i32_ge_s_assign(instr), + Instr::I32GeUAssign(instr) => self.execute_i32_ge_u_assign(instr), + Instr::I64EqAssign(instr) => self.execute_i64_eq_assign(instr), + Instr::I64NeAssign(instr) => self.execute_i64_ne_assign(instr), + Instr::I64LtSAssign(instr) => self.execute_i64_lt_s_assign(instr), + Instr::I64LtUAssign(instr) => self.execute_i64_lt_u_assign(instr), + Instr::I64LeSAssign(instr) => self.execute_i64_le_s_assign(instr), + Instr::I64LeUAssign(instr) => self.execute_i64_le_u_assign(instr), + Instr::I64GtSAssign(instr) => self.execute_i64_gt_s_assign(instr), + Instr::I64GtUAssign(instr) => self.execute_i64_gt_u_assign(instr), + Instr::I64GeSAssign(instr) => self.execute_i64_ge_s_assign(instr), + Instr::I64GeUAssign(instr) => self.execute_i64_ge_u_assign(instr), + Instr::F32EqAssign(instr) => self.execute_f32_eq_assign(instr), + Instr::F32NeAssign(instr) => self.execute_f32_ne_assign(instr), + Instr::F32LtAssign(instr) => self.execute_f32_lt_assign(instr), + Instr::F32LeAssign(instr) => self.execute_f32_le_assign(instr), + Instr::F32GtAssign(instr) => self.execute_f32_gt_assign(instr), + Instr::F32GeAssign(instr) => self.execute_f32_ge_assign(instr), + Instr::F64EqAssign(instr) => self.execute_f64_eq_assign(instr), + Instr::F64NeAssign(instr) => self.execute_f64_ne_assign(instr), + Instr::F64LtAssign(instr) => self.execute_f64_lt_assign(instr), + Instr::F64LeAssign(instr) => self.execute_f64_le_assign(instr), + Instr::F64GtAssign(instr) => self.execute_f64_gt_assign(instr), + Instr::F64GeAssign(instr) => self.execute_f64_ge_assign(instr), + Instr::I32AddAssign(instr) => self.execute_i32_add_assign(instr), + Instr::I32SubAssign(instr) => self.execute_i32_sub_assign(instr), + Instr::I32MulAssign(instr) => self.execute_i32_mul_assign(instr), + Instr::I32DivSAssign(instr) => self.execute_i32_div_s_assign(instr)?, + Instr::I32DivUAssign(instr) => self.execute_i32_div_u_assign(instr)?, + Instr::I32RemSAssign(instr) => self.execute_i32_rem_s_assign(instr)?, + Instr::I32RemUAssign(instr) => self.execute_i32_rem_u_assign(instr)?, + Instr::I32AndAssign(instr) => self.execute_i32_and_assign(instr), + Instr::I32OrAssign(instr) => self.execute_i32_or_assign(instr), + Instr::I32XorAssign(instr) => self.execute_i32_xor_assign(instr), + Instr::I32ShlAssign(instr) => self.execute_i32_shl_assign(instr), + Instr::I32ShrSAssign(instr) => self.execute_i32_shr_s_assign(instr), + Instr::I32ShrUAssign(instr) => self.execute_i32_shr_u_assign(instr), + Instr::I32RotlAssign(instr) => self.execute_i32_rotl_assign(instr), + Instr::I32RotrAssign(instr) => self.execute_i32_rotr_assign(instr), + Instr::I64AddAssign(instr) => self.execute_i64_add_assign(instr), + Instr::I64SubAssign(instr) => self.execute_i64_sub_assign(instr), + Instr::I64MulAssign(instr) => self.execute_i64_mul_assign(instr), + Instr::I64DivSAssign(instr) => self.execute_i64_div_s_assign(instr)?, + Instr::I64DivUAssign(instr) => self.execute_i64_div_u_assign(instr)?, + Instr::I64RemSAssign(instr) => self.execute_i64_rem_s_assign(instr)?, + Instr::I64RemUAssign(instr) => self.execute_i64_rem_u_assign(instr)?, + Instr::I64AndAssign(instr) => self.execute_i64_and_assign(instr), + Instr::I64OrAssign(instr) => self.execute_i64_or_assign(instr), + Instr::I64XorAssign(instr) => self.execute_i64_xor_assign(instr), + Instr::I64ShlAssign(instr) => self.execute_i64_shl_assign(instr), + Instr::I64ShrSAssign(instr) => self.execute_i64_shr_s_assign(instr), + Instr::I64ShrUAssign(instr) => self.execute_i64_shr_u_assign(instr), + Instr::I64RotlAssign(instr) => self.execute_i64_rotl_assign(instr), + Instr::I64RotrAssign(instr) => self.execute_i64_rotr_assign(instr), + Instr::F32AddAssign(instr) => self.execute_f32_add_assign(instr), + Instr::F32SubAssign(instr) => self.execute_f32_sub_assign(instr), + Instr::F32MulAssign(instr) => self.execute_f32_mul_assign(instr), + Instr::F32DivAssign(instr) => self.execute_f32_div_assign(instr), + Instr::F32MinAssign(instr) => self.execute_f32_min_assign(instr), + Instr::F32MaxAssign(instr) => self.execute_f32_max_assign(instr), + Instr::F32CopysignAssign(instr) => self.execute_f32_copysign_assign(instr), + Instr::F64AddAssign(instr) => self.execute_f64_add_assign(instr), + Instr::F64SubAssign(instr) => self.execute_f64_sub_assign(instr), + Instr::F64MulAssign(instr) => self.execute_f64_mul_assign(instr), + Instr::F64DivAssign(instr) => self.execute_f64_div_assign(instr), + Instr::F64MinAssign(instr) => self.execute_f64_min_assign(instr), + Instr::F64MaxAssign(instr) => self.execute_f64_max_assign(instr), + Instr::F64CopysignAssign(instr) => self.execute_f64_copysign_assign(instr), + Instr::I32EqAssignImm(instr) => self.execute_i32_eq_assign_imm(instr), + Instr::I32NeAssignImm(instr) => self.execute_i32_ne_assign_imm(instr), + Instr::I32LtSAssignImm(instr) => self.execute_i32_lt_s_assign_imm(instr), + Instr::I32LtUAssignImm(instr) => self.execute_i32_lt_u_assign_imm(instr), + Instr::I32LeSAssignImm(instr) => self.execute_i32_le_s_assign_imm(instr), + Instr::I32LeUAssignImm(instr) => self.execute_i32_le_u_assign_imm(instr), + Instr::I32GtSAssignImm(instr) => self.execute_i32_gt_s_assign_imm(instr), + Instr::I32GtUAssignImm(instr) => self.execute_i32_gt_u_assign_imm(instr), + Instr::I32GeSAssignImm(instr) => self.execute_i32_ge_s_assign_imm(instr), + Instr::I32GeUAssignImm(instr) => self.execute_i32_ge_u_assign_imm(instr), + Instr::I64EqAssignImm32(instr) => self.execute_i64_eq_assign_imm32(instr), + Instr::I64NeAssignImm32(instr) => self.execute_i64_ne_assign_imm32(instr), + Instr::I64LtSAssignImm32(instr) => self.execute_i64_lt_s_assign_imm32(instr), + Instr::I64LtUAssignImm32(instr) => self.execute_i64_lt_u_assign_imm32(instr), + Instr::I64LeSAssignImm32(instr) => self.execute_i64_le_s_assign_imm32(instr), + Instr::I64LeUAssignImm32(instr) => self.execute_i64_le_u_assign_imm32(instr), + Instr::I64GtSAssignImm32(instr) => self.execute_i64_gt_s_assign_imm32(instr), + Instr::I64GtUAssignImm32(instr) => self.execute_i64_gt_u_assign_imm32(instr), + Instr::I64GeSAssignImm32(instr) => self.execute_i64_ge_s_assign_imm32(instr), + Instr::I64GeUAssignImm32(instr) => self.execute_i64_ge_u_assign_imm32(instr), + Instr::F32EqAssignImm(instr) => self.execute_f32_eq_assign_imm(instr), + Instr::F32NeAssignImm(instr) => self.execute_f32_ne_assign_imm(instr), + Instr::F32LtAssignImm(instr) => self.execute_f32_lt_assign_imm(instr), + Instr::F32LeAssignImm(instr) => self.execute_f32_le_assign_imm(instr), + Instr::F32GtAssignImm(instr) => self.execute_f32_gt_assign_imm(instr), + Instr::F32GeAssignImm(instr) => self.execute_f32_ge_assign_imm(instr), + Instr::F64EqAssignImm32(instr) => self.execute_f64_eq_assign_imm32(instr), + Instr::F64NeAssignImm32(instr) => self.execute_f64_ne_assign_imm32(instr), + Instr::F64LtAssignImm32(instr) => self.execute_f64_lt_assign_imm32(instr), + Instr::F64LeAssignImm32(instr) => self.execute_f64_le_assign_imm32(instr), + Instr::F64GtAssignImm32(instr) => self.execute_f64_gt_assign_imm32(instr), + Instr::F64GeAssignImm32(instr) => self.execute_f64_ge_assign_imm32(instr), + Instr::I32AddAssignImm(instr) => self.execute_i32_add_assign_imm(instr), + Instr::I32SubAssignImm(instr) => self.execute_i32_sub_assign_imm(instr), + Instr::I32MulAssignImm(instr) => self.execute_i32_mul_assign_imm(instr), + Instr::I32DivSAssignImm(instr) => self.execute_i32_div_s_assign_imm(instr)?, + Instr::I32DivUAssignImm(instr) => self.execute_i32_div_u_assign_imm(instr)?, + Instr::I32RemSAssignImm(instr) => self.execute_i32_rem_s_assign_imm(instr)?, + Instr::I32RemUAssignImm(instr) => self.execute_i32_rem_u_assign_imm(instr)?, + Instr::I32AndAssignImm(instr) => self.execute_i32_and_assign_imm(instr), + Instr::I32OrAssignImm(instr) => self.execute_i32_or_assign_imm(instr), + Instr::I32XorAssignImm(instr) => self.execute_i32_xor_assign_imm(instr), + Instr::I32ShlAssignImm(instr) => self.execute_i32_shl_assign_imm(instr), + Instr::I32ShrSAssignImm(instr) => self.execute_i32_shr_s_assign_imm(instr), + Instr::I32ShrUAssignImm(instr) => self.execute_i32_shr_u_assign_imm(instr), + Instr::I32RotlAssignImm(instr) => self.execute_i32_rotl_assign_imm(instr), + Instr::I32RotrAssignImm(instr) => self.execute_i32_rotr_assign_imm(instr), + Instr::I64AddAssignImm32(instr) => self.execute_i64_add_assign_imm32(instr), + Instr::I64SubAssignImm32(instr) => self.execute_i64_sub_assign_imm32(instr), + Instr::I64MulAssignImm32(instr) => self.execute_i64_mul_assign_imm32(instr), + Instr::I64DivSAssignImm32(instr) => self.execute_i64_div_s_assign_imm32(instr)?, + Instr::I64DivUAssignImm32(instr) => self.execute_i64_div_u_assign_imm32(instr)?, + Instr::I64RemSAssignImm32(instr) => self.execute_i64_rem_s_assign_imm32(instr)?, + Instr::I64RemUAssignImm32(instr) => self.execute_i64_rem_u_assign_imm32(instr)?, + Instr::I64AndAssignImm32(instr) => self.execute_i64_and_assign_imm32(instr), + Instr::I64OrAssignImm32(instr) => self.execute_i64_or_assign_imm32(instr), + Instr::I64XorAssignImm32(instr) => self.execute_i64_xor_assign_imm32(instr), + Instr::I64ShlAssignImm32(instr) => self.execute_i64_shl_assign_imm32(instr), + Instr::I64ShrSAssignImm32(instr) => self.execute_i64_shr_s_assign_imm32(instr), + Instr::I64ShrUAssignImm32(instr) => self.execute_i64_shr_u_assign_imm32(instr), + Instr::I64RotlAssignImm32(instr) => self.execute_i64_rotl_assign_imm32(instr), + Instr::I64RotrAssignImm32(instr) => self.execute_i64_rotr_assign_imm32(instr), + Instr::F32AddAssignImm(instr) => self.execute_f32_add_assign_imm(instr), + Instr::F32SubAssignImm(instr) => self.execute_f32_sub_assign_imm(instr), + Instr::F32MulAssignImm(instr) => self.execute_f32_mul_assign_imm(instr), + Instr::F32DivAssignImm(instr) => self.execute_f32_div_assign_imm(instr), + Instr::F32MinAssignImm(instr) => self.execute_f32_min_assign_imm(instr), + Instr::F32MaxAssignImm(instr) => self.execute_f32_max_assign_imm(instr), + Instr::F32CopysignAssignImm(instr) => self.execute_f32_copysign_assign_imm(instr), + Instr::F64AddAssignImm32(instr) => self.execute_f64_add_assign_imm32(instr), + Instr::F64SubAssignImm32(instr) => self.execute_f64_sub_assign_imm32(instr), + Instr::F64MulAssignImm32(instr) => self.execute_f64_mul_assign_imm32(instr), + Instr::F64DivAssignImm32(instr) => self.execute_f64_div_assign_imm32(instr), + Instr::F64MinAssignImm32(instr) => self.execute_f64_min_assign_imm32(instr), + Instr::F64MaxAssignImm32(instr) => self.execute_f64_max_assign_imm32(instr), + Instr::F64CopysignAssignImm32(instr) => { + self.execute_f64_copysign_assign_imm32(instr) + } } } } @@ -1194,6 +1350,61 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { self.set_register(instr.result, op(lhs, rhs)?); self.try_next_instr() } + + /// Executes a generic binary op-assign [`Instruction`]. + fn execute_binary_assign( + &mut self, + instr: BinAssignInstr, + op: fn(UntypedValue, UntypedValue) -> UntypedValue, + ) { + let lhs = self.get_register(instr.inout); + let rhs = self.get_register(instr.rhs); + self.set_register(instr.inout, op(lhs, rhs)); + self.next_instr(); + } + + /// Executes a fallible generic binary op-assign [`Instruction`]. + fn try_execute_binary_assign( + &mut self, + instr: BinAssignInstr, + op: fn(UntypedValue, UntypedValue) -> Result, + ) -> Result<(), TrapCode> { + let lhs = self.get_register(instr.inout); + let rhs = self.get_register(instr.rhs); + self.set_register(instr.inout, op(lhs, rhs)?); + self.try_next_instr() + } + + /// Executes a generic binary op-assign [`Instruction`] with 32-bit encoded immediate value. + fn execute_binary_assign_imm32( + &mut self, + instr: BinAssignInstrImm32, + op: fn(UntypedValue, UntypedValue) -> UntypedValue, + ) where + T: From>, + UntypedValue: From, + { + let lhs = self.get_register(instr.inout); + let rhs = UntypedValue::from(::from(instr.rhs)); + self.set_register(instr.inout, op(lhs, rhs)); + self.next_instr(); + } + + /// Executes a fallible generic binary op-assign [`Instruction`] with 32-bit encoded immediate value. + fn try_execute_binary_assign_imm32( + &mut self, + instr: BinAssignInstrImm32, + op: fn(UntypedValue, UntypedValue) -> Result, + ) -> Result<(), TrapCode> + where + T: From>, + UntypedValue: From, + { + let lhs = self.get_register(instr.inout); + let rhs = UntypedValue::from(::from(instr.rhs)); + self.set_register(instr.inout, op(lhs, rhs)?); + self.try_next_instr() + } } impl<'ctx, 'engine> Executor<'ctx, 'engine> { diff --git a/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs b/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs index d1695dd9c4..eba060e2d0 100644 --- a/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs +++ b/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs @@ -1,7 +1,14 @@ use super::Executor; use crate::{ core::{TrapCode, UntypedValue}, - engine::regmach::bytecode::{BinInstr, BinInstrImm16, CopysignImmInstr, Sign}, + engine::regmach::bytecode::{ + BinAssignInstr, + BinAssignInstrImm32, + BinInstr, + BinInstrImm16, + CopysignImmInstr, + Sign, + }, }; #[cfg(doc)] @@ -236,3 +243,235 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { self.next_instr() } } + +macro_rules! impl_binary_assign { + ( $( (Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { + $( + #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")] + #[inline(always)] + pub fn $fn_name(&mut self, instr: BinAssignInstr) { + self.execute_binary_assign(instr, $op) + } + )* + }; +} +impl<'ctx, 'engine> Executor<'ctx, 'engine> { + impl_binary_assign! { + (Instruction::I32EqAssign, execute_i32_eq_assign, UntypedValue::i32_eq), + (Instruction::I32NeAssign, execute_i32_ne_assign, UntypedValue::i32_ne), + (Instruction::I32LtSAssign, execute_i32_lt_s_assign, UntypedValue::i32_lt_s), + (Instruction::I32LtUAssign, execute_i32_lt_u_assign, UntypedValue::i32_lt_u), + (Instruction::I32LeSAssign, execute_i32_le_s_assign, UntypedValue::i32_le_s), + (Instruction::I32LeUAssign, execute_i32_le_u_assign, UntypedValue::i32_le_u), + (Instruction::I32GtSAssign, execute_i32_gt_s_assign, UntypedValue::i32_gt_s), + (Instruction::I32GtUAssign, execute_i32_gt_u_assign, UntypedValue::i32_gt_u), + (Instruction::I32GeSAssign, execute_i32_ge_s_assign, UntypedValue::i32_ge_s), + (Instruction::I32GeUAssign, execute_i32_ge_u_assign, UntypedValue::i32_ge_u), + + (Instruction::I64EqAssign, execute_i64_eq_assign, UntypedValue::i64_eq), + (Instruction::I64NeAssign, execute_i64_ne_assign, UntypedValue::i64_ne), + (Instruction::I64LtSAssign, execute_i64_lt_s_assign, UntypedValue::i64_lt_s), + (Instruction::I64LtUAssign, execute_i64_lt_u_assign, UntypedValue::i64_lt_u), + (Instruction::I64LeSAssign, execute_i64_le_s_assign, UntypedValue::i64_le_s), + (Instruction::I64LeUAssign, execute_i64_le_u_assign, UntypedValue::i64_le_u), + (Instruction::I64GtSAssign, execute_i64_gt_s_assign, UntypedValue::i64_gt_s), + (Instruction::I64GtUAssign, execute_i64_gt_u_assign, UntypedValue::i64_gt_u), + (Instruction::I64GeSAssign, execute_i64_ge_s_assign, UntypedValue::i64_ge_s), + (Instruction::I64GeUAssign, execute_i64_ge_u_assign, UntypedValue::i64_ge_u), + + (Instruction::F32EqAssign, execute_f32_eq_assign, UntypedValue::f32_eq), + (Instruction::F32NeAssign, execute_f32_ne_assign, UntypedValue::f32_ne), + (Instruction::F32LtAssign, execute_f32_lt_assign, UntypedValue::f32_lt), + (Instruction::F32LeAssign, execute_f32_le_assign, UntypedValue::f32_le), + (Instruction::F32GtAssign, execute_f32_gt_assign, UntypedValue::f32_gt), + (Instruction::F32GeAssign, execute_f32_ge_assign, UntypedValue::f32_ge), + + (Instruction::F64EqAssign, execute_f64_eq_assign, UntypedValue::f64_eq), + (Instruction::F64NeAssign, execute_f64_ne_assign, UntypedValue::f64_ne), + (Instruction::F64LtAssign, execute_f64_lt_assign, UntypedValue::f64_lt), + (Instruction::F64LeAssign, execute_f64_le_assign, UntypedValue::f64_le), + (Instruction::F64GtAssign, execute_f64_gt_assign, UntypedValue::f64_gt), + (Instruction::F64GeAssign, execute_f64_ge_assign, UntypedValue::f64_ge), + + (Instruction::I32AddAssign, execute_i32_add_assign, UntypedValue::i32_add), + (Instruction::I32SubAssign, execute_i32_sub_assign, UntypedValue::i32_sub), + (Instruction::I32MulAssign, execute_i32_mul_assign, UntypedValue::i32_mul), + (Instruction::I32AndAssign, execute_i32_and_assign, UntypedValue::i32_and), + (Instruction::I32OrAssign, execute_i32_or_assign, UntypedValue::i32_or), + (Instruction::I32XorAssign, execute_i32_xor_assign, UntypedValue::i32_xor), + (Instruction::I32ShlAssign, execute_i32_shl_assign, UntypedValue::i32_shl), + (Instruction::I32ShrSAssign, execute_i32_shr_s_assign, UntypedValue::i32_shr_s), + (Instruction::I32ShrUAssign, execute_i32_shr_u_assign, UntypedValue::i32_shr_u), + (Instruction::I32RotlAssign, execute_i32_rotl_assign, UntypedValue::i32_rotl), + (Instruction::I32RotrAssign, execute_i32_rotr_assign, UntypedValue::i32_rotr), + + (Instruction::I64AddAssign, execute_i64_add_assign, UntypedValue::i64_add), + (Instruction::I64SubAssign, execute_i64_sub_assign, UntypedValue::i64_sub), + (Instruction::I64MulAssign, execute_i64_mul_assign, UntypedValue::i64_mul), + (Instruction::I64AndAssign, execute_i64_and_assign, UntypedValue::i64_and), + (Instruction::I64OrAssign, execute_i64_or_assign, UntypedValue::i64_or), + (Instruction::I64XorAssign, execute_i64_xor_assign, UntypedValue::i64_xor), + (Instruction::I64ShlAssign, execute_i64_shl_assign, UntypedValue::i64_shl), + (Instruction::I64ShrSAssign, execute_i64_shr_s_assign, UntypedValue::i64_shr_s), + (Instruction::I64ShrUAssign, execute_i64_shr_u_assign, UntypedValue::i64_shr_u), + (Instruction::I64RotlAssign, execute_i64_rotl_assign, UntypedValue::i64_rotl), + (Instruction::I64RotrAssign, execute_i64_rotr_assign, UntypedValue::i64_rotr), + + (Instruction::F32AddAssign, execute_f32_add_assign, UntypedValue::f32_add), + (Instruction::F32SubAssign, execute_f32_sub_assign, UntypedValue::f32_sub), + (Instruction::F32MulAssign, execute_f32_mul_assign, UntypedValue::f32_mul), + (Instruction::F32DivAssign, execute_f32_div_assign, UntypedValue::f32_div), + (Instruction::F32MinAssign, execute_f32_min_assign, UntypedValue::f32_min), + (Instruction::F32MaxAssign, execute_f32_max_assign, UntypedValue::f32_max), + (Instruction::F32CopysignAssign, execute_f32_copysign_assign, UntypedValue::f32_copysign), + + (Instruction::F64AddAssign, execute_f64_add_assign, UntypedValue::f64_add), + (Instruction::F64SubAssign, execute_f64_sub_assign, UntypedValue::f64_sub), + (Instruction::F64MulAssign, execute_f64_mul_assign, UntypedValue::f64_mul), + (Instruction::F64DivAssign, execute_f64_div_assign, UntypedValue::f64_div), + (Instruction::F64MinAssign, execute_f64_min_assign, UntypedValue::f64_min), + (Instruction::F64MaxAssign, execute_f64_max_assign, UntypedValue::f64_max), + (Instruction::F64CopysignAssign, execute_f64_copysign_assign, UntypedValue::f64_copysign), + } +} + +macro_rules! impl_fallible_binary_assign { + ( $( (Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { + $( + #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")] + #[inline(always)] + pub fn $fn_name(&mut self, instr: BinAssignInstr) -> Result<(), TrapCode> { + self.try_execute_binary_assign(instr, $op) + } + )* + }; +} +impl<'ctx, 'engine> Executor<'ctx, 'engine> { + impl_fallible_binary_assign! { + (Instruction::I32DivSAssign, execute_i32_div_s_assign, UntypedValue::i32_div_s), + (Instruction::I32DivUAssign, execute_i32_div_u_assign, UntypedValue::i32_div_u), + (Instruction::I32RemSAssign, execute_i32_rem_s_assign, UntypedValue::i32_rem_s), + (Instruction::I32RemUAssign, execute_i32_rem_u_assign, UntypedValue::i32_rem_u), + + (Instruction::I64DivSAssign, execute_i64_div_s_assign, UntypedValue::i64_div_s), + (Instruction::I64DivUAssign, execute_i64_div_u_assign, UntypedValue::i64_div_u), + (Instruction::I64RemSAssign, execute_i64_rem_s_assign, UntypedValue::i64_rem_s), + (Instruction::I64RemUAssign, execute_i64_rem_u_assign, UntypedValue::i64_rem_u), + } +} + +macro_rules! impl_binary_assign_imm32 { + ( $( ($ty:ty, Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { + $( + #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")] + #[inline(always)] + pub fn $fn_name(&mut self, instr: BinAssignInstrImm32<$ty>) { + self.execute_binary_assign_imm32(instr, $op) + } + )* + }; +} +impl<'ctx, 'engine> Executor<'ctx, 'engine> { + impl_binary_assign_imm32! { + (i32, Instruction::I32EqAssignImm, execute_i32_eq_assign_imm, UntypedValue::i32_eq), + (i32, Instruction::I32NeAssignImm, execute_i32_ne_assign_imm, UntypedValue::i32_ne), + (i32, Instruction::I32LtSAssignImm, execute_i32_lt_s_assign_imm, UntypedValue::i32_lt_s), + (u32, Instruction::I32LtUAssignImm, execute_i32_lt_u_assign_imm, UntypedValue::i32_lt_u), + (i32, Instruction::I32LeSAssignImm, execute_i32_le_s_assign_imm, UntypedValue::i32_le_s), + (u32, Instruction::I32LeUAssignImm, execute_i32_le_u_assign_imm, UntypedValue::i32_le_u), + (i32, Instruction::I32GtSAssignImm, execute_i32_gt_s_assign_imm, UntypedValue::i32_gt_s), + (u32, Instruction::I32GtUAssignImm, execute_i32_gt_u_assign_imm, UntypedValue::i32_gt_u), + (i32, Instruction::I32GeSAssignImm, execute_i32_ge_s_assign_imm, UntypedValue::i32_ge_s), + (u32, Instruction::I32GeUAssignImm, execute_i32_ge_u_assign_imm, UntypedValue::i32_ge_u), + + (i64, Instruction::I64EqAssignImm32, execute_i64_eq_assign_imm32, UntypedValue::i64_eq), + (i64, Instruction::I64NeAssignImm32, execute_i64_ne_assign_imm32, UntypedValue::i64_ne), + (i64, Instruction::I64LtSAssignImm32, execute_i64_lt_s_assign_imm32, UntypedValue::i64_lt_s), + (u64, Instruction::I64LtUAssignImm32, execute_i64_lt_u_assign_imm32, UntypedValue::i64_lt_u), + (i64, Instruction::I64LeSAssignImm32, execute_i64_le_s_assign_imm32, UntypedValue::i64_le_s), + (u64, Instruction::I64LeUAssignImm32, execute_i64_le_u_assign_imm32, UntypedValue::i64_le_u), + (i64, Instruction::I64GtSAssignImm32, execute_i64_gt_s_assign_imm32, UntypedValue::i64_gt_s), + (u64, Instruction::I64GtUAssignImm32, execute_i64_gt_u_assign_imm32, UntypedValue::i64_gt_u), + (i64, Instruction::I64GeSAssignImm32, execute_i64_ge_s_assign_imm32, UntypedValue::i64_ge_s), + (u64, Instruction::I64GeUAssignImm32, execute_i64_ge_u_assign_imm32, UntypedValue::i64_ge_u), + + (f32, Instruction::F32EqAssignImm, execute_f32_eq_assign_imm, UntypedValue::f32_eq), + (f32, Instruction::F32NeAssignImm, execute_f32_ne_assign_imm, UntypedValue::f32_ne), + (f32, Instruction::F32LtAssignImm, execute_f32_lt_assign_imm, UntypedValue::f32_lt), + (f32, Instruction::F32LeAssignImm, execute_f32_le_assign_imm, UntypedValue::f32_le), + (f32, Instruction::F32GtAssignImm, execute_f32_gt_assign_imm, UntypedValue::f32_gt), + (f32, Instruction::F32GeAssignImm, execute_f32_ge_assign_imm, UntypedValue::f32_ge), + + (f64, Instruction::F64EqAssignImm32, execute_f64_eq_assign_imm32, UntypedValue::f64_eq), + (f64, Instruction::F64NeAssignImm32, execute_f64_ne_assign_imm32, UntypedValue::f64_ne), + (f64, Instruction::F64LtAssignImm32, execute_f64_lt_assign_imm32, UntypedValue::f64_lt), + (f64, Instruction::F64LeAssignImm32, execute_f64_le_assign_imm32, UntypedValue::f64_le), + (f64, Instruction::F64GtAssignImm32, execute_f64_gt_assign_imm32, UntypedValue::f64_gt), + (f64, Instruction::F64GeAssignImm32, execute_f64_ge_assign_imm32, UntypedValue::f64_ge), + + (i32, Instruction::I32AddAssignImm, execute_i32_add_assign_imm, UntypedValue::i32_add), + (i32, Instruction::I32SubAssignImm, execute_i32_sub_assign_imm, UntypedValue::i32_sub), + (i32, Instruction::I32MulAssignImm, execute_i32_mul_assign_imm, UntypedValue::i32_mul), + (i32, Instruction::I32AndAssignImm, execute_i32_and_assign_imm, UntypedValue::i32_and), + (i32, Instruction::I32OrAssignImm, execute_i32_or_assign_imm, UntypedValue::i32_or), + (i32, Instruction::I32XorAssignImm, execute_i32_xor_assign_imm, UntypedValue::i32_xor), + (i32, Instruction::I32ShlAssignImm, execute_i32_shl_assign_imm, UntypedValue::i32_shl), + (i32, Instruction::I32ShrSAssignImm, execute_i32_shr_s_assign_imm, UntypedValue::i32_shr_s), + (u32, Instruction::I32ShrUAssignImm, execute_i32_shr_u_assign_imm, UntypedValue::i32_shr_u), + (i32, Instruction::I32RotlAssignImm, execute_i32_rotl_assign_imm, UntypedValue::i32_rotl), + (i32, Instruction::I32RotrAssignImm, execute_i32_rotr_assign_imm, UntypedValue::i32_rotr), + + (i64, Instruction::I64AddAssignImm32, execute_i64_add_assign_imm32, UntypedValue::i64_add), + (i64, Instruction::I64SubAssignImm32, execute_i64_sub_assign_imm32, UntypedValue::i64_sub), + (i64, Instruction::I64MulAssignImm32, execute_i64_mul_assign_imm32, UntypedValue::i64_mul), + (i64, Instruction::I64AndAssignImm32, execute_i64_and_assign_imm32, UntypedValue::i64_and), + (i64, Instruction::I64OrAssignImm32, execute_i64_or_assign_imm32, UntypedValue::i64_or), + (i64, Instruction::I64XorAssignImm32, execute_i64_xor_assign_imm32, UntypedValue::i64_xor), + (i64, Instruction::I64ShlAssignImm32, execute_i64_shl_assign_imm32, UntypedValue::i64_shl), + (i64, Instruction::I64ShrSAssignImm32, execute_i64_shr_s_assign_imm32, UntypedValue::i64_shr_s), + (u64, Instruction::I64ShrUAssignImm32, execute_i64_shr_u_assign_imm32, UntypedValue::i64_shr_u), + (i64, Instruction::I64RotlAssignImm32, execute_i64_rotl_assign_imm32, UntypedValue::i64_rotl), + (i64, Instruction::I64RotrAssignImm32, execute_i64_rotr_assign_imm32, UntypedValue::i64_rotr), + + (f32, Instruction::F32AddAssignImm, execute_f32_add_assign_imm, UntypedValue::f32_add), + (f32, Instruction::F32SubAssignImm, execute_f32_sub_assign_imm, UntypedValue::f32_sub), + (f32, Instruction::F32MulAssignImm, execute_f32_mul_assign_imm, UntypedValue::f32_mul), + (f32, Instruction::F32DivAssignImm, execute_f32_div_assign_imm, UntypedValue::f32_div), + (f32, Instruction::F32MinAssignImm, execute_f32_min_assign_imm, UntypedValue::f32_min), + (f32, Instruction::F32MaxAssignImm, execute_f32_max_assign_imm, UntypedValue::f32_max), + (f32, Instruction::F32CopysignAssignImm, execute_f32_copysign_assign_imm, UntypedValue::f32_copysign), + + (f64, Instruction::F64AddAssignImm32, execute_f64_add_assign_imm32, UntypedValue::f64_add), + (f64, Instruction::F64SubAssignImm32, execute_f64_sub_assign_imm32, UntypedValue::f64_sub), + (f64, Instruction::F64MulAssignImm32, execute_f64_mul_assign_imm32, UntypedValue::f64_mul), + (f64, Instruction::F64DivAssignImm32, execute_f64_div_assign_imm32, UntypedValue::f64_div), + (f64, Instruction::F64MinAssignImm32, execute_f64_min_assign_imm32, UntypedValue::f64_min), + (f64, Instruction::F64MaxAssignImm32, execute_f64_max_assign_imm32, UntypedValue::f64_max), + (f64, Instruction::F64CopysignAssignImm32, execute_f64_copysign_assign_imm32, UntypedValue::f64_copysign), + } +} + +macro_rules! impl_fallible_binary_assign_imm32 { + ( $( ($ty:ty, Instruction::$var_name:ident, $fn_name:ident, $op:expr) ),* $(,)? ) => { + $( + #[doc = concat!("Executes an [`Instruction::", stringify!($var_name), "`].")] + #[inline(always)] + pub fn $fn_name(&mut self, instr: BinAssignInstrImm32<$ty>) -> Result<(), TrapCode> { + self.try_execute_binary_assign_imm32(instr, $op) + } + )* + }; +} +impl<'ctx, 'engine> Executor<'ctx, 'engine> { + impl_fallible_binary_assign_imm32! { + (i32, Instruction::I32DivSAssignImm, execute_i32_div_s_assign_imm, UntypedValue::i32_div_s), + (u32, Instruction::I32DivUAssignImm, execute_i32_div_u_assign_imm, UntypedValue::i32_div_u), + (i32, Instruction::I32RemSAssignImm, execute_i32_rem_s_assign_imm, UntypedValue::i32_rem_s), + (u32, Instruction::I32RemUAssignImm, execute_i32_rem_u_assign_imm, UntypedValue::i32_rem_u), + + (i64, Instruction::I64DivSAssignImm32, execute_i64_div_s_assign_imm32, UntypedValue::i64_div_s), + (u64, Instruction::I64DivUAssignImm32, execute_i64_div_u_assign_imm32, UntypedValue::i64_div_u), + (i64, Instruction::I64RemSAssignImm32, execute_i64_rem_s_assign_imm32, UntypedValue::i64_rem_s), + (u64, Instruction::I64RemUAssignImm32, execute_i64_rem_u_assign_imm32, UntypedValue::i64_rem_u), + } +} From 0d08c623162dd6492d43527eec9aee46f77e9396 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 11:03:21 +0100 Subject: [PATCH 06/17] add translation for new op-assign (non imm) instructions --- .../src/engine/regmach/translator/mod.rs | 28 +++++-- .../src/engine/regmach/translator/visit.rs | 76 +++++++++++++++++++ 2 files changed, 97 insertions(+), 7 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/translator/mod.rs b/crates/wasmi/src/engine/regmach/translator/mod.rs index e20c28580d..896a52d46f 100644 --- a/crates/wasmi/src/engine/regmach/translator/mod.rs +++ b/crates/wasmi/src/engine/regmach/translator/mod.rs @@ -691,8 +691,15 @@ impl<'parser> FuncTranslator<'parser> { lhs: Register, rhs: Register, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_assign_instr: fn(result: Register, rhs: Register) -> Instruction, ) -> Result<(), TranslationError> { let result = self.alloc.stack.push_dynamic()?; + if result == lhs { + self.alloc + .instr_encoder + .push_instr(make_assign_instr(result, rhs))?; + return Ok(()); + } self.alloc .instr_encoder .push_instr(make_instr(result, lhs, rhs))?; @@ -839,6 +846,7 @@ impl<'parser> FuncTranslator<'parser> { fn translate_binary( &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, make_instr_imm16: fn(result: Register, lhs: Register, rhs: Const16) -> Instruction, make_instr_imm16_rev: fn(result: Register, lhs: Const16, rhs: Register) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, @@ -868,7 +876,7 @@ impl<'parser> FuncTranslator<'parser> { // Case: the custom logic applied its optimization and we can return. return Ok(()); } - self.push_binary_instr(lhs, rhs, make_instr) + self.push_binary_instr(lhs, rhs, make_instr, make_instr_assign) } (TypedProvider::Register(lhs), TypedProvider::Const(rhs)) => { if make_instr_reg_imm_opt(self, lhs, T::from(rhs))? { @@ -923,6 +931,7 @@ impl<'parser> FuncTranslator<'parser> { fn translate_fbinary( &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, make_instr_opt: fn( &mut Self, @@ -950,7 +959,7 @@ impl<'parser> FuncTranslator<'parser> { // Case: the custom logic applied its optimization and we can return. return Ok(()); } - self.push_binary_instr(lhs, rhs, make_instr) + self.push_binary_instr(lhs, rhs, make_instr, make_instr_assign) } (TypedProvider::Register(lhs), TypedProvider::Const(rhs)) => { if make_instr_reg_imm_opt(self, lhs, T::from(rhs))? { @@ -991,6 +1000,7 @@ impl<'parser> FuncTranslator<'parser> { fn translate_fcopysign( &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, make_instr_imm: fn(result: Register, lhs: Register, rhs: Sign) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, ) -> Result<(), TranslationError> @@ -1005,7 +1015,7 @@ impl<'parser> FuncTranslator<'parser> { self.alloc.stack.push_register(lhs)?; return Ok(()); } - self.push_binary_instr(lhs, rhs, make_instr) + self.push_binary_instr(lhs, rhs, make_instr, make_instr_assign) } (TypedProvider::Register(lhs), TypedProvider::Const(rhs)) => { let sign = T::from(rhs).sign(); @@ -1047,6 +1057,7 @@ impl<'parser> FuncTranslator<'parser> { fn translate_binary_commutative( &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, make_instr_imm16: fn(result: Register, lhs: Register, rhs: Const16) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, make_instr_opt: fn( @@ -1066,7 +1077,7 @@ impl<'parser> FuncTranslator<'parser> { // Case: the custom logic applied its optimization and we can return. return Ok(()); } - self.push_binary_instr(lhs, rhs, make_instr) + self.push_binary_instr(lhs, rhs, make_instr, make_instr_assign) } (TypedProvider::Register(reg_in), TypedProvider::Const(imm_in)) | (TypedProvider::Const(imm_in), TypedProvider::Register(reg_in)) => { @@ -1109,6 +1120,7 @@ impl<'parser> FuncTranslator<'parser> { fn translate_fbinary_commutative( &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, make_instr_opt: fn( &mut Self, @@ -1127,7 +1139,7 @@ impl<'parser> FuncTranslator<'parser> { // Case: the custom logic applied its optimization and we can return. return Ok(()); } - self.push_binary_instr(lhs, rhs, make_instr) + self.push_binary_instr(lhs, rhs, make_instr, make_instr_assign) } (TypedProvider::Register(reg_in), TypedProvider::Const(imm_in)) | (TypedProvider::Const(imm_in), TypedProvider::Register(reg_in)) => { @@ -1168,6 +1180,7 @@ impl<'parser> FuncTranslator<'parser> { fn translate_shift( &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, make_instr_imm: fn(result: Register, lhs: Register, rhs: Const16) -> Instruction, make_instr_imm16_rev: fn(result: Register, lhs: Const16, rhs: Register) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, @@ -1184,7 +1197,7 @@ impl<'parser> FuncTranslator<'parser> { bail_unreachable!(self); match self.alloc.stack.pop2() { (TypedProvider::Register(lhs), TypedProvider::Register(rhs)) => { - self.push_binary_instr(lhs, rhs, make_instr) + self.push_binary_instr(lhs, rhs, make_instr, make_instr_assign) } (TypedProvider::Register(lhs), TypedProvider::Const(rhs)) => { let rhs = T::from(rhs).as_shift_amount(); @@ -1243,6 +1256,7 @@ impl<'parser> FuncTranslator<'parser> { pub fn translate_divrem( &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, make_instr_imm16: fn(result: Register, lhs: Register, rhs: Const16) -> Instruction, make_instr_imm16_rev: fn(result: Register, lhs: Const16, rhs: Register) -> Instruction, consteval: fn(TypedValue, TypedValue) -> Result, @@ -1267,7 +1281,7 @@ impl<'parser> FuncTranslator<'parser> { // Custom optimization was applied: return early return Ok(()); } - self.push_binary_instr(lhs, rhs, make_instr) + self.push_binary_instr(lhs, rhs, make_instr, make_instr_assign) } (TypedProvider::Register(lhs), TypedProvider::Const(rhs)) => { if T::from(rhs).eq_zero() { diff --git a/crates/wasmi/src/engine/regmach/translator/visit.rs b/crates/wasmi/src/engine/regmach/translator/visit.rs index 0162f73d87..889b8f7cfe 100644 --- a/crates/wasmi/src/engine/regmach/translator/visit.rs +++ b/crates/wasmi/src/engine/regmach/translator/visit.rs @@ -1208,6 +1208,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_eq(&mut self) -> Self::Output { self.translate_binary_commutative::( Instruction::i32_eq, + Instruction::i32_eq_assign, Instruction::i32_eq_imm16, TypedValue::i32_eq, |this, lhs: Register, rhs: Register| { @@ -1225,6 +1226,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_ne(&mut self) -> Self::Output { self.translate_binary_commutative::( Instruction::i32_ne, + Instruction::i32_ne_assign, Instruction::i32_ne_imm16, TypedValue::i32_ne, |this, lhs: Register, rhs: Register| { @@ -1242,6 +1244,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_lt_s(&mut self) -> Self::Output { self.translate_binary( Instruction::i32_lt_s, + Instruction::i32_lt_s_assign, Instruction::i32_lt_s_imm16, swap_ops!(Instruction::i32_gt_s_imm16), TypedValue::i32_lt_s, @@ -1275,6 +1278,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_lt_u(&mut self) -> Self::Output { self.translate_binary( Instruction::i32_lt_u, + Instruction::i32_lt_u_assign, Instruction::i32_lt_u_imm16, swap_ops!(Instruction::i32_gt_u_imm16), TypedValue::i32_lt_u, @@ -1308,6 +1312,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_gt_s(&mut self) -> Self::Output { self.translate_binary( Instruction::i32_gt_s, + Instruction::i32_gt_s_assign, Instruction::i32_gt_s_imm16, swap_ops!(Instruction::i32_lt_s_imm16), TypedValue::i32_gt_s, @@ -1341,6 +1346,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_gt_u(&mut self) -> Self::Output { self.translate_binary( Instruction::i32_gt_u, + Instruction::i32_gt_u_assign, Instruction::i32_gt_u_imm16, swap_ops!(Instruction::i32_lt_u_imm16), TypedValue::i32_gt_u, @@ -1374,6 +1380,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_le_s(&mut self) -> Self::Output { self.translate_binary( Instruction::i32_le_s, + Instruction::i32_le_s_assign, Instruction::i32_le_s_imm16, swap_ops!(Instruction::i32_ge_s_imm16), TypedValue::i32_le_s, @@ -1407,6 +1414,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_le_u(&mut self) -> Self::Output { self.translate_binary( Instruction::i32_le_u, + Instruction::i32_le_u_assign, Instruction::i32_le_u_imm16, swap_ops!(Instruction::i32_ge_u_imm16), TypedValue::i32_le_u, @@ -1440,6 +1448,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_ge_s(&mut self) -> Self::Output { self.translate_binary( Instruction::i32_ge_s, + Instruction::i32_ge_s_assign, Instruction::i32_ge_s_imm16, swap_ops!(Instruction::i32_le_s_imm16), TypedValue::i32_ge_s, @@ -1473,6 +1482,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_ge_u(&mut self) -> Self::Output { self.translate_binary( Instruction::i32_ge_u, + Instruction::i32_ge_u_assign, Instruction::i32_ge_u_imm16, swap_ops!(Instruction::i32_le_u_imm16), TypedValue::i32_ge_u, @@ -1513,6 +1523,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_eq(&mut self) -> Self::Output { self.translate_binary_commutative::( Instruction::i64_eq, + Instruction::i64_eq_assign, Instruction::i64_eq_imm16, TypedValue::i64_eq, |this, lhs: Register, rhs: Register| { @@ -1530,6 +1541,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_ne(&mut self) -> Self::Output { self.translate_binary_commutative::( Instruction::i64_ne, + Instruction::i64_ne_assign, Instruction::i64_ne_imm16, TypedValue::i64_ne, |this, lhs: Register, rhs: Register| { @@ -1547,6 +1559,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_lt_s(&mut self) -> Self::Output { self.translate_binary( Instruction::i64_lt_s, + Instruction::i64_lt_s_assign, Instruction::i64_lt_s_imm16, swap_ops!(Instruction::i64_gt_s_imm16), TypedValue::i64_lt_s, @@ -1580,6 +1593,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_lt_u(&mut self) -> Self::Output { self.translate_binary( Instruction::i64_lt_u, + Instruction::i64_lt_u_assign, Instruction::i64_lt_u_imm16, swap_ops!(Instruction::i64_gt_u_imm16), TypedValue::i64_lt_u, @@ -1613,6 +1627,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_gt_s(&mut self) -> Self::Output { self.translate_binary( Instruction::i64_gt_s, + Instruction::i64_gt_s_assign, Instruction::i64_gt_s_imm16, swap_ops!(Instruction::i64_lt_s_imm16), TypedValue::i64_gt_s, @@ -1646,6 +1661,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_gt_u(&mut self) -> Self::Output { self.translate_binary( Instruction::i64_gt_u, + Instruction::i64_gt_u_assign, Instruction::i64_gt_u_imm16, swap_ops!(Instruction::i64_lt_u_imm16), TypedValue::i64_gt_u, @@ -1679,6 +1695,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_le_s(&mut self) -> Self::Output { self.translate_binary( Instruction::i64_le_s, + Instruction::i64_le_s_assign, Instruction::i64_le_s_imm16, swap_ops!(Instruction::i64_ge_s_imm16), TypedValue::i64_le_s, @@ -1712,6 +1729,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_le_u(&mut self) -> Self::Output { self.translate_binary( Instruction::i64_le_u, + Instruction::i64_le_u_assign, Instruction::i64_le_u_imm16, swap_ops!(Instruction::i64_ge_u_imm16), TypedValue::i64_le_u, @@ -1745,6 +1763,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_ge_s(&mut self) -> Self::Output { self.translate_binary( Instruction::i64_ge_s, + Instruction::i64_ge_s_assign, Instruction::i64_ge_s_imm16, swap_ops!(Instruction::i64_le_s_imm16), TypedValue::i64_ge_s, @@ -1778,6 +1797,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_ge_u(&mut self) -> Self::Output { self.translate_binary( Instruction::i64_ge_u, + Instruction::i64_ge_u_assign, Instruction::i64_ge_u_imm16, swap_ops!(Instruction::i64_le_u_imm16), TypedValue::i64_ge_u, @@ -1811,6 +1831,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_eq(&mut self) -> Self::Output { self.translate_fbinary_commutative::( Instruction::f32_eq, + Instruction::f32_eq_assign, TypedValue::f32_eq, Self::no_custom_opt, |this, _reg_in: Register, imm_in: f32| { @@ -1827,6 +1848,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_ne(&mut self) -> Self::Output { self.translate_fbinary_commutative::( Instruction::f32_ne, + Instruction::f32_ne_assign, TypedValue::f32_ne, Self::no_custom_opt, |this, _reg_in: Register, imm_in: f32| { @@ -1843,6 +1865,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_lt(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f32_lt, + Instruction::f32_lt_assign, TypedValue::f32_lt, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -1884,6 +1907,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_gt(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f32_gt, + Instruction::f32_gt_assign, TypedValue::f32_gt, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -1925,6 +1949,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_le(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f32_le, + Instruction::f32_le_assign, TypedValue::f32_le, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -1956,6 +1981,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_ge(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f32_ge, + Instruction::f32_ge_assign, TypedValue::f32_ge, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -1987,6 +2013,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_eq(&mut self) -> Self::Output { self.translate_fbinary_commutative::( Instruction::f64_eq, + Instruction::f64_eq_assign, TypedValue::f64_eq, Self::no_custom_opt, |this, _reg_in: Register, imm_in: f64| { @@ -2003,6 +2030,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_ne(&mut self) -> Self::Output { self.translate_fbinary_commutative::( Instruction::f64_ne, + Instruction::f64_ne_assign, TypedValue::f64_ne, Self::no_custom_opt, |this, _reg_in: Register, imm_in: f64| { @@ -2019,6 +2047,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_lt(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f64_lt, + Instruction::f64_lt_assign, TypedValue::f64_lt, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -2060,6 +2089,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_gt(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f64_gt, + Instruction::f64_gt_assign, TypedValue::f64_gt, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -2101,6 +2131,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_le(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f64_le, + Instruction::f64_le_assign, TypedValue::f64_le, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -2132,6 +2163,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_ge(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f64_ge, + Instruction::f64_ge_assign, TypedValue::f64_ge, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -2175,6 +2207,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_add(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i32_add, + Instruction::i32_add_assign, Instruction::i32_add_imm16, TypedValue::i32_add, Self::no_custom_opt, @@ -2192,6 +2225,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_sub(&mut self) -> Self::Output { self.translate_binary( Instruction::i32_sub, + Instruction::i32_sub_assign, Instruction::i32_sub_imm16, Instruction::i32_sub_imm16_rev, TypedValue::i32_sub, @@ -2218,6 +2252,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_mul(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i32_mul, + Instruction::i32_mul_assign, Instruction::i32_mul_imm16, TypedValue::i32_mul, Self::no_custom_opt, @@ -2240,6 +2275,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_div_s(&mut self) -> Self::Output { self.translate_divrem( Instruction::i32_div_s, + Instruction::i32_div_s_assign, Instruction::i32_div_s_imm16, Instruction::i32_div_s_imm16_rev, TypedValue::i32_div_s, @@ -2258,6 +2294,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_div_u(&mut self) -> Self::Output { self.translate_divrem::( Instruction::i32_div_u, + Instruction::i32_div_u_assign, Instruction::i32_div_u_imm16, Instruction::i32_div_u_imm16_rev, TypedValue::i32_div_u, @@ -2276,6 +2313,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_rem_s(&mut self) -> Self::Output { self.translate_divrem( Instruction::i32_rem_s, + Instruction::i32_rem_s_assign, Instruction::i32_rem_s_imm16, Instruction::i32_rem_s_imm16_rev, TypedValue::i32_rem_s, @@ -2294,6 +2332,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_rem_u(&mut self) -> Self::Output { self.translate_divrem::( Instruction::i32_rem_u, + Instruction::i32_rem_u_assign, Instruction::i32_rem_u_imm16, Instruction::i32_rem_u_imm16_rev, TypedValue::i32_rem_u, @@ -2312,6 +2351,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_and(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i32_and, + Instruction::i32_and_assign, Instruction::i32_and_imm16, TypedValue::i32_and, |this, lhs, rhs| { @@ -2344,6 +2384,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_or(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i32_or, + Instruction::i32_or_assign, Instruction::i32_or_imm16, TypedValue::i32_or, |this, lhs, rhs| { @@ -2376,6 +2417,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_xor(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i32_xor, + Instruction::i32_xor_assign, Instruction::i32_xor_imm16, TypedValue::i32_xor, |this, lhs, rhs| { @@ -2400,6 +2442,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_shl(&mut self) -> Self::Output { self.translate_shift::( Instruction::i32_shl, + Instruction::i32_shl_assign, Instruction::i32_shl_imm, Instruction::i32_shl_imm16_rev, TypedValue::i32_shl, @@ -2410,6 +2453,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_shr_s(&mut self) -> Self::Output { self.translate_shift( Instruction::i32_shr_s, + Instruction::i32_shr_s_assign, Instruction::i32_shr_s_imm, Instruction::i32_shr_s_imm16_rev, TypedValue::i32_shr_s, @@ -2427,6 +2471,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_shr_u(&mut self) -> Self::Output { self.translate_shift::( Instruction::i32_shr_u, + Instruction::i32_shr_u_assign, Instruction::i32_shr_u_imm, Instruction::i32_shr_u_imm16_rev, TypedValue::i32_shr_u, @@ -2437,6 +2482,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_rotl(&mut self) -> Self::Output { self.translate_shift( Instruction::i32_rotl, + Instruction::i32_rotl_assign, Instruction::i32_rotl_imm, Instruction::i32_rotl_imm16_rev, TypedValue::i32_rotl, @@ -2454,6 +2500,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i32_rotr(&mut self) -> Self::Output { self.translate_shift( Instruction::i32_rotr, + Instruction::i32_rotr_assign, Instruction::i32_rotr_imm, Instruction::i32_rotr_imm16_rev, TypedValue::i32_rotr, @@ -2483,6 +2530,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_add(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i64_add, + Instruction::i64_add_assign, Instruction::i64_add_imm16, TypedValue::i64_add, Self::no_custom_opt, @@ -2500,6 +2548,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_sub(&mut self) -> Self::Output { self.translate_binary( Instruction::i64_sub, + Instruction::i64_sub_assign, Instruction::i64_sub_imm16, Instruction::i64_sub_imm16_rev, TypedValue::i64_sub, @@ -2526,6 +2575,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_mul(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i64_mul, + Instruction::i64_mul_assign, Instruction::i64_mul_imm16, TypedValue::i64_mul, Self::no_custom_opt, @@ -2548,6 +2598,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_div_s(&mut self) -> Self::Output { self.translate_divrem( Instruction::i64_div_s, + Instruction::i64_div_s_assign, Instruction::i64_div_s_imm16, Instruction::i64_div_s_imm16_rev, TypedValue::i64_div_s, @@ -2566,6 +2617,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_div_u(&mut self) -> Self::Output { self.translate_divrem::( Instruction::i64_div_u, + Instruction::i64_div_u_assign, Instruction::i64_div_u_imm16, Instruction::i64_div_u_imm16_rev, TypedValue::i64_div_u, @@ -2584,6 +2636,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_rem_s(&mut self) -> Self::Output { self.translate_divrem( Instruction::i64_rem_s, + Instruction::i64_rem_s_assign, Instruction::i64_rem_s_imm16, Instruction::i64_rem_s_imm16_rev, TypedValue::i64_rem_s, @@ -2602,6 +2655,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_rem_u(&mut self) -> Self::Output { self.translate_divrem::( Instruction::i64_rem_u, + Instruction::i64_rem_u_assign, Instruction::i64_rem_u_imm16, Instruction::i64_rem_u_imm16_rev, TypedValue::i64_rem_u, @@ -2620,6 +2674,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_and(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i64_and, + Instruction::i64_and_assign, Instruction::i64_and_imm16, TypedValue::i64_and, |this, lhs, rhs| { @@ -2652,6 +2707,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_or(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i64_or, + Instruction::i64_or_assign, Instruction::i64_or_imm16, TypedValue::i64_or, |this, lhs, rhs| { @@ -2684,6 +2740,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_xor(&mut self) -> Self::Output { self.translate_binary_commutative( Instruction::i64_xor, + Instruction::i64_xor_assign, Instruction::i64_xor_imm16, TypedValue::i64_xor, |this, lhs, rhs| { @@ -2708,6 +2765,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_shl(&mut self) -> Self::Output { self.translate_shift::( Instruction::i64_shl, + Instruction::i64_shl_assign, Instruction::i64_shl_imm, Instruction::i64_shl_imm16_rev, TypedValue::i64_shl, @@ -2718,6 +2776,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_shr_s(&mut self) -> Self::Output { self.translate_shift( Instruction::i64_shr_s, + Instruction::i64_shr_s_assign, Instruction::i64_shr_s_imm, Instruction::i64_shr_s_imm16_rev, TypedValue::i64_shr_s, @@ -2735,6 +2794,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_shr_u(&mut self) -> Self::Output { self.translate_shift::( Instruction::i64_shr_u, + Instruction::i64_shr_u_assign, Instruction::i64_shr_u_imm, Instruction::i64_shr_u_imm16_rev, TypedValue::i64_shr_u, @@ -2745,6 +2805,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_rotl(&mut self) -> Self::Output { self.translate_shift( Instruction::i64_rotl, + Instruction::i64_rotl_assign, Instruction::i64_rotl_imm, Instruction::i64_rotl_imm16_rev, TypedValue::i64_rotl, @@ -2762,6 +2823,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_i64_rotr(&mut self) -> Self::Output { self.translate_shift( Instruction::i64_rotr, + Instruction::i64_rotr_assign, Instruction::i64_rotr_imm, Instruction::i64_rotr_imm16_rev, TypedValue::i64_rotr, @@ -2807,6 +2869,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_add(&mut self) -> Self::Output { self.translate_fbinary_commutative( Instruction::f32_add, + Instruction::f32_add_assign, TypedValue::f32_add, Self::no_custom_opt, Self::no_custom_opt::, @@ -2816,6 +2879,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_sub(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f32_sub, + Instruction::f32_sub_assign, TypedValue::f32_sub, Self::no_custom_opt, Self::no_custom_opt::, @@ -2829,6 +2893,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_mul(&mut self) -> Self::Output { self.translate_fbinary_commutative::( Instruction::f32_mul, + Instruction::f32_mul_assign, TypedValue::f32_mul, Self::no_custom_opt, // Unfortunately we cannot apply `x * 0` or `0 * x` optimizations @@ -2841,6 +2906,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_div(&mut self) -> Self::Output { self.translate_fbinary::( Instruction::f32_div, + Instruction::f32_div_assign, TypedValue::f32_div, Self::no_custom_opt, Self::no_custom_opt, @@ -2851,6 +2917,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_min(&mut self) -> Self::Output { self.translate_fbinary_commutative( Instruction::f32_min, + Instruction::f32_min_assign, TypedValue::f32_min, Self::no_custom_opt, |this, reg: Register, value: f32| { @@ -2867,6 +2934,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_max(&mut self) -> Self::Output { self.translate_fbinary_commutative( Instruction::f32_max, + Instruction::f32_max_assign, TypedValue::f32_max, Self::no_custom_opt, |this, reg: Register, value: f32| { @@ -2883,6 +2951,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f32_copysign(&mut self) -> Self::Output { self.translate_fcopysign::( Instruction::f32_copysign, + Instruction::f32_copysign_assign, Instruction::f32_copysign_imm, TypedValue::f32_copysign, ) @@ -2919,6 +2988,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_add(&mut self) -> Self::Output { self.translate_fbinary_commutative( Instruction::f64_add, + Instruction::f64_add_assign, TypedValue::f64_add, Self::no_custom_opt, Self::no_custom_opt::, @@ -2928,6 +2998,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_sub(&mut self) -> Self::Output { self.translate_fbinary( Instruction::f64_sub, + Instruction::f64_sub_assign, TypedValue::f64_sub, Self::no_custom_opt, Self::no_custom_opt::, @@ -2941,6 +3012,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_mul(&mut self) -> Self::Output { self.translate_fbinary_commutative::( Instruction::f64_mul, + Instruction::f64_mul_assign, TypedValue::f64_mul, Self::no_custom_opt, // Unfortunately we cannot apply `x * 0` or `0 * x` optimizations @@ -2953,6 +3025,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_div(&mut self) -> Self::Output { self.translate_fbinary::( Instruction::f64_div, + Instruction::f64_div_assign, TypedValue::f64_div, Self::no_custom_opt, Self::no_custom_opt, @@ -2963,6 +3036,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_min(&mut self) -> Self::Output { self.translate_fbinary_commutative( Instruction::f64_min, + Instruction::f64_min_assign, TypedValue::f64_min, Self::no_custom_opt, |this, reg: Register, value: f64| { @@ -2979,6 +3053,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_max(&mut self) -> Self::Output { self.translate_fbinary_commutative( Instruction::f64_max, + Instruction::f64_max_assign, TypedValue::f64_max, Self::no_custom_opt, |this, reg: Register, value: f64| { @@ -2995,6 +3070,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { fn visit_f64_copysign(&mut self) -> Self::Output { self.translate_fcopysign::( Instruction::f64_copysign, + Instruction::f64_copysign_assign, Instruction::f64_copysign_imm, TypedValue::f64_copysign, ) From 52fe4d72fcfec2d14dc29dfca9ef08023c98a494 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 13:59:55 +0100 Subject: [PATCH 07/17] add basic translation op-assign with immediates --- .../src/engine/regmach/bytecode/construct.rs | 17 ++++- .../src/engine/regmach/bytecode/immediate.rs | 6 ++ .../wasmi/src/engine/regmach/bytecode/mod.rs | 9 ++- .../src/engine/regmach/bytecode/utils.rs | 53 ++++++------- .../src/engine/regmach/executor/instrs.rs | 4 +- .../engine/regmach/executor/instrs/binary.rs | 35 ++++++--- .../src/engine/regmach/translator/mod.rs | 62 ++++++++++++--- .../engine/regmach/translator/result_mut.rs | 18 ++--- .../src/engine/regmach/translator/visit.rs | 76 +++++++++++++++++++ .../regmach/translator/visit_register.rs | 8 +- 10 files changed, 214 insertions(+), 74 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/bytecode/construct.rs b/crates/wasmi/src/engine/regmach/bytecode/construct.rs index f02afe8ad3..ba1c543ca6 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/construct.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/construct.rs @@ -2,6 +2,7 @@ use super::{ utils::{BranchOffset16, CopysignImmInstr, Sign}, AnyConst32, BinAssignInstr, + BinAssignInstrImm, BinAssignInstrImm32, BinInstr, BinInstrImm16, @@ -409,7 +410,7 @@ constructor_for_op_assign_imm32! { fn i32_xor_assign_imm(i32) -> Self::I32XorAssignImm; fn i32_shl_assign_imm(i32) -> Self::I32ShlAssignImm; fn i32_shr_s_assign_imm(i32) -> Self::I32ShrSAssignImm; - fn i32_shr_u_assign_imm(u32) -> Self::I32ShrUAssignImm; + fn i32_shr_u_assign_imm(i32) -> Self::I32ShrUAssignImm; fn i32_rotl_assign_imm(i32) -> Self::I32RotlAssignImm; fn i32_rotr_assign_imm(i32) -> Self::I32RotrAssignImm; @@ -425,7 +426,7 @@ constructor_for_op_assign_imm32! { fn i64_xor_assign_imm32(i64) -> Self::I64XorAssignImm32; fn i64_shl_assign_imm32(i64) -> Self::I64ShlAssignImm32; fn i64_shr_s_assign_imm32(i64) -> Self::I64ShrSAssignImm32; - fn i64_shr_u_assign_imm32(u64) -> Self::I64ShrUAssignImm32; + fn i64_shr_u_assign_imm32(i64) -> Self::I64ShrUAssignImm32; fn i64_rotl_assign_imm32(i64) -> Self::I64RotlAssignImm32; fn i64_rotr_assign_imm32(i64) -> Self::I64RotrAssignImm32; @@ -435,7 +436,6 @@ constructor_for_op_assign_imm32! { fn f32_div_assign_imm(f32) -> Self::F32DivAssignImm; fn f32_min_assign_imm(f32) -> Self::F32MinAssignImm; fn f32_max_assign_imm(f32) -> Self::F32MaxAssignImm; - fn f32_copysign_assign_imm(f32) -> Self::F32CopysignAssignImm; fn f64_add_assign_imm32(f64) -> Self::F64AddAssignImm32; fn f64_sub_assign_imm32(f64) -> Self::F64SubAssignImm32; @@ -443,7 +443,6 @@ constructor_for_op_assign_imm32! { fn f64_div_assign_imm32(f64) -> Self::F64DivAssignImm32; fn f64_min_assign_imm32(f64) -> Self::F64MinAssignImm32; fn f64_max_assign_imm32(f64) -> Self::F64MaxAssignImm32; - fn f64_copysign_assign_imm32(f64) -> Self::F64CopysignAssignImm32; } impl Instruction { @@ -744,11 +743,21 @@ impl Instruction { Self::F32CopysignImm(CopysignImmInstr { result, lhs, rhs }) } + /// Creates a new [`Instruction::F32CopysignAssignImm`] instruction. + pub fn f32_copysign_assign_imm(result: Register, rhs: Sign) -> Self { + Self::F32CopysignAssignImm(BinAssignInstrImm::new(result, rhs)) + } + /// Creates a new [`Instruction::F64CopysignImm`] instruction. pub fn f64_copysign_imm(result: Register, lhs: Register, rhs: Sign) -> Self { Self::F64CopysignImm(CopysignImmInstr { result, lhs, rhs }) } + /// Creates a new [`Instruction::F64CopysignAssignImm`] instruction. + pub fn f64_copysign_assign_imm(result: Register, rhs: Sign) -> Self { + Self::F64CopysignAssignImm(BinAssignInstrImm::new(result, rhs)) + } + /// Creates a new [`Instruction::Select`]. pub fn select(result: Register, condition: Register, lhs: Register) -> Self { Self::Select { diff --git a/crates/wasmi/src/engine/regmach/bytecode/immediate.rs b/crates/wasmi/src/engine/regmach/bytecode/immediate.rs index dfaf338673..2d7cf9a185 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/immediate.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/immediate.rs @@ -237,6 +237,12 @@ impl From for Const32 { } } +impl From for Const32 { + fn from(value: f32) -> Self { + Self::new(AnyConst32::from_f32(F32::from(value))) + } +} + impl From for Const32 { fn from(value: f32) -> Self { Self::new(AnyConst32::from_f32(F32::from(value))) diff --git a/crates/wasmi/src/engine/regmach/bytecode/mod.rs b/crates/wasmi/src/engine/regmach/bytecode/mod.rs index f644026b32..edc4906cbb 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/mod.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/mod.rs @@ -11,6 +11,7 @@ pub(crate) use self::{ provider::{Provider, ProviderSliceStack, UntypedProvider}, utils::{ BinAssignInstr, + BinAssignInstrImm, BinAssignInstrImm32, BinInstr, BinInstrImm16, @@ -3178,7 +3179,7 @@ pub enum Instruction { I32ShlAssign(BinAssignInstr), I32ShlAssignImm(BinAssignInstrImm32), I32ShrUAssign(BinAssignInstr), - I32ShrUAssignImm(BinAssignInstrImm32), + I32ShrUAssignImm(BinAssignInstrImm32), I32ShrSAssign(BinAssignInstr), I32ShrSAssignImm(BinAssignInstrImm32), I32RotlAssign(BinAssignInstr), @@ -3209,7 +3210,7 @@ pub enum Instruction { I64ShlAssign(BinAssignInstr), I64ShlAssignImm32(BinAssignInstrImm32), I64ShrUAssign(BinAssignInstr), - I64ShrUAssignImm32(BinAssignInstrImm32), + I64ShrUAssignImm32(BinAssignInstrImm32), I64ShrSAssign(BinAssignInstr), I64ShrSAssignImm32(BinAssignInstrImm32), I64RotlAssign(BinAssignInstr), @@ -3230,7 +3231,7 @@ pub enum Instruction { F32MaxAssign(BinAssignInstr), F32MaxAssignImm(BinAssignInstrImm32), F32CopysignAssign(BinAssignInstr), - F32CopysignAssignImm(BinAssignInstrImm32), + F32CopysignAssignImm(BinAssignInstrImm), F64AddAssign(BinAssignInstr), F64AddAssignImm32(BinAssignInstrImm32), @@ -3245,7 +3246,7 @@ pub enum Instruction { F64MaxAssign(BinAssignInstr), F64MaxAssignImm32(BinAssignInstrImm32), F64CopysignAssign(BinAssignInstr), - F64CopysignAssignImm32(BinAssignInstrImm32), + F64CopysignAssignImm(BinAssignInstrImm), } impl Instruction { diff --git a/crates/wasmi/src/engine/regmach/bytecode/utils.rs b/crates/wasmi/src/engine/regmach/bytecode/utils.rs index 02b6e92a34..9655821603 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/utils.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/utils.rs @@ -314,38 +314,21 @@ impl BinAssignInstr { } } -/// A binary-assign [`Register`] based instruction. -#[derive(Copy, Clone)] -pub struct BinAssignInstrImm32 { +/// A binary-assign instruction with 32-bit encoded immediate value. +pub type BinAssignInstrImm32 = BinAssignInstrImm>; + +/// A binary-assign instruction with generic immediate value. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct BinAssignInstrImm { /// The register storing both the result and left-hand side value. pub inout: Register, /// The constant right-hand side value. - pub rhs: Const32, -} - -impl PartialEq for BinAssignInstrImm32 { - fn eq(&self, other: &Self) -> bool { - self.inout == other.inout && self.rhs == other.rhs - } -} - -impl Eq for BinAssignInstrImm32 {} - -impl fmt::Debug for BinAssignInstrImm32 -where - Const32: fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BinAssignInstrImm32") - .field("inout", &self.inout) - .field("rhs", &self.rhs) - .finish() - } + pub rhs: T, } -impl BinAssignInstrImm32 { +impl BinAssignInstrImm { /// Creates a new [`BinAssignInstr`]. - pub fn new(inout: Register, rhs: Const32) -> Self { + pub fn new(inout: Register, rhs: T) -> Self { Self { inout, rhs } } } @@ -513,6 +496,24 @@ pub enum Sign { Neg, } +impl Sign { + /// Converts the [`Sign`] into an `f32` value. + pub fn to_f32(self) -> f32 { + match self { + Self::Pos => 1.0_f32, + Self::Neg => -1.0_f32, + } + } + + /// Converts the [`Sign`] into an `f64` value. + pub fn to_f64(self) -> f64 { + match self { + Self::Pos => 1.0_f64, + Self::Neg => -1.0_f64, + } + } +} + /// The `f32.copysign` or `f64.copysign` instruction with an immediate value. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct CopysignImmInstr { diff --git a/crates/wasmi/src/engine/regmach/executor/instrs.rs b/crates/wasmi/src/engine/regmach/executor/instrs.rs index 9639c841d8..1ad3c9f7c3 100644 --- a/crates/wasmi/src/engine/regmach/executor/instrs.rs +++ b/crates/wasmi/src/engine/regmach/executor/instrs.rs @@ -989,9 +989,7 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { Instr::F64DivAssignImm32(instr) => self.execute_f64_div_assign_imm32(instr), Instr::F64MinAssignImm32(instr) => self.execute_f64_min_assign_imm32(instr), Instr::F64MaxAssignImm32(instr) => self.execute_f64_max_assign_imm32(instr), - Instr::F64CopysignAssignImm32(instr) => { - self.execute_f64_copysign_assign_imm32(instr) - } + Instr::F64CopysignAssignImm(instr) => self.execute_f64_copysign_assign_imm(instr), } } } diff --git a/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs b/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs index eba060e2d0..57198ec2f7 100644 --- a/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs +++ b/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs @@ -3,6 +3,7 @@ use crate::{ core::{TrapCode, UntypedValue}, engine::regmach::bytecode::{ BinAssignInstr, + BinAssignInstrImm, BinAssignInstrImm32, BinInstr, BinInstrImm16, @@ -223,25 +224,37 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { #[inline(always)] pub fn execute_f32_copysign_imm(&mut self, instr: CopysignImmInstr) { let lhs = self.get_register(instr.lhs); - let rhs = match instr.rhs { - Sign::Pos => 1.0_f32, - Sign::Neg => -1.0_f32, - }; + let rhs = instr.rhs.to_f32(); self.set_register(instr.result, UntypedValue::f32_copysign(lhs, rhs.into())); self.next_instr() } + /// Executes an [`Instruction::F32CopysignAssignImm`]. + #[inline(always)] + pub fn execute_f32_copysign_assign_imm(&mut self, instr: BinAssignInstrImm) { + let lhs = self.get_register(instr.inout); + let rhs = instr.rhs.to_f32(); + self.set_register(instr.inout, UntypedValue::f32_copysign(lhs, rhs.into())); + self.next_instr() + } + /// Executes an [`Instruction::F64CopysignImm`]. #[inline(always)] pub fn execute_f64_copysign_imm(&mut self, instr: CopysignImmInstr) { let lhs = self.get_register(instr.lhs); - let rhs = match instr.rhs { - Sign::Pos => 1.0_f64, - Sign::Neg => -1.0_f64, - }; + let rhs = instr.rhs.to_f64(); self.set_register(instr.result, UntypedValue::f64_copysign(lhs, rhs.into())); self.next_instr() } + + /// Executes an [`Instruction::F64CopysignAssignImm`]. + #[inline(always)] + pub fn execute_f64_copysign_assign_imm(&mut self, instr: BinAssignInstrImm) { + let lhs = self.get_register(instr.inout); + let rhs = instr.rhs.to_f64(); + self.set_register(instr.inout, UntypedValue::f64_copysign(lhs, rhs.into())); + self.next_instr() + } } macro_rules! impl_binary_assign { @@ -417,7 +430,7 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { (i32, Instruction::I32XorAssignImm, execute_i32_xor_assign_imm, UntypedValue::i32_xor), (i32, Instruction::I32ShlAssignImm, execute_i32_shl_assign_imm, UntypedValue::i32_shl), (i32, Instruction::I32ShrSAssignImm, execute_i32_shr_s_assign_imm, UntypedValue::i32_shr_s), - (u32, Instruction::I32ShrUAssignImm, execute_i32_shr_u_assign_imm, UntypedValue::i32_shr_u), + (i32, Instruction::I32ShrUAssignImm, execute_i32_shr_u_assign_imm, UntypedValue::i32_shr_u), (i32, Instruction::I32RotlAssignImm, execute_i32_rotl_assign_imm, UntypedValue::i32_rotl), (i32, Instruction::I32RotrAssignImm, execute_i32_rotr_assign_imm, UntypedValue::i32_rotr), @@ -429,7 +442,7 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { (i64, Instruction::I64XorAssignImm32, execute_i64_xor_assign_imm32, UntypedValue::i64_xor), (i64, Instruction::I64ShlAssignImm32, execute_i64_shl_assign_imm32, UntypedValue::i64_shl), (i64, Instruction::I64ShrSAssignImm32, execute_i64_shr_s_assign_imm32, UntypedValue::i64_shr_s), - (u64, Instruction::I64ShrUAssignImm32, execute_i64_shr_u_assign_imm32, UntypedValue::i64_shr_u), + (i64, Instruction::I64ShrUAssignImm32, execute_i64_shr_u_assign_imm32, UntypedValue::i64_shr_u), (i64, Instruction::I64RotlAssignImm32, execute_i64_rotl_assign_imm32, UntypedValue::i64_rotl), (i64, Instruction::I64RotrAssignImm32, execute_i64_rotr_assign_imm32, UntypedValue::i64_rotr), @@ -439,7 +452,6 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { (f32, Instruction::F32DivAssignImm, execute_f32_div_assign_imm, UntypedValue::f32_div), (f32, Instruction::F32MinAssignImm, execute_f32_min_assign_imm, UntypedValue::f32_min), (f32, Instruction::F32MaxAssignImm, execute_f32_max_assign_imm, UntypedValue::f32_max), - (f32, Instruction::F32CopysignAssignImm, execute_f32_copysign_assign_imm, UntypedValue::f32_copysign), (f64, Instruction::F64AddAssignImm32, execute_f64_add_assign_imm32, UntypedValue::f64_add), (f64, Instruction::F64SubAssignImm32, execute_f64_sub_assign_imm32, UntypedValue::f64_sub), @@ -447,7 +459,6 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { (f64, Instruction::F64DivAssignImm32, execute_f64_div_assign_imm32, UntypedValue::f64_div), (f64, Instruction::F64MinAssignImm32, execute_f64_min_assign_imm32, UntypedValue::f64_min), (f64, Instruction::F64MaxAssignImm32, execute_f64_max_assign_imm32, UntypedValue::f64_max), - (f64, Instruction::F64CopysignAssignImm32, execute_f64_copysign_assign_imm32, UntypedValue::f64_copysign), } } diff --git a/crates/wasmi/src/engine/regmach/translator/mod.rs b/crates/wasmi/src/engine/regmach/translator/mod.rs index 896a52d46f..b8335550b1 100644 --- a/crates/wasmi/src/engine/regmach/translator/mod.rs +++ b/crates/wasmi/src/engine/regmach/translator/mod.rs @@ -776,11 +776,20 @@ impl<'parser> FuncTranslator<'parser> { lhs: Register, rhs: T, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_instr_assign_imm32: fn(result: Register, rhs: Const32) -> Instruction, ) -> Result<(), TranslationError> where - T: Into, + T: Copy + Into + TryInto>, { let result = self.alloc.stack.push_dynamic()?; + if result == lhs { + if let Ok(rhs) = rhs.try_into() { + self.alloc + .instr_encoder + .push_instr(make_instr_assign_imm32(result, rhs))?; + return Ok(()); + } + } let rhs = self.alloc.stack.alloc_const(rhs)?; self.alloc .instr_encoder @@ -847,6 +856,7 @@ impl<'parser> FuncTranslator<'parser> { &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, + make_instr_assign_imm32: fn(result: Register, rhs: Const32) -> Instruction, make_instr_imm16: fn(result: Register, lhs: Register, rhs: Const16) -> Instruction, make_instr_imm16_rev: fn(result: Register, lhs: Const16, rhs: Register) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, @@ -867,7 +877,12 @@ impl<'parser> FuncTranslator<'parser> { ) -> Result, ) -> Result<(), TranslationError> where - T: Copy + From + Into + TryInto>, + T: Copy + + From + + Into + + Into + + TryInto> + + TryInto>, { bail_unreachable!(self); match self.alloc.stack.pop2() { @@ -887,7 +902,7 @@ impl<'parser> FuncTranslator<'parser> { // Optimization was applied: return early. return Ok(()); } - self.push_binary_instr_imm(lhs, rhs, make_instr) + self.push_binary_instr_imm(lhs, T::from(rhs), make_instr, make_instr_assign_imm32) } (TypedProvider::Const(lhs), TypedProvider::Register(rhs)) => { if make_instr_imm_reg_opt(self, T::from(lhs), rhs)? { @@ -932,6 +947,7 @@ impl<'parser> FuncTranslator<'parser> { &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, + make_instr_assign_imm32: fn(result: Register, rhs: Const32) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, make_instr_opt: fn( &mut Self, @@ -950,7 +966,7 @@ impl<'parser> FuncTranslator<'parser> { ) -> Result, ) -> Result<(), TranslationError> where - T: WasmFloat, + T: WasmFloat + Into + TryInto>, { bail_unreachable!(self); match self.alloc.stack.pop2() { @@ -971,7 +987,7 @@ impl<'parser> FuncTranslator<'parser> { self.alloc.stack.push_const(rhs); return Ok(()); } - self.push_binary_instr_imm(lhs, rhs, make_instr) + self.push_binary_instr_imm(lhs, T::from(rhs), make_instr, make_instr_assign_imm32) } (TypedProvider::Const(lhs), TypedProvider::Register(rhs)) => { if make_instr_imm_reg_opt(self, T::from(lhs), rhs)? { @@ -1058,6 +1074,7 @@ impl<'parser> FuncTranslator<'parser> { &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, + make_instr_assign_imm32: fn(result: Register, rhs: Const32) -> Instruction, make_instr_imm16: fn(result: Register, lhs: Register, rhs: Const16) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, make_instr_opt: fn( @@ -1068,7 +1085,7 @@ impl<'parser> FuncTranslator<'parser> { make_instr_imm_opt: fn(&mut Self, lhs: Register, rhs: T) -> Result, ) -> Result<(), TranslationError> where - T: Copy + From + TryInto>, + T: Copy + From + TryInto> + TryInto> + Into, { bail_unreachable!(self); match self.alloc.stack.pop2() { @@ -1089,7 +1106,12 @@ impl<'parser> FuncTranslator<'parser> { // Optimization was applied: return early. return Ok(()); } - self.push_binary_instr_imm(reg_in, imm_in, make_instr) + self.push_binary_instr_imm( + reg_in, + T::from(imm_in), + make_instr, + make_instr_assign_imm32, + ) } (TypedProvider::Const(lhs), TypedProvider::Const(rhs)) => { self.push_binary_consteval(lhs, rhs, consteval) @@ -1121,6 +1143,7 @@ impl<'parser> FuncTranslator<'parser> { &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, + make_instr_assign_imm32: fn(result: Register, rhs: Const32) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, make_instr_opt: fn( &mut Self, @@ -1130,7 +1153,7 @@ impl<'parser> FuncTranslator<'parser> { make_instr_imm_opt: fn(&mut Self, lhs: Register, rhs: T) -> Result, ) -> Result<(), TranslationError> where - T: WasmFloat, + T: WasmFloat + Into + TryInto>, { bail_unreachable!(self); match self.alloc.stack.pop2() { @@ -1152,7 +1175,12 @@ impl<'parser> FuncTranslator<'parser> { self.alloc.stack.push_const(T::from(imm_in)); return Ok(()); } - self.push_binary_instr_imm(reg_in, imm_in, make_instr) + self.push_binary_instr_imm( + reg_in, + T::from(imm_in), + make_instr, + make_instr_assign_imm32, + ) } (TypedProvider::Const(lhs), TypedProvider::Const(rhs)) => { self.push_binary_consteval(lhs, rhs, consteval) @@ -1181,6 +1209,7 @@ impl<'parser> FuncTranslator<'parser> { &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, + make_instr_assign_imm32: fn(result: Register, rhs: Const32) -> Instruction, make_instr_imm: fn(result: Register, lhs: Register, rhs: Const16) -> Instruction, make_instr_imm16_rev: fn(result: Register, lhs: Const16, rhs: Register) -> Instruction, consteval: fn(TypedValue, TypedValue) -> TypedValue, @@ -1193,6 +1222,7 @@ impl<'parser> FuncTranslator<'parser> { where T: WasmInteger, Const16: From, + Const32: From, { bail_unreachable!(self); match self.alloc.stack.pop2() { @@ -1207,6 +1237,15 @@ impl<'parser> FuncTranslator<'parser> { return Ok(()); } let result = self.alloc.stack.push_dynamic()?; + if result == lhs { + self.alloc + .instr_encoder + .push_instr(make_instr_assign_imm32( + result, + >::from(i32::from(rhs)), + ))?; + return Ok(()); + } self.alloc.instr_encoder.push_instr(make_instr_imm( result, lhs, @@ -1257,6 +1296,7 @@ impl<'parser> FuncTranslator<'parser> { &mut self, make_instr: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, make_instr_assign: fn(result: Register, rhs: Register) -> Instruction, + make_instr_assign_imm32: fn(result: Register, rhs: Const32) -> Instruction, make_instr_imm16: fn(result: Register, lhs: Register, rhs: Const16) -> Instruction, make_instr_imm16_rev: fn(result: Register, lhs: Const16, rhs: Register) -> Instruction, consteval: fn(TypedValue, TypedValue) -> Result, @@ -1272,7 +1312,7 @@ impl<'parser> FuncTranslator<'parser> { ) -> Result, ) -> Result<(), TranslationError> where - T: WasmInteger, + T: WasmInteger + Into + TryInto>, { bail_unreachable!(self); match self.alloc.stack.pop2() { @@ -1297,7 +1337,7 @@ impl<'parser> FuncTranslator<'parser> { // Optimization was applied: return early. return Ok(()); } - self.push_binary_instr_imm(lhs, rhs, make_instr) + self.push_binary_instr_imm(lhs, T::from(rhs), make_instr, make_instr_assign_imm32) } (TypedProvider::Const(lhs), TypedProvider::Register(rhs)) => { if self.try_push_binary_instr_imm16_rev(T::from(lhs), rhs, make_instr_imm16_rev)? { diff --git a/crates/wasmi/src/engine/regmach/translator/result_mut.rs b/crates/wasmi/src/engine/regmach/translator/result_mut.rs index 93c6f69c61..66a148d983 100644 --- a/crates/wasmi/src/engine/regmach/translator/result_mut.rs +++ b/crates/wasmi/src/engine/regmach/translator/result_mut.rs @@ -10,7 +10,7 @@ use crate::{ regmach::{ bytecode::{ BinAssignInstr, - BinAssignInstrImm32, + BinAssignInstrImm, BinInstr, BinInstrImm16, CopysignImmInstr, @@ -621,11 +621,11 @@ impl Instruction { Instruction::I32XorAssignImm(instr) | Instruction::I32ShlAssignImm(instr) | Instruction::I32ShrSAssignImm(instr) | + Instruction::I32ShrUAssignImm(instr) | Instruction::I32RotlAssignImm(instr) | Instruction::I32RotrAssignImm(instr) => instr.result_mut(), Instruction::I32DivUAssignImm(instr) | - Instruction::I32RemUAssignImm(instr) | - Instruction::I32ShrUAssignImm(instr) => instr.result_mut(), + Instruction::I32RemUAssignImm(instr) => instr.result_mut(), Instruction::I64AddAssignImm32(instr) | Instruction::I64SubAssignImm32(instr) | Instruction::I64MulAssignImm32(instr) | @@ -636,25 +636,25 @@ impl Instruction { Instruction::I64XorAssignImm32(instr) | Instruction::I64ShlAssignImm32(instr) | Instruction::I64ShrSAssignImm32(instr) | + Instruction::I64ShrUAssignImm32(instr) | Instruction::I64RotlAssignImm32(instr) | Instruction::I64RotrAssignImm32(instr) => instr.result_mut(), Instruction::I64DivUAssignImm32(instr) | - Instruction::I64RemUAssignImm32(instr) | - Instruction::I64ShrUAssignImm32(instr) => instr.result_mut(), + Instruction::I64RemUAssignImm32(instr) => instr.result_mut(), Instruction::F32AddAssignImm(instr) | Instruction::F32SubAssignImm(instr) | Instruction::F32MulAssignImm(instr) | Instruction::F32DivAssignImm(instr) | Instruction::F32MinAssignImm(instr) | - Instruction::F32MaxAssignImm(instr) | + Instruction::F32MaxAssignImm(instr) => instr.result_mut(), Instruction::F32CopysignAssignImm(instr) => instr.result_mut(), Instruction::F64AddAssignImm32(instr) | Instruction::F64SubAssignImm32(instr) | Instruction::F64MulAssignImm32(instr) | Instruction::F64DivAssignImm32(instr) | Instruction::F64MinAssignImm32(instr) | - Instruction::F64MaxAssignImm32(instr) | - Instruction::F64CopysignAssignImm32(instr) => instr.result_mut(), + Instruction::F64MaxAssignImm32(instr) => instr.result_mut(), + Instruction::F64CopysignAssignImm(instr) => instr.result_mut(), } } } @@ -721,7 +721,7 @@ impl BinAssignInstr { } } -impl BinAssignInstrImm32 { +impl BinAssignInstrImm { /// Returns the single `result` [`Register`] of the [`BinAssignInstrImm32`] if any. pub fn result_mut(&mut self) -> Option<&mut Register> { Some(&mut self.inout) diff --git a/crates/wasmi/src/engine/regmach/translator/visit.rs b/crates/wasmi/src/engine/regmach/translator/visit.rs index 889b8f7cfe..ca2eafb29e 100644 --- a/crates/wasmi/src/engine/regmach/translator/visit.rs +++ b/crates/wasmi/src/engine/regmach/translator/visit.rs @@ -1209,6 +1209,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative::( Instruction::i32_eq, Instruction::i32_eq_assign, + Instruction::i32_eq_assign_imm, Instruction::i32_eq_imm16, TypedValue::i32_eq, |this, lhs: Register, rhs: Register| { @@ -1227,6 +1228,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative::( Instruction::i32_ne, Instruction::i32_ne_assign, + Instruction::i32_ne_assign_imm, Instruction::i32_ne_imm16, TypedValue::i32_ne, |this, lhs: Register, rhs: Register| { @@ -1245,6 +1247,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i32_lt_s, Instruction::i32_lt_s_assign, + Instruction::i32_lt_s_assign_imm, Instruction::i32_lt_s_imm16, swap_ops!(Instruction::i32_gt_s_imm16), TypedValue::i32_lt_s, @@ -1279,6 +1282,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i32_lt_u, Instruction::i32_lt_u_assign, + Instruction::i32_lt_u_assign_imm, Instruction::i32_lt_u_imm16, swap_ops!(Instruction::i32_gt_u_imm16), TypedValue::i32_lt_u, @@ -1313,6 +1317,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i32_gt_s, Instruction::i32_gt_s_assign, + Instruction::i32_gt_s_assign_imm, Instruction::i32_gt_s_imm16, swap_ops!(Instruction::i32_lt_s_imm16), TypedValue::i32_gt_s, @@ -1347,6 +1352,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i32_gt_u, Instruction::i32_gt_u_assign, + Instruction::i32_gt_u_assign_imm, Instruction::i32_gt_u_imm16, swap_ops!(Instruction::i32_lt_u_imm16), TypedValue::i32_gt_u, @@ -1381,6 +1387,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i32_le_s, Instruction::i32_le_s_assign, + Instruction::i32_le_s_assign_imm, Instruction::i32_le_s_imm16, swap_ops!(Instruction::i32_ge_s_imm16), TypedValue::i32_le_s, @@ -1415,6 +1422,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i32_le_u, Instruction::i32_le_u_assign, + Instruction::i32_le_u_assign_imm, Instruction::i32_le_u_imm16, swap_ops!(Instruction::i32_ge_u_imm16), TypedValue::i32_le_u, @@ -1449,6 +1457,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i32_ge_s, Instruction::i32_ge_s_assign, + Instruction::i32_ge_s_assign_imm, Instruction::i32_ge_s_imm16, swap_ops!(Instruction::i32_le_s_imm16), TypedValue::i32_ge_s, @@ -1483,6 +1492,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i32_ge_u, Instruction::i32_ge_u_assign, + Instruction::i32_ge_u_assign_imm, Instruction::i32_ge_u_imm16, swap_ops!(Instruction::i32_le_u_imm16), TypedValue::i32_ge_u, @@ -1524,6 +1534,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative::( Instruction::i64_eq, Instruction::i64_eq_assign, + Instruction::i64_eq_assign_imm32, Instruction::i64_eq_imm16, TypedValue::i64_eq, |this, lhs: Register, rhs: Register| { @@ -1542,6 +1553,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative::( Instruction::i64_ne, Instruction::i64_ne_assign, + Instruction::i64_ne_assign_imm32, Instruction::i64_ne_imm16, TypedValue::i64_ne, |this, lhs: Register, rhs: Register| { @@ -1560,6 +1572,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i64_lt_s, Instruction::i64_lt_s_assign, + Instruction::i64_lt_s_assign_imm32, Instruction::i64_lt_s_imm16, swap_ops!(Instruction::i64_gt_s_imm16), TypedValue::i64_lt_s, @@ -1594,6 +1607,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i64_lt_u, Instruction::i64_lt_u_assign, + Instruction::i64_lt_u_assign_imm32, Instruction::i64_lt_u_imm16, swap_ops!(Instruction::i64_gt_u_imm16), TypedValue::i64_lt_u, @@ -1628,6 +1642,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i64_gt_s, Instruction::i64_gt_s_assign, + Instruction::i64_gt_s_assign_imm32, Instruction::i64_gt_s_imm16, swap_ops!(Instruction::i64_lt_s_imm16), TypedValue::i64_gt_s, @@ -1662,6 +1677,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i64_gt_u, Instruction::i64_gt_u_assign, + Instruction::i64_gt_u_assign_imm32, Instruction::i64_gt_u_imm16, swap_ops!(Instruction::i64_lt_u_imm16), TypedValue::i64_gt_u, @@ -1696,6 +1712,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i64_le_s, Instruction::i64_le_s_assign, + Instruction::i64_le_s_assign_imm32, Instruction::i64_le_s_imm16, swap_ops!(Instruction::i64_ge_s_imm16), TypedValue::i64_le_s, @@ -1730,6 +1747,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i64_le_u, Instruction::i64_le_u_assign, + Instruction::i64_le_u_assign_imm32, Instruction::i64_le_u_imm16, swap_ops!(Instruction::i64_ge_u_imm16), TypedValue::i64_le_u, @@ -1764,6 +1782,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i64_ge_s, Instruction::i64_ge_s_assign, + Instruction::i64_ge_s_assign_imm32, Instruction::i64_ge_s_imm16, swap_ops!(Instruction::i64_le_s_imm16), TypedValue::i64_ge_s, @@ -1798,6 +1817,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i64_ge_u, Instruction::i64_ge_u_assign, + Instruction::i64_ge_u_assign_imm32, Instruction::i64_ge_u_imm16, swap_ops!(Instruction::i64_le_u_imm16), TypedValue::i64_ge_u, @@ -1832,6 +1852,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative::( Instruction::f32_eq, Instruction::f32_eq_assign, + Instruction::f32_eq_assign_imm, TypedValue::f32_eq, Self::no_custom_opt, |this, _reg_in: Register, imm_in: f32| { @@ -1849,6 +1870,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative::( Instruction::f32_ne, Instruction::f32_ne_assign, + Instruction::f32_ne_assign_imm, TypedValue::f32_ne, Self::no_custom_opt, |this, _reg_in: Register, imm_in: f32| { @@ -1866,6 +1888,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f32_lt, Instruction::f32_lt_assign, + Instruction::f32_lt_assign_imm, TypedValue::f32_lt, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -1908,6 +1931,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f32_gt, Instruction::f32_gt_assign, + Instruction::f32_gt_assign_imm, TypedValue::f32_gt, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -1950,6 +1974,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f32_le, Instruction::f32_le_assign, + Instruction::f32_le_assign_imm, TypedValue::f32_le, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -1982,6 +2007,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f32_ge, Instruction::f32_ge_assign, + Instruction::f32_ge_assign_imm, TypedValue::f32_ge, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -2014,6 +2040,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative::( Instruction::f64_eq, Instruction::f64_eq_assign, + Instruction::f64_eq_assign_imm32, TypedValue::f64_eq, Self::no_custom_opt, |this, _reg_in: Register, imm_in: f64| { @@ -2031,6 +2058,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative::( Instruction::f64_ne, Instruction::f64_ne_assign, + Instruction::f64_ne_assign_imm32, TypedValue::f64_ne, Self::no_custom_opt, |this, _reg_in: Register, imm_in: f64| { @@ -2048,6 +2076,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f64_lt, Instruction::f64_lt_assign, + Instruction::f64_lt_assign_imm32, TypedValue::f64_lt, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -2090,6 +2119,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f64_gt, Instruction::f64_gt_assign, + Instruction::f64_gt_assign_imm32, TypedValue::f64_gt, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -2132,6 +2162,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f64_le, Instruction::f64_le_assign, + Instruction::f64_le_assign_imm32, TypedValue::f64_le, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -2164,6 +2195,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f64_ge, Instruction::f64_ge_assign, + Instruction::f64_ge_assign_imm32, TypedValue::f64_ge, |this, lhs: Register, rhs: Register| { if lhs == rhs { @@ -2208,6 +2240,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i32_add, Instruction::i32_add_assign, + Instruction::i32_add_assign_imm, Instruction::i32_add_imm16, TypedValue::i32_add, Self::no_custom_opt, @@ -2226,6 +2259,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i32_sub, Instruction::i32_sub_assign, + Instruction::i32_sub_assign_imm, Instruction::i32_sub_imm16, Instruction::i32_sub_imm16_rev, TypedValue::i32_sub, @@ -2253,6 +2287,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i32_mul, Instruction::i32_mul_assign, + Instruction::i32_mul_assign_imm, Instruction::i32_mul_imm16, TypedValue::i32_mul, Self::no_custom_opt, @@ -2276,6 +2311,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_divrem( Instruction::i32_div_s, Instruction::i32_div_s_assign, + Instruction::i32_div_s_assign_imm, Instruction::i32_div_s_imm16, Instruction::i32_div_s_imm16_rev, TypedValue::i32_div_s, @@ -2295,6 +2331,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_divrem::( Instruction::i32_div_u, Instruction::i32_div_u_assign, + Instruction::i32_div_u_assign_imm, Instruction::i32_div_u_imm16, Instruction::i32_div_u_imm16_rev, TypedValue::i32_div_u, @@ -2314,6 +2351,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_divrem( Instruction::i32_rem_s, Instruction::i32_rem_s_assign, + Instruction::i32_rem_s_assign_imm, Instruction::i32_rem_s_imm16, Instruction::i32_rem_s_imm16_rev, TypedValue::i32_rem_s, @@ -2333,6 +2371,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_divrem::( Instruction::i32_rem_u, Instruction::i32_rem_u_assign, + Instruction::i32_rem_u_assign_imm, Instruction::i32_rem_u_imm16, Instruction::i32_rem_u_imm16_rev, TypedValue::i32_rem_u, @@ -2352,6 +2391,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i32_and, Instruction::i32_and_assign, + Instruction::i32_and_assign_imm, Instruction::i32_and_imm16, TypedValue::i32_and, |this, lhs, rhs| { @@ -2385,6 +2425,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i32_or, Instruction::i32_or_assign, + Instruction::i32_or_assign_imm, Instruction::i32_or_imm16, TypedValue::i32_or, |this, lhs, rhs| { @@ -2418,6 +2459,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i32_xor, Instruction::i32_xor_assign, + Instruction::i32_xor_assign_imm, Instruction::i32_xor_imm16, TypedValue::i32_xor, |this, lhs, rhs| { @@ -2443,6 +2485,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift::( Instruction::i32_shl, Instruction::i32_shl_assign, + Instruction::i32_shl_assign_imm, Instruction::i32_shl_imm, Instruction::i32_shl_imm16_rev, TypedValue::i32_shl, @@ -2454,6 +2497,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift( Instruction::i32_shr_s, Instruction::i32_shr_s_assign, + Instruction::i32_shr_s_assign_imm, Instruction::i32_shr_s_imm, Instruction::i32_shr_s_imm16_rev, TypedValue::i32_shr_s, @@ -2472,6 +2516,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift::( Instruction::i32_shr_u, Instruction::i32_shr_u_assign, + Instruction::i32_shr_u_assign_imm, Instruction::i32_shr_u_imm, Instruction::i32_shr_u_imm16_rev, TypedValue::i32_shr_u, @@ -2483,6 +2528,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift( Instruction::i32_rotl, Instruction::i32_rotl_assign, + Instruction::i32_rotl_assign_imm, Instruction::i32_rotl_imm, Instruction::i32_rotl_imm16_rev, TypedValue::i32_rotl, @@ -2501,6 +2547,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift( Instruction::i32_rotr, Instruction::i32_rotr_assign, + Instruction::i32_rotr_assign_imm, Instruction::i32_rotr_imm, Instruction::i32_rotr_imm16_rev, TypedValue::i32_rotr, @@ -2531,6 +2578,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i64_add, Instruction::i64_add_assign, + Instruction::i64_shl_assign_imm32, Instruction::i64_add_imm16, TypedValue::i64_add, Self::no_custom_opt, @@ -2549,6 +2597,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary( Instruction::i64_sub, Instruction::i64_sub_assign, + Instruction::i64_sub_assign_imm32, Instruction::i64_sub_imm16, Instruction::i64_sub_imm16_rev, TypedValue::i64_sub, @@ -2576,6 +2625,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i64_mul, Instruction::i64_mul_assign, + Instruction::i64_mul_assign_imm32, Instruction::i64_mul_imm16, TypedValue::i64_mul, Self::no_custom_opt, @@ -2599,6 +2649,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_divrem( Instruction::i64_div_s, Instruction::i64_div_s_assign, + Instruction::i64_div_s_assign_imm32, Instruction::i64_div_s_imm16, Instruction::i64_div_s_imm16_rev, TypedValue::i64_div_s, @@ -2618,6 +2669,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_divrem::( Instruction::i64_div_u, Instruction::i64_div_u_assign, + Instruction::i64_div_u_assign_imm32, Instruction::i64_div_u_imm16, Instruction::i64_div_u_imm16_rev, TypedValue::i64_div_u, @@ -2637,6 +2689,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_divrem( Instruction::i64_rem_s, Instruction::i64_rem_s_assign, + Instruction::i64_rem_s_assign_imm32, Instruction::i64_rem_s_imm16, Instruction::i64_rem_s_imm16_rev, TypedValue::i64_rem_s, @@ -2656,6 +2709,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_divrem::( Instruction::i64_rem_u, Instruction::i64_rem_u_assign, + Instruction::i64_rem_u_assign_imm32, Instruction::i64_rem_u_imm16, Instruction::i64_rem_u_imm16_rev, TypedValue::i64_rem_u, @@ -2675,6 +2729,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i64_and, Instruction::i64_and_assign, + Instruction::i64_and_assign_imm32, Instruction::i64_and_imm16, TypedValue::i64_and, |this, lhs, rhs| { @@ -2708,6 +2763,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i64_or, Instruction::i64_or_assign, + Instruction::i64_or_assign_imm32, Instruction::i64_or_imm16, TypedValue::i64_or, |this, lhs, rhs| { @@ -2741,6 +2797,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_binary_commutative( Instruction::i64_xor, Instruction::i64_xor_assign, + Instruction::i64_xor_assign_imm32, Instruction::i64_xor_imm16, TypedValue::i64_xor, |this, lhs, rhs| { @@ -2766,6 +2823,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift::( Instruction::i64_shl, Instruction::i64_shl_assign, + Instruction::i64_shl_assign_imm32, Instruction::i64_shl_imm, Instruction::i64_shl_imm16_rev, TypedValue::i64_shl, @@ -2777,6 +2835,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift( Instruction::i64_shr_s, Instruction::i64_shr_s_assign, + Instruction::i64_shr_s_assign_imm32, Instruction::i64_shr_s_imm, Instruction::i64_shr_s_imm16_rev, TypedValue::i64_shr_s, @@ -2795,6 +2854,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift::( Instruction::i64_shr_u, Instruction::i64_shr_u_assign, + Instruction::i64_shr_u_assign_imm32, Instruction::i64_shr_u_imm, Instruction::i64_shr_u_imm16_rev, TypedValue::i64_shr_u, @@ -2806,6 +2866,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift( Instruction::i64_rotl, Instruction::i64_rotl_assign, + Instruction::i64_rotl_assign_imm32, Instruction::i64_rotl_imm, Instruction::i64_rotl_imm16_rev, TypedValue::i64_rotl, @@ -2824,6 +2885,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_shift( Instruction::i64_rotr, Instruction::i64_rotr_assign, + Instruction::i64_rotr_assign_imm32, Instruction::i64_rotr_imm, Instruction::i64_rotr_imm16_rev, TypedValue::i64_rotr, @@ -2870,6 +2932,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative( Instruction::f32_add, Instruction::f32_add_assign, + Instruction::f32_add_assign_imm, TypedValue::f32_add, Self::no_custom_opt, Self::no_custom_opt::, @@ -2880,6 +2943,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f32_sub, Instruction::f32_sub_assign, + Instruction::f32_sub_assign_imm, TypedValue::f32_sub, Self::no_custom_opt, Self::no_custom_opt::, @@ -2894,6 +2958,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative::( Instruction::f32_mul, Instruction::f32_mul_assign, + Instruction::f32_mul_assign_imm, TypedValue::f32_mul, Self::no_custom_opt, // Unfortunately we cannot apply `x * 0` or `0 * x` optimizations @@ -2907,6 +2972,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary::( Instruction::f32_div, Instruction::f32_div_assign, + Instruction::f32_div_assign_imm, TypedValue::f32_div, Self::no_custom_opt, Self::no_custom_opt, @@ -2918,6 +2984,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative( Instruction::f32_min, Instruction::f32_min_assign, + Instruction::f32_min_assign_imm, TypedValue::f32_min, Self::no_custom_opt, |this, reg: Register, value: f32| { @@ -2935,6 +3002,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative( Instruction::f32_max, Instruction::f32_max_assign, + Instruction::f32_max_assign_imm, TypedValue::f32_max, Self::no_custom_opt, |this, reg: Register, value: f32| { @@ -2952,6 +3020,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fcopysign::( Instruction::f32_copysign, Instruction::f32_copysign_assign, + // Instruction::f32_copysign_assign_imm, Instruction::f32_copysign_imm, TypedValue::f32_copysign, ) @@ -2989,6 +3058,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative( Instruction::f64_add, Instruction::f64_add_assign, + Instruction::f64_add_assign_imm32, TypedValue::f64_add, Self::no_custom_opt, Self::no_custom_opt::, @@ -2999,6 +3069,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary( Instruction::f64_sub, Instruction::f64_sub_assign, + Instruction::f64_sub_assign_imm32, TypedValue::f64_sub, Self::no_custom_opt, Self::no_custom_opt::, @@ -3013,6 +3084,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative::( Instruction::f64_mul, Instruction::f64_mul_assign, + Instruction::f64_mul_assign_imm32, TypedValue::f64_mul, Self::no_custom_opt, // Unfortunately we cannot apply `x * 0` or `0 * x` optimizations @@ -3026,6 +3098,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary::( Instruction::f64_div, Instruction::f64_div_assign, + Instruction::f64_div_assign_imm32, TypedValue::f64_div, Self::no_custom_opt, Self::no_custom_opt, @@ -3037,6 +3110,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative( Instruction::f64_min, Instruction::f64_min_assign, + Instruction::f64_min_assign_imm32, TypedValue::f64_min, Self::no_custom_opt, |this, reg: Register, value: f64| { @@ -3054,6 +3128,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fbinary_commutative( Instruction::f64_max, Instruction::f64_max_assign, + Instruction::f64_max_assign_imm32, TypedValue::f64_max, Self::no_custom_opt, |this, reg: Register, value: f64| { @@ -3071,6 +3146,7 @@ impl<'a> VisitOperator<'a> for FuncTranslator<'a> { self.translate_fcopysign::( Instruction::f64_copysign, Instruction::f64_copysign_assign, + // Instruction::f64_copysign_assign_imm32, Instruction::f64_copysign_imm, TypedValue::f64_copysign, ) diff --git a/crates/wasmi/src/engine/regmach/translator/visit_register.rs b/crates/wasmi/src/engine/regmach/translator/visit_register.rs index c70b35958a..7c748e8786 100644 --- a/crates/wasmi/src/engine/regmach/translator/visit_register.rs +++ b/crates/wasmi/src/engine/regmach/translator/visit_register.rs @@ -1,6 +1,6 @@ use crate::engine::regmach::bytecode::{ BinAssignInstr, - BinAssignInstrImm32, + BinAssignInstrImm, BinInstr, BinInstrImm16, BranchBinOpInstr, @@ -681,9 +681,7 @@ impl VisitInputRegisters for Instruction { Instruction::F64MaxAssign(instr) => instr.visit_input_registers(f), Instruction::F64MaxAssignImm32(instr) => instr.visit_input_registers(f), Instruction::F64CopysignAssign(instr) => instr.visit_input_registers(f), - Instruction::F64CopysignAssignImm32(instr) => instr.visit_input_registers(f), - - + Instruction::F64CopysignAssignImm(instr) => instr.visit_input_registers(f), } } } @@ -702,7 +700,7 @@ impl BinAssignInstr { } } -impl BinAssignInstrImm32 { +impl BinAssignInstrImm { fn visit_input_registers(&mut self, _f: impl FnMut(&mut Register)) { // Nothing to do. } From b4abb50622478ade486314c09e8b88ba91858eee Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 14:11:22 +0100 Subject: [PATCH 08/17] clean up BinInstrImm32 and CopysignImmInstr types --- .../src/engine/regmach/bytecode/construct.rs | 7 ++-- .../wasmi/src/engine/regmach/bytecode/mod.rs | 6 +-- .../src/engine/regmach/bytecode/utils.rs | 39 +++++-------------- .../engine/regmach/executor/instrs/binary.rs | 14 +++---- .../engine/regmach/translator/result_mut.rs | 12 +----- .../regmach/translator/visit_register.rs | 29 +------------- 6 files changed, 27 insertions(+), 80 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/bytecode/construct.rs b/crates/wasmi/src/engine/regmach/bytecode/construct.rs index ba1c543ca6..2afcc70cbf 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/construct.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/construct.rs @@ -1,10 +1,11 @@ use super::{ - utils::{BranchOffset16, CopysignImmInstr, Sign}, + utils::{BranchOffset16, Sign}, AnyConst32, BinAssignInstr, BinAssignInstrImm, BinAssignInstrImm32, BinInstr, + BinInstrImm, BinInstrImm16, BranchBinOpInstr, BranchBinOpInstrImm, @@ -740,7 +741,7 @@ impl Instruction { /// Creates a new [`Instruction::F32CopysignImm`] instruction. pub fn f32_copysign_imm(result: Register, lhs: Register, rhs: Sign) -> Self { - Self::F32CopysignImm(CopysignImmInstr { result, lhs, rhs }) + Self::F32CopysignImm(BinInstrImm::new(result, lhs, rhs)) } /// Creates a new [`Instruction::F32CopysignAssignImm`] instruction. @@ -750,7 +751,7 @@ impl Instruction { /// Creates a new [`Instruction::F64CopysignImm`] instruction. pub fn f64_copysign_imm(result: Register, lhs: Register, rhs: Sign) -> Self { - Self::F64CopysignImm(CopysignImmInstr { result, lhs, rhs }) + Self::F64CopysignImm(BinInstrImm::new(result, lhs, rhs)) } /// Creates a new [`Instruction::F64CopysignAssignImm`] instruction. diff --git a/crates/wasmi/src/engine/regmach/bytecode/mod.rs b/crates/wasmi/src/engine/regmach/bytecode/mod.rs index edc4906cbb..3b7bbf6b12 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/mod.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/mod.rs @@ -14,12 +14,12 @@ pub(crate) use self::{ BinAssignInstrImm, BinAssignInstrImm32, BinInstr, + BinInstrImm, BinInstrImm16, BranchBinOpInstr, BranchBinOpInstrImm, BranchOffset16, CallIndirectParams, - CopysignImmInstr, LoadAtInstr, LoadInstr, LoadOffset16Instr, @@ -2958,9 +2958,9 @@ pub enum Instruction { /// Wasm `f64.copysign` instruction: `r0 = copysign(r1, r2)` F64Copysign(BinInstr), /// Wasm `f32.copysign` instruction with immediate: `r0 = copysign(r1, c0)` - F32CopysignImm(CopysignImmInstr), + F32CopysignImm(BinInstrImm), /// Wasm `f64.copysign` instruction with immediate: `r0 = copysign(r1, c0)` - F64CopysignImm(CopysignImmInstr), + F64CopysignImm(BinInstrImm), /// Wasm `i32.wrap_i64` instruction. I32WrapI64(UnaryInstr), diff --git a/crates/wasmi/src/engine/regmach/bytecode/utils.rs b/crates/wasmi/src/engine/regmach/bytecode/utils.rs index 9655821603..e6ab26864d 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/utils.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/utils.rs @@ -249,13 +249,16 @@ impl BinInstr { } } -/// A binary instruction with an immediate right-hand side value. +/// A binary instruction with a 16-bit encoded immediate value. +pub type BinInstrImm16 = BinInstrImm>; + +/// A binary instruction with an immediate value. /// /// # Note /// /// Optimized for small constant values that fit into 16-bit. -#[derive(Copy, Clone, PartialEq, Eq)] -pub struct BinInstrImm16 { +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub struct BinInstrImm { /// The register storing the result of the computation. pub result: Register, /// The register holding one of the operands. @@ -271,25 +274,12 @@ pub struct BinInstrImm16 { /// /// The instruction decides if this operand is the left-hand or /// right-hand operand for the computation. - pub imm_in: Const16, + pub imm_in: T, } -impl fmt::Debug for BinInstrImm16 -where - Const16: fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("BinInstrImm16") - .field("result", &self.result) - .field("reg_in", &self.reg_in) - .field("imm_in", &self.imm_in) - .finish() - } -} - -impl BinInstrImm16 { +impl BinInstrImm { /// Creates a new [`BinInstrImm16`]. - pub fn new(result: Register, reg_in: Register, imm_in: Const16) -> Self { + pub fn new(result: Register, reg_in: Register, imm_in: T) -> Self { Self { result, reg_in, @@ -514,17 +504,6 @@ impl Sign { } } -/// The `f32.copysign` or `f64.copysign` instruction with an immediate value. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub struct CopysignImmInstr { - /// The result register. - pub result: Register, - /// The input register. - pub lhs: Register, - /// The sign to copy. - pub rhs: Sign, -} - /// Auxiliary [`Instruction`] parameter to encode call parameters for indirect call instructions. #[derive(Debug, Copy, Clone, PartialEq, Eq)] pub struct CallIndirectParams { diff --git a/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs b/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs index 57198ec2f7..c66b52ca75 100644 --- a/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs +++ b/crates/wasmi/src/engine/regmach/executor/instrs/binary.rs @@ -6,8 +6,8 @@ use crate::{ BinAssignInstrImm, BinAssignInstrImm32, BinInstr, + BinInstrImm, BinInstrImm16, - CopysignImmInstr, Sign, }, }; @@ -222,9 +222,9 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { impl<'ctx, 'engine> Executor<'ctx, 'engine> { /// Executes an [`Instruction::F32CopysignImm`]. #[inline(always)] - pub fn execute_f32_copysign_imm(&mut self, instr: CopysignImmInstr) { - let lhs = self.get_register(instr.lhs); - let rhs = instr.rhs.to_f32(); + pub fn execute_f32_copysign_imm(&mut self, instr: BinInstrImm) { + let lhs = self.get_register(instr.reg_in); + let rhs = instr.imm_in.to_f32(); self.set_register(instr.result, UntypedValue::f32_copysign(lhs, rhs.into())); self.next_instr() } @@ -240,9 +240,9 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { /// Executes an [`Instruction::F64CopysignImm`]. #[inline(always)] - pub fn execute_f64_copysign_imm(&mut self, instr: CopysignImmInstr) { - let lhs = self.get_register(instr.lhs); - let rhs = instr.rhs.to_f64(); + pub fn execute_f64_copysign_imm(&mut self, instr: BinInstrImm) { + let lhs = self.get_register(instr.reg_in); + let rhs = instr.imm_in.to_f64(); self.set_register(instr.result, UntypedValue::f64_copysign(lhs, rhs.into())); self.next_instr() } diff --git a/crates/wasmi/src/engine/regmach/translator/result_mut.rs b/crates/wasmi/src/engine/regmach/translator/result_mut.rs index 66a148d983..06a3fb29e0 100644 --- a/crates/wasmi/src/engine/regmach/translator/result_mut.rs +++ b/crates/wasmi/src/engine/regmach/translator/result_mut.rs @@ -12,8 +12,7 @@ use crate::{ BinAssignInstr, BinAssignInstrImm, BinInstr, - BinInstrImm16, - CopysignImmInstr, + BinInstrImm, Instruction, LoadAtInstr, LoadInstr, @@ -756,7 +755,7 @@ impl BinInstr { } } -impl BinInstrImm16 { +impl BinInstrImm { /// Returns the single `result` [`Register`] of the [`BinInstrImm16`] if any. pub fn result_mut(&mut self) -> Option<&mut Register> { Some(&mut self.result) @@ -769,10 +768,3 @@ impl UnaryInstr { Some(&mut self.result) } } - -impl CopysignImmInstr { - /// Returns the single `result` [`Register`] of the [`CopysignImmInstr`] if any. - pub fn result_mut(&mut self) -> Option<&mut Register> { - Some(&mut self.result) - } -} diff --git a/crates/wasmi/src/engine/regmach/translator/visit_register.rs b/crates/wasmi/src/engine/regmach/translator/visit_register.rs index 7c748e8786..1fafffcf1f 100644 --- a/crates/wasmi/src/engine/regmach/translator/visit_register.rs +++ b/crates/wasmi/src/engine/regmach/translator/visit_register.rs @@ -2,11 +2,10 @@ use crate::engine::regmach::bytecode::{ BinAssignInstr, BinAssignInstrImm, BinInstr, - BinInstrImm16, + BinInstrImm, BranchBinOpInstr, BranchBinOpInstrImm, Const16, - CopysignImmInstr, Instruction, LoadAtInstr, LoadInstr, @@ -814,36 +813,12 @@ impl VisitInputRegisters for BinInstr { } } -impl VisitInputRegisters for BinInstrImm16 { +impl VisitInputRegisters for BinInstrImm { fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Register)) { f(&mut self.reg_in) } } -impl VisitInputRegisters for BinInstrImm16 { - fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Register)) { - f(&mut self.reg_in) - } -} - -impl VisitInputRegisters for BinInstrImm16 { - fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Register)) { - f(&mut self.reg_in) - } -} - -impl VisitInputRegisters for BinInstrImm16 { - fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Register)) { - f(&mut self.reg_in) - } -} - -impl VisitInputRegisters for CopysignImmInstr { - fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Register)) { - f(&mut self.lhs) - } -} - impl VisitInputRegisters for RegisterSpan { fn visit_input_registers(&mut self, mut f: impl FnMut(&mut Register)) { f(self.head_mut()) From c4d3288301cccdd24a969e474456336a66625792 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 18:58:13 +0100 Subject: [PATCH 09/17] minor cleanup of result_mut impl --- crates/wasmi/src/engine/regmach/translator/result_mut.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/translator/result_mut.rs b/crates/wasmi/src/engine/regmach/translator/result_mut.rs index 06a3fb29e0..0b45c26f7a 100644 --- a/crates/wasmi/src/engine/regmach/translator/result_mut.rs +++ b/crates/wasmi/src/engine/regmach/translator/result_mut.rs @@ -70,7 +70,7 @@ impl Instruction { Instruction::Branch { .. } | Instruction::BranchEqz { .. } | Instruction::BranchNez { .. } | - Instruction::BranchTable { .. } => None, + Instruction::BranchTable { .. } | Instruction::BranchI32Eq(_) | Instruction::BranchI32EqImm(_) | Instruction::BranchI32Ne(_) | @@ -161,7 +161,7 @@ impl Instruction { Instruction::TableGetImm { result, .. } | Instruction::TableSize { result, .. } => Some(result), Instruction::TableSet { .. } | - Instruction::TableSetAt { .. } => None, + Instruction::TableSetAt { .. } | Instruction::TableCopy { .. } | Instruction::TableCopyTo { .. } | Instruction::TableCopyFrom { .. } | From 2b88c9299a64e599d26ef6d7cf16e9c58493ad68 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 18:58:24 +0100 Subject: [PATCH 10/17] initial implementation of relink_result --- .../src/engine/regmach/bytecode/immediate.rs | 24 + .../src/engine/regmach/translator/mod.rs | 1 + .../regmach/translator/relink_result.rs | 718 ++++++++++++++++++ 3 files changed, 743 insertions(+) create mode 100644 crates/wasmi/src/engine/regmach/translator/relink_result.rs diff --git a/crates/wasmi/src/engine/regmach/bytecode/immediate.rs b/crates/wasmi/src/engine/regmach/bytecode/immediate.rs index 2d7cf9a185..0800be49af 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/immediate.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/immediate.rs @@ -213,6 +213,30 @@ impl PartialEq for Const32 { impl Eq for Const32 {} +impl From> for Const32 +where + T: From>, + Const32: From, +{ + fn from(value: Const16) -> Self { + Self::from(T::from(value)) + } +} + +impl From> for Const32 { + fn from(value: Const16) -> Self { + // Note: The `as` cast is lossless as guaranteed by `Const16`. + Self::from(i64::from(value) as i32) + } +} + +impl From> for Const32 { + fn from(value: Const16) -> Self { + // Note: The `as` cast is lossless as guaranteed by `Const16`. + Self::from(u64::from(value) as u32) + } +} + impl From for Const32 { fn from(value: i32) -> Self { Self::new(AnyConst32::from_i32(value)) diff --git a/crates/wasmi/src/engine/regmach/translator/mod.rs b/crates/wasmi/src/engine/regmach/translator/mod.rs index b8335550b1..a9827f5986 100644 --- a/crates/wasmi/src/engine/regmach/translator/mod.rs +++ b/crates/wasmi/src/engine/regmach/translator/mod.rs @@ -3,6 +3,7 @@ mod control_frame; mod control_stack; mod instr_encoder; +mod relink_result; mod result_mut; mod stack; mod typed_value; diff --git a/crates/wasmi/src/engine/regmach/translator/relink_result.rs b/crates/wasmi/src/engine/regmach/translator/relink_result.rs new file mode 100644 index 0000000000..b80f91f11b --- /dev/null +++ b/crates/wasmi/src/engine/regmach/translator/relink_result.rs @@ -0,0 +1,718 @@ +use core::mem; + +use wasmi_core::UntypedValue; + +use crate::{ + engine::{ + bytecode::{FuncIdx, SignatureIdx}, + regmach::{ + bytecode::{ + BinAssignInstr, + BinAssignInstrImm, + BinInstr, + BinInstrImm, + Const16, + Const32, + Instruction, + LoadAtInstr, + LoadInstr, + LoadOffset16Instr, + Register, + RegisterSpan, + }, + code_map::CompiledFuncEntity, + translator::stack::ValueStack, + }, + CompiledFunc, + TranslationError, + }, + module::ModuleResources, +}; + +macro_rules! relink_binop { + ($this:expr, $instr:ident, $new_result:ident, $old_result:ident, $make_instr:expr) => { + match relink_binop($instr, $new_result, $old_result, $make_instr)? { + RelinkResult::Unchanged => Ok(false), + RelinkResult::Relinked => Ok(true), + RelinkResult::Exchanged(new_instr) => relink_exchange($this, new_instr), + } + }; +} + +macro_rules! relink_binop_imm16 { + ($ty:ty, $this:expr, $instr:ident, $new_result:ident, $old_result:ident, $make_instr:expr) => { + match relink_binop_imm16::<$ty>($instr, $new_result, $old_result, $make_instr)? { + RelinkResult::Unchanged => Ok(false), + RelinkResult::Relinked => Ok(true), + RelinkResult::Exchanged(new_instr) => relink_exchange($this, new_instr), + } + }; +} + +macro_rules! relink_binop_assign { + ($this:expr, $instr:ident, $new_result:ident, $old_result:ident, $make_instr:expr) => { + match relink_binop_assign($instr, $new_result, $old_result, $make_instr) { + None => Ok(false), + Some(new_instr) => relink_exchange($this, new_instr), + } + }; +} + +macro_rules! relink_binop_assign_imm { + ($ty:ty, $this:expr, $instr:ident, $stack:ident, $new_result:ident, $old_result:ident, $make_instr:expr, $make_instr_imm:expr) => { + match relink_binop_assign_imm::<$ty>( + $instr, + $stack, + $new_result, + $old_result, + $make_instr, + $make_instr_imm, + )? { + None => Ok(false), + Some(new_instr) => relink_exchange($this, new_instr), + } + }; +} + +macro_rules! relink_binop_assign_fimm { + ($ty:ty, $this:expr, $instr:ident, $stack:ident, $new_result:ident, $old_result:ident, $make_instr:expr) => { + match relink_binop_assign_fimm::<$ty>( + $instr, + $stack, + $new_result, + $old_result, + $make_instr, + )? { + None => Ok(false), + Some(new_instr) => relink_exchange($this, new_instr), + } + }; +} + +impl Instruction { + #[rustfmt::skip] + pub fn relink_result( + &mut self, + stack: &mut ValueStack, + res: &ModuleResources, + new_result: Register, + old_result: Register, + ) -> Result { + use Instruction as I; + match self { + I::TableIdx(_) + | I::DataSegmentIdx(_) + | I::ElementSegmentIdx(_) + | I::Const32(_) + | I::I64Const32(_) + | I::F64Const32(_) + | I::Register(_) + | I::Register2(_) + | I::Register3(_) + | I::RegisterList(_) + | I::CallIndirectParams(_) + | I::CallIndirectParamsImm16(_) + | I::Trap(_) + | I::ConsumeFuel(_) + | I::Return + | I::ReturnReg { .. } + | I::ReturnReg2 { .. } + | I::ReturnReg3 { .. } + | I::ReturnImm32 { .. } + | I::ReturnI64Imm32 { .. } + | I::ReturnF64Imm32 { .. } + | I::ReturnSpan { .. } + | I::ReturnMany { .. } + | I::ReturnNez { .. } + | I::ReturnNezReg { .. } + | I::ReturnNezReg2 { .. } + | I::ReturnNezImm32 { .. } + | I::ReturnNezI64Imm32 { .. } + | I::ReturnNezF64Imm32 { .. } + | I::ReturnNezSpan { .. } + | I::ReturnNezMany { .. } + | I::Branch { .. } + | I::BranchEqz { .. } + | I::BranchNez { .. } + | I::BranchTable { .. } + | I::BranchI32Eq(_) + | I::BranchI32EqImm(_) + | I::BranchI32Ne(_) + | I::BranchI32NeImm(_) + | I::BranchI32LtS(_) + | I::BranchI32LtSImm(_) + | I::BranchI32LtU(_) + | I::BranchI32LtUImm(_) + | I::BranchI32LeS(_) + | I::BranchI32LeSImm(_) + | I::BranchI32LeU(_) + | I::BranchI32LeUImm(_) + | I::BranchI32GtS(_) + | I::BranchI32GtSImm(_) + | I::BranchI32GtU(_) + | I::BranchI32GtUImm(_) + | I::BranchI32GeS(_) + | I::BranchI32GeSImm(_) + | I::BranchI32GeU(_) + | I::BranchI32GeUImm(_) + | I::BranchI64Eq(_) + | I::BranchI64EqImm(_) + | I::BranchI64Ne(_) + | I::BranchI64NeImm(_) + | I::BranchI64LtS(_) + | I::BranchI64LtSImm(_) + | I::BranchI64LtU(_) + | I::BranchI64LtUImm(_) + | I::BranchI64LeS(_) + | I::BranchI64LeSImm(_) + | I::BranchI64LeU(_) + | I::BranchI64LeUImm(_) + | I::BranchI64GtS(_) + | I::BranchI64GtSImm(_) + | I::BranchI64GtU(_) + | I::BranchI64GtUImm(_) + | I::BranchI64GeS(_) + | I::BranchI64GeSImm(_) + | I::BranchI64GeU(_) + | I::BranchI64GeUImm(_) + | I::BranchF32Eq(_) + | I::BranchF32Ne(_) + | I::BranchF32Lt(_) + | I::BranchF32Le(_) + | I::BranchF32Gt(_) + | I::BranchF32Ge(_) + | I::BranchF64Eq(_) + | I::BranchF64Ne(_) + | I::BranchF64Lt(_) + | I::BranchF64Le(_) + | I::BranchF64Gt(_) + | I::BranchF64Ge(_) => Ok(false), + I::Copy { result, .. } + | I::CopyImm32 { result, .. } + | I::CopyI64Imm32 { result, .. } + | I::CopyF64Imm32 { result, .. } => relink_simple(result, new_result, old_result), + I::CopySpan { .. } + | I::CopySpanNonOverlapping { .. } + | I::Copy2 { .. } + | I::CopyMany { .. } + | I::CopyManyNonOverlapping { .. } + | I::ReturnCallInternal0 { .. } + | I::ReturnCallInternal { .. } + | I::ReturnCallImported0 { .. } + | I::ReturnCallImported { .. } + | I::ReturnCallIndirect0 { .. } + | I::ReturnCallIndirect { .. } => Ok(false), + I::CallInternal0 { results, func } | I::CallInternal { results, func } => { + relink_call_internal(results, *func, res, new_result, old_result) + } + I::CallImported0 { results, func } | I::CallImported { results, func } => { + relink_call_imported(results, *func, res, new_result, old_result) + } + I::CallIndirect0 { results, func_type } | I::CallIndirect { results, func_type } => { + relink_call_indirect(results, *func_type, res, new_result, old_result) + } + I::Select { result, .. } + | I::SelectRev { result, .. } + | I::SelectImm32 { + result_or_condition: result, + .. + } + | I::SelectI64Imm32 { + result_or_condition: result, + .. + } + | I::SelectF64Imm32 { + result_or_condition: result, + .. + } => { + // Note: the `result_or_condition` necessarily points to the actual `result` + // register since we make sure elsewhere that only the correct instruction + // word is given to this method. + relink_simple(result, new_result, old_result) + } + I::RefFunc { result, .. } + | I::TableGet { result, .. } + | I::TableGetImm { result, .. } + | I::TableSize { result, .. } => relink_simple(result, new_result, old_result), + I::TableSet { .. } + | I::TableSetAt { .. } + | I::TableCopy { .. } + | I::TableCopyTo { .. } + | I::TableCopyFrom { .. } + | I::TableCopyFromTo { .. } + | I::TableCopyExact { .. } + | I::TableCopyToExact { .. } + | I::TableCopyFromExact { .. } + | I::TableCopyFromToExact { .. } + | I::TableInit { .. } + | I::TableInitTo { .. } + | I::TableInitFrom { .. } + | I::TableInitFromTo { .. } + | I::TableInitExact { .. } + | I::TableInitToExact { .. } + | I::TableInitFromExact { .. } + | I::TableInitFromToExact { .. } + | I::TableFill { .. } + | I::TableFillAt { .. } + | I::TableFillExact { .. } + | I::TableFillAtExact { .. } => Ok(false), + I::TableGrow { result, .. } | I::TableGrowImm { result, .. } => { + relink_simple(result, new_result, old_result) + } + I::ElemDrop(_) | I::DataDrop(_) => Ok(false), + I::MemorySize { result } + | I::MemoryGrow { result, .. } + | I::MemoryGrowBy { result, .. } => relink_simple(result, new_result, old_result), + I::MemoryCopy { .. } + | I::MemoryCopyTo { .. } + | I::MemoryCopyFrom { .. } + | I::MemoryCopyFromTo { .. } + | I::MemoryCopyExact { .. } + | I::MemoryCopyToExact { .. } + | I::MemoryCopyFromExact { .. } + | I::MemoryCopyFromToExact { .. } + | I::MemoryFill { .. } + | I::MemoryFillAt { .. } + | I::MemoryFillImm { .. } + | I::MemoryFillExact { .. } + | I::MemoryFillAtImm { .. } + | I::MemoryFillAtExact { .. } + | I::MemoryFillImmExact { .. } + | I::MemoryFillAtImmExact { .. } + | I::MemoryInit { .. } + | I::MemoryInitTo { .. } + | I::MemoryInitFrom { .. } + | I::MemoryInitFromTo { .. } + | I::MemoryInitExact { .. } + | I::MemoryInitToExact { .. } + | I::MemoryInitFromExact { .. } + | I::MemoryInitFromToExact { .. } => Ok(false), + I::GlobalGet { result, .. } => relink_simple(result, new_result, old_result), + I::GlobalSet { .. } | I::GlobalSetI32Imm16 { .. } | I::GlobalSetI64Imm16 { .. } => { + Ok(false) + } + I::I32Load(instr) => relink_simple(instr, new_result, old_result), + I::I32LoadAt(instr) => relink_simple(instr, new_result, old_result), + I::I32LoadOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I64Load(instr) => relink_simple(instr, new_result, old_result), + I::I64LoadAt(instr) => relink_simple(instr, new_result, old_result), + I::I64LoadOffset16(instr) => relink_simple(instr, new_result, old_result), + I::F32Load(instr) => relink_simple(instr, new_result, old_result), + I::F32LoadAt(instr) => relink_simple(instr, new_result, old_result), + I::F32LoadOffset16(instr) => relink_simple(instr, new_result, old_result), + I::F64Load(instr) => relink_simple(instr, new_result, old_result), + I::F64LoadAt(instr) => relink_simple(instr, new_result, old_result), + I::F64LoadOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I32Load8s(instr) => relink_simple(instr, new_result, old_result), + I::I32Load8sAt(instr) => relink_simple(instr, new_result, old_result), + I::I32Load8sOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I32Load8u(instr) => relink_simple(instr, new_result, old_result), + I::I32Load8uAt(instr) => relink_simple(instr, new_result, old_result), + I::I32Load8uOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I32Load16s(instr) => relink_simple(instr, new_result, old_result), + I::I32Load16sAt(instr) => relink_simple(instr, new_result, old_result), + I::I32Load16sOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I32Load16u(instr) => relink_simple(instr, new_result, old_result), + I::I32Load16uAt(instr) => relink_simple(instr, new_result, old_result), + I::I32Load16uOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I64Load8s(instr) => relink_simple(instr, new_result, old_result), + I::I64Load8sAt(instr) => relink_simple(instr, new_result, old_result), + I::I64Load8sOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I64Load8u(instr) => relink_simple(instr, new_result, old_result), + I::I64Load8uAt(instr) => relink_simple(instr, new_result, old_result), + I::I64Load8uOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I64Load16s(instr) => relink_simple(instr, new_result, old_result), + I::I64Load16sAt(instr) => relink_simple(instr, new_result, old_result), + I::I64Load16sOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I64Load16u(instr) => relink_simple(instr, new_result, old_result), + I::I64Load16uAt(instr) => relink_simple(instr, new_result, old_result), + I::I64Load16uOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I64Load32s(instr) => relink_simple(instr, new_result, old_result), + I::I64Load32sAt(instr) => relink_simple(instr, new_result, old_result), + I::I64Load32sOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I64Load32u(instr) => relink_simple(instr, new_result, old_result), + I::I64Load32uAt(instr) => relink_simple(instr, new_result, old_result), + I::I64Load32uOffset16(instr) => relink_simple(instr, new_result, old_result), + I::I32Store(_) + | I::I32StoreOffset16(_) + | I::I32StoreOffset16Imm16(_) + | I::I32StoreAt(_) + | I::I32StoreAtImm16(_) + | I::I32Store8(_) + | I::I32Store8Offset16(_) + | I::I32Store8Offset16Imm(_) + | I::I32Store8At(_) + | I::I32Store8AtImm(_) + | I::I32Store16(_) + | I::I32Store16Offset16(_) + | I::I32Store16Offset16Imm(_) + | I::I32Store16At(_) + | I::I32Store16AtImm(_) + | I::I64Store(_) + | I::I64StoreOffset16(_) + | I::I64StoreOffset16Imm16(_) + | I::I64StoreAt(_) + | I::I64StoreAtImm16(_) + | I::I64Store8(_) + | I::I64Store8Offset16(_) + | I::I64Store8Offset16Imm(_) + | I::I64Store8At(_) + | I::I64Store8AtImm(_) + | I::I64Store16(_) + | I::I64Store16Offset16(_) + | I::I64Store16Offset16Imm(_) + | I::I64Store16At(_) + | I::I64Store16AtImm(_) + | I::I64Store32(_) + | I::I64Store32Offset16(_) + | I::I64Store32Offset16Imm16(_) + | I::I64Store32At(_) + | I::I64Store32AtImm16(_) + | I::F32Store(_) + | I::F32StoreOffset16(_) + | I::F32StoreAt(_) + | I::F64Store(_) + | I::F64StoreOffset16(_) + | I::F64StoreAt(_) => Ok(false), + I::I32Eq(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_eq_assign), + I::I64Eq(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_eq_assign), + I::I32Ne(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_ne_assign), + I::I64Ne(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_ne_assign), + I::I32LtS(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_lt_s_assign), + I::I64LtS(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_lt_s_assign), + I::I32LtU(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_lt_u_assign), + I::I64LtU(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_lt_u_assign), + I::I32LeS(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_le_s_assign), + I::I64LeS(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_le_s_assign), + I::I32LeU(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_le_u_assign), + I::I64LeU(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_le_u_assign), + I::I32GtS(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_gt_s_assign), + I::I64GtS(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_gt_s_assign), + I::I32GtU(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_gt_u_assign), + I::I64GtU(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_gt_u_assign), + I::I32GeS(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_ge_s_assign), + I::I64GeS(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_ge_s_assign), + I::I32GeU(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_ge_u_assign), + I::I64GeU(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_ge_u_assign), + I::F32Eq(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_eq_assign), + I::F32Ne(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_ne_assign), + I::F32Lt(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_lt_assign), + I::F32Le(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_le_assign), + I::F32Gt(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_gt_assign), + I::F32Ge(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_ge_assign), + I::F64Eq(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_eq_assign), + I::F64Ne(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_ne_assign), + I::F64Lt(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_lt_assign), + I::F64Le(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_le_assign), + I::F64Gt(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_gt_assign), + I::F64Ge(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_ge_assign), + I::I32EqImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_eq_assign_imm), + I::I32NeImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_ne_assign_imm), + I::I32LtSImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_lt_s_assign_imm), + I::I32LtUImm16(instr) => relink_binop_imm16!(u32, self, instr, new_result, old_result, I::i32_lt_u_assign_imm), + I::I32LeSImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_le_s_assign_imm), + I::I32LeUImm16(instr) => relink_binop_imm16!(u32, self, instr, new_result, old_result, I::i32_le_u_assign_imm), + I::I32GtSImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_gt_s_assign_imm), + I::I32GtUImm16(instr) => relink_binop_imm16!(u32, self, instr, new_result, old_result, I::i32_gt_u_assign_imm), + I::I32GeSImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_ge_s_assign_imm), + I::I32GeUImm16(instr) => relink_binop_imm16!(u32, self, instr, new_result, old_result, I::i32_ge_u_assign_imm), + I::I64EqImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_eq_assign_imm32), + I::I64NeImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_ne_assign_imm32), + I::I64LtSImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_lt_s_assign_imm32), + I::I64LtUImm16(instr) => relink_binop_imm16!(u64, self, instr, new_result, old_result, I::i64_lt_u_assign_imm32), + I::I64LeSImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_le_s_assign_imm32), + I::I64LeUImm16(instr) => relink_binop_imm16!(u64, self, instr, new_result, old_result, I::i64_le_u_assign_imm32), + I::I64GtSImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_gt_s_assign_imm32), + I::I64GtUImm16(instr) => relink_binop_imm16!(u64, self, instr, new_result, old_result, I::i64_gt_u_assign_imm32), + I::I64GeSImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_ge_s_assign_imm32), + I::I64GeUImm16(instr) => relink_binop_imm16!(u64, self, instr, new_result, old_result, I::i64_ge_u_assign_imm32), + I::I32EqAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_eq), + I::I32NeAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_ne), + I::I32LtSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_lt_s), + I::I32LtUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_lt_u), + I::I32LeSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_le_s), + I::I32LeUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_le_u), + I::I32GtSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_gt_s), + I::I32GtUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_gt_u), + I::I32GeSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_ge_s), + I::I32GeUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_ge_u), + I::I64EqAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_eq), + I::I64NeAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_ne), + I::I64LtSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_lt_s), + I::I64LtUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_lt_u), + I::I64LeSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_le_s), + I::I64LeUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_le_u), + I::I64GtSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_gt_s), + I::I64GtUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_gt_u), + I::I64GeSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_ge_s), + I::I64GeUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_ge_u), + I::F32EqAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_eq), + I::F32NeAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_ne), + I::F32LtAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_lt), + I::F32LeAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_le), + I::F32GtAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_gt), + I::F32GeAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_ge), + I::F64EqAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_eq), + I::F64NeAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_ne), + I::F64LtAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_lt), + I::F64LeAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_le), + I::F64GtAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_gt), + I::F64GeAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_ge), + I::I32EqAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_eq, I::i32_eq_imm16), + I::I32NeAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_ne, I::i32_ne_imm16), + I::I32LtSAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_lt_s, I::i32_lt_s_imm16), + I::I32LtUAssignImm(instr) => relink_binop_assign_imm!(u32, self, instr, stack, new_result, old_result, I::i32_lt_u, I::i32_lt_u_imm16), + I::I32LeSAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_le_s, I::i32_le_s_imm16), + I::I32LeUAssignImm(instr) => relink_binop_assign_imm!(u32, self, instr, stack, new_result, old_result, I::i32_le_u, I::i32_le_u_imm16), + I::I32GtSAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_gt_s, I::i32_gt_s_imm16), + I::I32GtUAssignImm(instr) => relink_binop_assign_imm!(u32, self, instr, stack, new_result, old_result, I::i32_gt_u, I::i32_gt_u_imm16), + I::I32GeSAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_ge_s, I::i32_ge_s_imm16), + I::I32GeUAssignImm(instr) => relink_binop_assign_imm!(u32, self, instr, stack, new_result, old_result, I::i32_ge_u, I::i32_ge_u_imm16), + I::I64EqAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_eq, I::i64_eq_imm16), + I::I64NeAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_ne, I::i64_ne_imm16), + I::I64LtSAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_lt_s, I::i64_lt_s_imm16), + I::I64LtUAssignImm32(instr) => relink_binop_assign_imm!(u64, self, instr, stack, new_result, old_result, I::i64_lt_u, I::i64_lt_u_imm16), + I::I64LeSAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_le_s, I::i64_le_s_imm16), + I::I64LeUAssignImm32(instr) => relink_binop_assign_imm!(u64, self, instr, stack, new_result, old_result, I::i64_le_u, I::i64_le_u_imm16), + I::I64GtSAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_gt_s, I::i64_gt_s_imm16), + I::I64GtUAssignImm32(instr) => relink_binop_assign_imm!(u64, self, instr, stack, new_result, old_result, I::i64_gt_u, I::i64_gt_u_imm16), + I::I64GeSAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_ge_s, I::i64_ge_s_imm16), + I::I64GeUAssignImm32(instr) => relink_binop_assign_imm!(u64, self, instr, stack, new_result, old_result, I::i64_ge_u, I::i64_ge_u_imm16), + I::F32EqAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_eq), + I::F32NeAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_ne), + I::F32LtAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_lt), + I::F32LeAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_le), + I::F32GtAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_gt), + I::F32GeAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_ge), + I::F64EqAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_eq), + I::F64NeAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_ne), + I::F64LtAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_lt), + I::F64LeAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_le), + I::F64GtAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_gt), + I::F64GeAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_ge), + _ => todo!(), + } + } +} + +fn relink_exchange( + instr: &mut Instruction, + new_instr: Instruction, +) -> Result { + _ = mem::replace(instr, new_instr); + Ok(true) +} + +fn relink_simple( + result: &mut T, + new_result: Register, + old_result: Register, +) -> Result +where + T: ResultMut, +{ + let result = result.result_mut(); + if *result != old_result { + // Note: This is a safeguard to prevent miscompilations. + return Ok(false); + } + debug_assert_ne!(*result, new_result); + *result = new_result; + Ok(true) +} + +fn relink_call_internal( + results: &mut RegisterSpan, + func: CompiledFunc, + res: &ModuleResources, + new_result: Register, + old_result: Register, +) -> Result { + let len_results = res + .engine() + .resolve_func_2(func, CompiledFuncEntity::len_results); + if len_results != 1 { + return Ok(false); + } + relink_simple(results.head_mut(), new_result, old_result) +} + +fn relink_call_imported( + results: &mut RegisterSpan, + func: FuncIdx, + res: &ModuleResources, + new_result: Register, + old_result: Register, +) -> Result { + let func_idx = func.to_u32().into(); + let func_type = res.get_type_of_func(func_idx); + let len_results = res + .engine() + .resolve_func_type(func_type, |func_type| func_type.results().len()); + if len_results != 1 { + return Ok(false); + } + relink_simple(results.head_mut(), new_result, old_result) +} + +fn relink_call_indirect( + results: &mut RegisterSpan, + func_type: SignatureIdx, + res: &ModuleResources, + new_result: Register, + old_result: Register, +) -> Result { + let func_type_idx = func_type.to_u32().into(); + let func_type = res.get_func_type(func_type_idx); + let len_results = res + .engine() + .resolve_func_type(func_type, |func_type| func_type.results().len()); + if len_results != 1 { + return Ok(false); + } + relink_simple(results.head_mut(), new_result, old_result) +} + +#[derive(Debug, Copy, Clone)] +enum RelinkResult { + Unchanged, + Relinked, + Exchanged(Instruction), +} + +fn relink_binop( + instr: &mut BinInstr, + new_result: Register, + old_result: Register, + make_bin: fn(result: Register, rhs: Register) -> Instruction, +) -> Result { + if !relink_simple(instr, new_result, old_result)? { + return Ok(RelinkResult::Unchanged); + } + if instr.result == instr.lhs { + let new_instr = make_bin(new_result, instr.rhs); + return Ok(RelinkResult::Exchanged(new_instr)); + } + Ok(RelinkResult::Relinked) +} + +fn relink_binop_imm16( + instr: &mut BinInstrImm>, + new_result: Register, + old_result: Register, + make_bin: fn(result: Register, rhs: Const32) -> Instruction, +) -> Result +where + Const16: Into>, +{ + if !relink_simple(instr, new_result, old_result)? { + return Ok(RelinkResult::Unchanged); + } + if instr.result == instr.reg_in { + let new_instr = make_bin(new_result, instr.imm_in.into()); + return Ok(RelinkResult::Exchanged(new_instr)); + } + Ok(RelinkResult::Relinked) +} + +fn relink_binop_assign( + instr: &BinAssignInstr, + new_result: Register, + old_result: Register, + make_bin: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, +) -> Option { + if instr.inout != old_result { + // Note: This is a safeguard to prevent bugs. + return None; + } + debug_assert_ne!(instr.inout, new_result); + Some(make_bin(new_result, instr.inout, instr.rhs)) +} + +fn relink_binop_assign_imm( + instr: &BinAssignInstrImm>, + stack: &mut ValueStack, + new_result: Register, + old_result: Register, + make_bin: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, + make_bin_imm: fn(result: Register, lhs: Register, rhs: Const16) -> Instruction, +) -> Result, TranslationError> +where + T: Copy + From> + Into, + Const16: TryFrom, +{ + if instr.inout != old_result { + // Note: This is a safeguard to prevent bugs. + return Ok(None); + } + debug_assert_ne!(instr.inout, new_result); + let rhs = T::from(instr.rhs); + let new_instr = match >::try_from(rhs) { + Ok(rhs) => make_bin_imm(new_result, instr.inout, rhs), + Err(_) => { + let rhs = stack.alloc_const(rhs)?; + make_bin(new_result, instr.inout, rhs) + } + }; + Ok(Some(new_instr)) +} + +fn relink_binop_assign_fimm( + instr: &BinAssignInstrImm>, + stack: &mut ValueStack, + new_result: Register, + old_result: Register, + make_bin: fn(result: Register, lhs: Register, rhs: Register) -> Instruction, +) -> Result, TranslationError> +where + T: From> + Into, +{ + if instr.inout != old_result { + // Note: This is a safeguard to prevent bugs. + return Ok(None); + } + debug_assert_ne!(instr.inout, new_result); + let rhs = stack.alloc_const(T::from(instr.rhs))?; + let new_instr = make_bin(new_result, instr.inout, rhs); + Ok(Some(new_instr)) +} + +trait ResultMut { + fn result_mut(&mut self) -> &mut Register; +} + +impl ResultMut for Register { + fn result_mut(&mut self) -> &mut Register { + self + } +} + +impl ResultMut for LoadInstr { + fn result_mut(&mut self) -> &mut Register { + &mut self.result + } +} + +impl ResultMut for LoadAtInstr { + fn result_mut(&mut self) -> &mut Register { + &mut self.result + } +} + +impl ResultMut for LoadOffset16Instr { + fn result_mut(&mut self) -> &mut Register { + &mut self.result + } +} + +impl ResultMut for BinInstr { + fn result_mut(&mut self) -> &mut Register { + &mut self.result + } +} + +impl ResultMut for BinInstrImm { + fn result_mut(&mut self) -> &mut Register { + &mut self.result + } +} From ecadef3c0236f753e9440f2c874a799867f08445 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 21:14:24 +0100 Subject: [PATCH 11/17] implement more of the relink_result --- .../regmach/translator/relink_result.rs | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) diff --git a/crates/wasmi/src/engine/regmach/translator/relink_result.rs b/crates/wasmi/src/engine/regmach/translator/relink_result.rs index b80f91f11b..8f8c632a60 100644 --- a/crates/wasmi/src/engine/regmach/translator/relink_result.rs +++ b/crates/wasmi/src/engine/regmach/translator/relink_result.rs @@ -19,6 +19,7 @@ use crate::{ LoadOffset16Instr, Register, RegisterSpan, + UnaryInstr, }, code_map::CompiledFuncEntity, translator::stack::ValueStack, @@ -490,6 +491,219 @@ impl Instruction { I::F64LeAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_le), I::F64GtAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_gt), I::F64GeAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_ge), + + I::I32Clz(instr) | + I::I32Ctz(instr) | + I::I32Popcnt(instr) | + I::I64Clz(instr) | + I::I64Ctz(instr) | + I::I64Popcnt(instr) => relink_simple(instr, new_result, old_result), + + I::I32Add(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_add_assign), + I::I32Sub(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_sub_assign), + I::I32Mul(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_mul_assign), + I::I32DivS(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_div_s_assign), + I::I32DivU(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_div_u_assign), + I::I32RemS(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_rem_s_assign), + I::I32RemU(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_rem_u_assign), + I::I32And(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_and_assign), + I::I32Or(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_or_assign), + I::I32Xor(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_xor_assign), + I::I32Shl(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_shl_assign), + I::I32ShrS(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_shr_s_assign), + I::I32ShrU(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_shr_u_assign), + I::I32Rotl(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_rotl_assign), + I::I32Rotr(instr) => relink_binop!(self, instr, new_result, old_result, I::i32_rotr_assign), + I::I64Add(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_add_assign), + I::I64Sub(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_sub_assign), + I::I64Mul(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_mul_assign), + I::I64DivS(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_div_s_assign), + I::I64DivU(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_div_u_assign), + I::I64RemS(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_rem_s_assign), + I::I64RemU(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_rem_u_assign), + I::I64And(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_and_assign), + I::I64Or(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_or_assign), + I::I64Xor(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_xor_assign), + I::I64Shl(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_shl_assign), + I::I64ShrS(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_shr_s_assign), + I::I64ShrU(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_shr_u_assign), + I::I64Rotl(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_rotl_assign), + I::I64Rotr(instr) => relink_binop!(self, instr, new_result, old_result, I::i64_rotr_assign), + + I::F32Abs(instr) | + I::F32Neg(instr) | + I::F32Ceil(instr) | + I::F32Floor(instr) | + I::F32Trunc(instr) | + I::F32Nearest(instr) | + I::F32Sqrt(instr) | + I::F64Abs(instr) | + I::F64Neg(instr) | + I::F64Ceil(instr) | + I::F64Floor(instr) | + I::F64Trunc(instr) | + I::F64Nearest(instr) | + I::F64Sqrt(instr) => relink_simple(instr, new_result, old_result), + + I::F32Add(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_add_assign), + I::F32Sub(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_sub_assign), + I::F32Mul(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_mul_assign), + I::F32Div(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_div_assign), + I::F32Min(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_min_assign), + I::F32Max(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_max_assign), + I::F32Copysign(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_copysign_assign), + I::F64Add(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_add_assign), + I::F64Sub(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_sub_assign), + I::F64Mul(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_mul_assign), + I::F64Div(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_div_assign), + I::F64Min(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_min_assign), + I::F64Max(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_max_assign), + I::F64Copysign(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_copysign_assign), + + I::I32AddImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_add_assign_imm), + I::I32SubImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_sub_assign_imm), + I::I32SubImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I32MulImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_mul_assign_imm), + I::I32DivSImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_div_s_assign_imm), + I::I32DivSImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I32DivUImm16(instr) => relink_binop_imm16!(u32, self, instr, new_result, old_result, I::i32_div_u_assign_imm), + I::I32DivUImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I32RemSImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_rem_s_assign_imm), + I::I32RemSImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I32RemUImm16(instr) => relink_binop_imm16!(u32, self, instr, new_result, old_result, I::i32_rem_u_assign_imm), + I::I32RemUImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I32AndImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_and_assign_imm), + I::I32OrImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_or_assign_imm), + I::I32XorImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_xor_assign_imm), + I::I32ShlImm(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_shl_assign_imm), + I::I32ShlImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I32ShrSImm(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_shr_s_assign_imm), + I::I32ShrSImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I32ShrUImm(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_shr_u_assign_imm), + I::I32ShrUImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I32RotlImm(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_rotl_assign_imm), + I::I32RotlImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I32RotrImm(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_rotr_assign_imm), + I::I32RotrImm16Rev(instr) => relink_simple(instr, new_result, old_result), + + I::I64AddImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_add_assign_imm32), + I::I64SubImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_sub_assign_imm32), + I::I64SubImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I64MulImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_mul_assign_imm32), + I::I64DivSImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_div_s_assign_imm32), + I::I64DivSImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I64DivUImm16(instr) => relink_binop_imm16!(u64, self, instr, new_result, old_result, I::i64_div_u_assign_imm32), + I::I64DivUImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I64RemSImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_rem_s_assign_imm32), + I::I64RemSImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I64RemUImm16(instr) => relink_binop_imm16!(u64, self, instr, new_result, old_result, I::i64_rem_u_assign_imm32), + I::I64RemUImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I64AndImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_and_assign_imm32), + I::I64OrImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_or_assign_imm32), + I::I64XorImm16(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_xor_assign_imm32), + I::I64ShlImm(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_shl_assign_imm32), + I::I64ShlImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I64ShrSImm(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_shr_s_assign_imm32), + I::I64ShrSImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I64ShrUImm(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_shr_u_assign_imm32), + I::I64ShrUImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I64RotlImm(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_rotl_assign_imm32), + I::I64RotlImm16Rev(instr) => relink_simple(instr, new_result, old_result), + I::I64RotrImm(instr) => relink_binop_imm16!(i64, self, instr, new_result, old_result, I::i64_rotr_assign_imm32), + I::I64RotrImm16Rev(instr) => relink_simple(instr, new_result, old_result), + + I::I32AddAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_add), + I::I32SubAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_sub), + I::I32MulAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_mul), + I::I32DivSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_div_s), + I::I32DivUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_div_u), + I::I32RemSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_rem_s), + I::I32RemUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_rem_u), + I::I32AndAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_and), + I::I32OrAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_or), + I::I32XorAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_xor), + I::I32ShlAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_shl), + I::I32ShrSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_shr_s), + I::I32ShrUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_shr_u), + I::I32RotlAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_rotl), + I::I32RotrAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i32_rotr), + I::I64AddAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_add), + I::I64SubAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_sub), + I::I64MulAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_mul), + I::I64DivSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_div_s), + I::I64DivUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_div_u), + I::I64RemSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_rem_s), + I::I64RemUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_rem_u), + I::I64AndAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_and), + I::I64OrAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_or), + I::I64XorAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_xor), + I::I64ShlAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_shl), + I::I64ShrSAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_shr_s), + I::I64ShrUAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_shr_u), + I::I64RotlAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_rotl), + I::I64RotrAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::i64_rotr), + + I::F32AddAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_add), + I::F32SubAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_sub), + I::F32MulAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_mul), + I::F32DivAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_div), + I::F32MinAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_min), + I::F32MaxAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_max), + I::F32CopysignAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f32_copysign), + I::F64AddAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_add), + I::F64SubAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_sub), + I::F64MulAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_mul), + I::F64DivAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_div), + I::F64MinAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_min), + I::F64MaxAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_max), + I::F64CopysignAssign(instr) => relink_binop_assign!(self, instr, new_result, old_result, I::f64_copysign), + + I::I32AddAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_add, I::i32_add_imm16), + I::I32SubAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_sub, I::i32_sub_imm16), + I::I32MulAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_mul, I::i32_mul_imm16), + I::I32DivSAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_div_s, I::i32_div_s_imm16), + I::I32DivUAssignImm(instr) => relink_binop_assign_imm!(u32, self, instr, stack, new_result, old_result, I::i32_div_u, I::i32_div_u_imm16), + I::I32RemSAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_rem_s, I::i32_rem_s_imm16), + I::I32RemUAssignImm(instr) => relink_binop_assign_imm!(u32, self, instr, stack, new_result, old_result, I::i32_rem_u, I::i32_rem_u_imm16), + I::I32AndAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_and, I::i32_and_imm16), + I::I32OrAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_or, I::i32_or_imm16), + I::I32XorAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_xor, I::i32_xor_imm16), + I::I32ShlAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_shl, I::i32_shl_imm), + I::I32ShrSAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_shr_s, I::i32_shr_s_imm), + I::I32ShrUAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_shr_u, I::i32_shr_u_imm), + I::I32RotlAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_rotl, I::i32_rotl_imm), + I::I32RotrAssignImm(instr) => relink_binop_assign_imm!(i32, self, instr, stack, new_result, old_result, I::i32_rotr, I::i32_rotr_imm), + I::I64AddAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_add, I::i64_add_imm16), + I::I64SubAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_sub, I::i64_sub_imm16), + I::I64MulAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_mul, I::i64_mul_imm16), + I::I64DivSAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_div_s, I::i64_div_s_imm16), + I::I64DivUAssignImm32(instr) => relink_binop_assign_imm!(u64, self, instr, stack, new_result, old_result, I::i64_div_u, I::i64_div_u_imm16), + I::I64RemSAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_rem_s, I::i64_rem_s_imm16), + I::I64RemUAssignImm32(instr) => relink_binop_assign_imm!(u64, self, instr, stack, new_result, old_result, I::i64_rem_u, I::i64_rem_u_imm16), + I::I64AndAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_and, I::i64_and_imm16), + I::I64OrAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_or, I::i64_or_imm16), + I::I64XorAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_xor, I::i64_xor_imm16), + I::I64ShlAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_shl, I::i64_shl_imm), + I::I64ShrSAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_shr_s, I::i64_shr_s_imm), + I::I64ShrUAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_shr_u, I::i64_shr_u_imm), + I::I64RotlAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_rotl, I::i64_rotl_imm), + I::I64RotrAssignImm32(instr) => relink_binop_assign_imm!(i64, self, instr, stack, new_result, old_result, I::i64_rotr, I::i64_rotr_imm), + + I::F32AddAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_add), + I::F32SubAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_sub), + I::F32MulAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_mul), + I::F32DivAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_div), + I::F32MinAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_min), + I::F32MaxAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_max), + // I::F32CopysignAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_copysign), + I::F64AddAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_add), + I::F64SubAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_sub), + I::F64MulAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_mul), + I::F64DivAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_div), + I::F64MinAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_min), + I::F64MaxAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_max), + // I::F64CopysignAssignImm(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_copysign), + _ => todo!(), } } @@ -705,6 +919,12 @@ impl ResultMut for LoadOffset16Instr { } } +impl ResultMut for UnaryInstr { + fn result_mut(&mut self) -> &mut Register { + &mut self.result + } +} + impl ResultMut for BinInstr { fn result_mut(&mut self) -> &mut Register { &mut self.result From 851727fd60e17fd3c5fa5a72c4a5f1c536f92043 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 21:18:50 +0100 Subject: [PATCH 12/17] add unary instructions to relink_result impl --- .../regmach/translator/relink_result.rs | 35 +++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/crates/wasmi/src/engine/regmach/translator/relink_result.rs b/crates/wasmi/src/engine/regmach/translator/relink_result.rs index 8f8c632a60..a0a9aa6dd9 100644 --- a/crates/wasmi/src/engine/regmach/translator/relink_result.rs +++ b/crates/wasmi/src/engine/regmach/translator/relink_result.rs @@ -704,6 +704,41 @@ impl Instruction { I::F64MaxAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_max), // I::F64CopysignAssignImm(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_copysign), + I::I32WrapI64(instr) | + I::I64ExtendI32S(instr) | + I::I64ExtendI32U(instr) | + I::I32TruncF32S(instr) | + I::I32TruncF32U(instr) | + I::I32TruncF64S(instr) | + I::I32TruncF64U(instr) | + I::I64TruncF32S(instr) | + I::I64TruncF32U(instr) | + I::I64TruncF64S(instr) | + I::I64TruncF64U(instr) | + I::I32TruncSatF32S(instr) | + I::I32TruncSatF32U(instr) | + I::I32TruncSatF64S(instr) | + I::I32TruncSatF64U(instr) | + I::I64TruncSatF32S(instr) | + I::I64TruncSatF32U(instr) | + I::I64TruncSatF64S(instr) | + I::I64TruncSatF64U(instr) | + I::I32Extend8S(instr) | + I::I32Extend16S(instr) | + I::I64Extend8S(instr) | + I::I64Extend16S(instr) | + I::I64Extend32S(instr) | + I::F32DemoteF64(instr) | + I::F64PromoteF32(instr) | + I::F32ConvertI32S(instr) | + I::F32ConvertI32U(instr) | + I::F32ConvertI64S(instr) | + I::F32ConvertI64U(instr) | + I::F64ConvertI32S(instr) | + I::F64ConvertI32U(instr) | + I::F64ConvertI64S(instr) | + I::F64ConvertI64U(instr) => relink_simple(instr, new_result, old_result), + _ => todo!(), } } From af059b1e19b90d2c29a3a57788610964619f4cad Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 21:29:25 +0100 Subject: [PATCH 13/17] finalize relink_result impl --- .../regmach/translator/relink_result.rs | 59 +++++++++++++++++-- 1 file changed, 55 insertions(+), 4 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/translator/relink_result.rs b/crates/wasmi/src/engine/regmach/translator/relink_result.rs index a0a9aa6dd9..c569ae0164 100644 --- a/crates/wasmi/src/engine/regmach/translator/relink_result.rs +++ b/crates/wasmi/src/engine/regmach/translator/relink_result.rs @@ -19,6 +19,7 @@ use crate::{ LoadOffset16Instr, Register, RegisterSpan, + Sign, UnaryInstr, }, code_map::CompiledFuncEntity, @@ -50,6 +51,16 @@ macro_rules! relink_binop_imm16 { }; } +macro_rules! relink_copysign_imm { + ($this:expr, $instr:ident, $new_result:ident, $old_result:ident, $make_instr:expr) => { + match relink_copysign_imm($instr, $new_result, $old_result, $make_instr)? { + RelinkResult::Unchanged => Ok(false), + RelinkResult::Relinked => Ok(true), + RelinkResult::Exchanged(new_instr) => relink_exchange($this, new_instr), + } + }; +} + macro_rules! relink_binop_assign { ($this:expr, $instr:ident, $new_result:ident, $old_result:ident, $make_instr:expr) => { match relink_binop_assign($instr, $new_result, $old_result, $make_instr) { @@ -90,6 +101,15 @@ macro_rules! relink_binop_assign_fimm { }; } +macro_rules! relink_copysign_assign { + ($this:expr, $instr:ident, $new_result:ident, $old_result:ident, $make_instr:expr) => { + match relink_copysign_assign($instr, $new_result, $old_result, $make_instr)? { + None => Ok(false), + Some(new_instr) => relink_exchange($this, new_instr), + } + }; +} + impl Instruction { #[rustfmt::skip] pub fn relink_result( @@ -552,6 +572,7 @@ impl Instruction { I::F32Min(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_min_assign), I::F32Max(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_max_assign), I::F32Copysign(instr) => relink_binop!(self, instr, new_result, old_result, I::f32_copysign_assign), + I::F32CopysignImm(instr) => relink_copysign_imm!(self, instr, new_result, old_result, I::f32_copysign_assign_imm), I::F64Add(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_add_assign), I::F64Sub(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_sub_assign), I::F64Mul(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_mul_assign), @@ -559,6 +580,7 @@ impl Instruction { I::F64Min(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_min_assign), I::F64Max(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_max_assign), I::F64Copysign(instr) => relink_binop!(self, instr, new_result, old_result, I::f64_copysign_assign), + I::F64CopysignImm(instr) => relink_copysign_imm!(self, instr, new_result, old_result, I::f64_copysign_assign_imm), I::I32AddImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_add_assign_imm), I::I32SubImm16(instr) => relink_binop_imm16!(i32, self, instr, new_result, old_result, I::i32_sub_assign_imm), @@ -695,14 +717,14 @@ impl Instruction { I::F32DivAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_div), I::F32MinAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_min), I::F32MaxAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_max), - // I::F32CopysignAssignImm(instr) => relink_binop_assign_fimm!(f32, self, instr, stack, new_result, old_result, I::f32_copysign), + I::F32CopysignAssignImm(instr) => relink_copysign_assign!(self, instr, new_result, old_result, I::f32_copysign_imm), I::F64AddAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_add), I::F64SubAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_sub), I::F64MulAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_mul), I::F64DivAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_div), I::F64MinAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_min), I::F64MaxAssignImm32(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_max), - // I::F64CopysignAssignImm(instr) => relink_binop_assign_fimm!(f64, self, instr, stack, new_result, old_result, I::f64_copysign), + I::F64CopysignAssignImm(instr) => relink_copysign_assign!(self, instr, new_result, old_result, I::f64_copysign_imm), I::I32WrapI64(instr) | I::I64ExtendI32S(instr) | @@ -738,8 +760,6 @@ impl Instruction { I::F64ConvertI32U(instr) | I::F64ConvertI64S(instr) | I::F64ConvertI64U(instr) => relink_simple(instr, new_result, old_result), - - _ => todo!(), } } } @@ -864,6 +884,22 @@ where Ok(RelinkResult::Relinked) } +fn relink_copysign_imm( + instr: &mut BinInstrImm, + new_result: Register, + old_result: Register, + make_bin: fn(result: Register, rhs: Sign) -> Instruction, +) -> Result { + if !relink_simple(instr, new_result, old_result)? { + return Ok(RelinkResult::Unchanged); + } + if instr.result == instr.reg_in { + let new_instr = make_bin(new_result, instr.imm_in); + return Ok(RelinkResult::Exchanged(new_instr)); + } + Ok(RelinkResult::Relinked) +} + fn relink_binop_assign( instr: &BinAssignInstr, new_result: Register, @@ -926,6 +962,21 @@ where Ok(Some(new_instr)) } +fn relink_copysign_assign( + instr: &BinAssignInstrImm, + new_result: Register, + old_result: Register, + make_binop: fn(result: Register, lhs: Register, rhs: Sign) -> Instruction, +) -> Result, TranslationError> { + if instr.inout != old_result { + // Note: This is a safeguard to prevent bugs. + return Ok(None); + } + debug_assert_ne!(instr.inout, new_result); + let new_instr = make_binop(new_result, instr.inout, instr.rhs); + Ok(Some(new_instr)) +} + trait ResultMut { fn result_mut(&mut self) -> &mut Register; } From 17ef7680a27c15cdb16e57b68fdd86e21cb85bd2 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 21:42:34 +0100 Subject: [PATCH 14/17] use the new relink_result in the local.set translation --- .../regmach/translator/instr_encoder.rs | 20 ++++++------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs b/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs index 9acd5ef197..9b2f2b2c74 100644 --- a/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs +++ b/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs @@ -710,22 +710,14 @@ impl InstrEncoder { // Thankfully most instructions are small enough. return fallback_case(self, stack, local, value, preserved); } - let Some(result) = self.instrs.get_mut(last_instr).result_mut(res) else { - // Can only apply the optimization if the last instruction has exactly one result. - return fallback_case(self, stack, local, value, preserved); - }; - if *result != returned_value { - // We only want to apply the optimization if indeed `x` in `local.set n x` - // is the same register as the result register of the last instruction. - // - // TODO: Find out in what cases `result != value`. Is this a bug or an edge case? - // Generally `result` should be equal to `value` since `value` refers to the - // `result` of the previous instruction. - // Therefore, instead of an `if` we originally had a `debug_assert`. - // (Note: the spidermonkey bench test failed without this change.) + if !self + .instrs + .get_mut(last_instr) + .relink_result(stack, res, local, returned_value)? + { + // It was not possible to relink the result of `last_instr` therefore we fallback. return fallback_case(self, stack, local, value, preserved); } - *result = local; if let Some(preserved) = preserved { // We were able to apply the optimization. // Preservation requires the copy to be before the optimized last instruction. From e7ec26737a8595079933e26e2ef2f3bed18c6f94 Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 21:45:57 +0100 Subject: [PATCH 15/17] remove old result_mut implementation --- .../src/engine/regmach/translator/mod.rs | 1 - .../engine/regmach/translator/result_mut.rs | 770 ------------------ 2 files changed, 771 deletions(-) delete mode 100644 crates/wasmi/src/engine/regmach/translator/result_mut.rs diff --git a/crates/wasmi/src/engine/regmach/translator/mod.rs b/crates/wasmi/src/engine/regmach/translator/mod.rs index a9827f5986..325818761a 100644 --- a/crates/wasmi/src/engine/regmach/translator/mod.rs +++ b/crates/wasmi/src/engine/regmach/translator/mod.rs @@ -4,7 +4,6 @@ mod control_frame; mod control_stack; mod instr_encoder; mod relink_result; -mod result_mut; mod stack; mod typed_value; mod utils; diff --git a/crates/wasmi/src/engine/regmach/translator/result_mut.rs b/crates/wasmi/src/engine/regmach/translator/result_mut.rs deleted file mode 100644 index 0b45c26f7a..0000000000 --- a/crates/wasmi/src/engine/regmach/translator/result_mut.rs +++ /dev/null @@ -1,770 +0,0 @@ -//! Implements methods on [`Instruction`] to query the single result [`Register`]. -//! -//! This is used for an optimization with `local.set` and `local.tee` translation -//! to replace the result of the previous [`Instruction`] instead of emitting a -//! copy instruction to model the `local.set` or `local.tee` translation. - -use crate::{ - engine::{ - bytecode::{FuncIdx, SignatureIdx}, - regmach::{ - bytecode::{ - BinAssignInstr, - BinAssignInstrImm, - BinInstr, - BinInstrImm, - Instruction, - LoadAtInstr, - LoadInstr, - LoadOffset16Instr, - Register, - RegisterSpan, - UnaryInstr, - }, - code_map::CompiledFuncEntity, - }, - CompiledFunc, - }, - module::ModuleResources, -}; - -impl Instruction { - /// Returns the single `result` [`Register`] of the [`Instruction`] if any. - /// - /// # Note - /// - /// - A call [`Instruction`] variants will return `Some` if they return a single value. - /// - A non-call [`Instruction`] with multiple `result` [`Register`] return `None`. - #[rustfmt::skip] - pub fn result_mut(&mut self, res: &ModuleResources) -> Option<&mut Register> { - match self { - Instruction::TableIdx(_) | - Instruction::DataSegmentIdx(_) | - Instruction::ElementSegmentIdx(_) | - Instruction::Const32(_) | - Instruction::I64Const32(_) | - Instruction::F64Const32(_) | - Instruction::Register(_) | - Instruction::Register2(_) | - Instruction::Register3(_) | - Instruction::RegisterList(_) | - Instruction::Trap(_) | - Instruction::ConsumeFuel(_) | - Instruction::Return | - Instruction::ReturnReg { .. } | - Instruction::ReturnReg2 { .. } | - Instruction::ReturnReg3 { .. } | - Instruction::ReturnImm32 { .. } | - Instruction::ReturnI64Imm32 { .. } | - Instruction::ReturnF64Imm32 { .. } | - Instruction::ReturnSpan { .. } | - Instruction::ReturnMany { .. } | - Instruction::ReturnNez { .. } | - Instruction::ReturnNezReg { .. } | - Instruction::ReturnNezReg2 { .. } | - Instruction::ReturnNezImm32 { .. } | - Instruction::ReturnNezI64Imm32 { .. } | - Instruction::ReturnNezF64Imm32 { .. } | - Instruction::ReturnNezSpan { .. } | - Instruction::ReturnNezMany { .. } | - Instruction::Branch { .. } | - Instruction::BranchEqz { .. } | - Instruction::BranchNez { .. } | - Instruction::BranchTable { .. } | - Instruction::BranchI32Eq(_) | - Instruction::BranchI32EqImm(_) | - Instruction::BranchI32Ne(_) | - Instruction::BranchI32NeImm(_) | - Instruction::BranchI32LtS(_) | - Instruction::BranchI32LtSImm(_) | - Instruction::BranchI32LtU(_) | - Instruction::BranchI32LtUImm(_) | - Instruction::BranchI32LeS(_) | - Instruction::BranchI32LeSImm(_) | - Instruction::BranchI32LeU(_) | - Instruction::BranchI32LeUImm(_) | - Instruction::BranchI32GtS(_) | - Instruction::BranchI32GtSImm(_) | - Instruction::BranchI32GtU(_) | - Instruction::BranchI32GtUImm(_) | - Instruction::BranchI32GeS(_) | - Instruction::BranchI32GeSImm(_) | - Instruction::BranchI32GeU(_) | - Instruction::BranchI32GeUImm(_) | - Instruction::BranchI64Eq(_) | - Instruction::BranchI64EqImm(_) | - Instruction::BranchI64Ne(_) | - Instruction::BranchI64NeImm(_) | - Instruction::BranchI64LtS(_) | - Instruction::BranchI64LtSImm(_) | - Instruction::BranchI64LtU(_) | - Instruction::BranchI64LtUImm(_) | - Instruction::BranchI64LeS(_) | - Instruction::BranchI64LeSImm(_) | - Instruction::BranchI64LeU(_) | - Instruction::BranchI64LeUImm(_) | - Instruction::BranchI64GtS(_) | - Instruction::BranchI64GtSImm(_) | - Instruction::BranchI64GtU(_) | - Instruction::BranchI64GtUImm(_) | - Instruction::BranchI64GeS(_) | - Instruction::BranchI64GeSImm(_) | - Instruction::BranchI64GeU(_) | - Instruction::BranchI64GeUImm(_) | - Instruction::BranchF32Eq(_) | - Instruction::BranchF32Ne(_) | - Instruction::BranchF32Lt(_) | - Instruction::BranchF32Le(_) | - Instruction::BranchF32Gt(_) | - Instruction::BranchF32Ge(_) | - Instruction::BranchF64Eq(_) | - Instruction::BranchF64Ne(_) | - Instruction::BranchF64Lt(_) | - Instruction::BranchF64Le(_) | - Instruction::BranchF64Gt(_) | - Instruction::BranchF64Ge(_) => None, - Instruction::Copy { result, .. } | - Instruction::CopyImm32 { result, .. } | - Instruction::CopyI64Imm32 { result, .. } | - Instruction::CopyF64Imm32 { result, .. } => Some(result), - Instruction::CopySpan { .. } | - Instruction::CopySpanNonOverlapping { .. } | - Instruction::Copy2 { .. } | - Instruction::CopyMany { .. } => None, - Instruction::CopyManyNonOverlapping { .. } => None, - Instruction::CallIndirectParams(_) | - Instruction::CallIndirectParamsImm16(_) | - Instruction::ReturnCallInternal0 { .. } | - Instruction::ReturnCallInternal { .. } | - Instruction::ReturnCallImported0 { .. } | - Instruction::ReturnCallImported { .. } | - Instruction::ReturnCallIndirect0 { .. } | - Instruction::ReturnCallIndirect { .. } => None, - Instruction::CallInternal0 { results, func } | - Instruction::CallInternal { results, func } => call_internal_result_mut(results, *func, res), - Instruction::CallImported0 { results, func } | - Instruction::CallImported { results, func } => call_imported_result_mut(results, *func, res), - Instruction::CallIndirect0 { results, func_type } | - Instruction::CallIndirect { results, func_type } => call_indirect_result_mut(results, *func_type, res), - Instruction::Select { result, .. } | - Instruction::SelectRev { result, .. } => Some(result), - Instruction::SelectImm32 { result_or_condition, .. } | - Instruction::SelectI64Imm32 { result_or_condition, .. } | - Instruction::SelectF64Imm32 { result_or_condition, .. } => { - // Note: the `result_or_condition` necessarily points to the actual `result` - // register since we make sure elsewhere that only the correct instruction - // word is given to this method. - Some(result_or_condition) - }, - Instruction::RefFunc { result, .. } | - Instruction::TableGet { result, .. } | - Instruction::TableGetImm { result, .. } | - Instruction::TableSize { result, .. } => Some(result), - Instruction::TableSet { .. } | - Instruction::TableSetAt { .. } | - Instruction::TableCopy { .. } | - Instruction::TableCopyTo { .. } | - Instruction::TableCopyFrom { .. } | - Instruction::TableCopyFromTo { .. } | - Instruction::TableCopyExact { .. } | - Instruction::TableCopyToExact { .. } | - Instruction::TableCopyFromExact { .. } | - Instruction::TableCopyFromToExact { .. } => None, - Instruction::TableInit { .. } | - Instruction::TableInitTo { .. } | - Instruction::TableInitFrom { .. } | - Instruction::TableInitFromTo { .. } | - Instruction::TableInitExact { .. } | - Instruction::TableInitToExact { .. } | - Instruction::TableInitFromExact { .. } | - Instruction::TableInitFromToExact { .. } => None, - Instruction::TableFill { .. } | - Instruction::TableFillAt { .. } | - Instruction::TableFillExact { .. } | - Instruction::TableFillAtExact { .. } => None, - Instruction::TableGrow { result, .. } | - Instruction::TableGrowImm { result, .. } => Some(result), - Instruction::ElemDrop(_) => None, - Instruction::DataDrop(_) => None, - Instruction::MemorySize { result } | - Instruction::MemoryGrow { result, .. } | - Instruction::MemoryGrowBy { result, .. } => Some(result), - Instruction::MemoryCopy { .. } | - Instruction::MemoryCopyTo { .. } | - Instruction::MemoryCopyFrom { .. } | - Instruction::MemoryCopyFromTo { .. } | - Instruction::MemoryCopyExact { .. } | - Instruction::MemoryCopyToExact { .. } | - Instruction::MemoryCopyFromExact { .. } | - Instruction::MemoryCopyFromToExact { .. } => None, - Instruction::MemoryFill { .. } | - Instruction::MemoryFillAt { .. } | - Instruction::MemoryFillImm { .. } | - Instruction::MemoryFillExact { .. } | - Instruction::MemoryFillAtImm { .. } | - Instruction::MemoryFillAtExact { .. } | - Instruction::MemoryFillImmExact { .. } | - Instruction::MemoryFillAtImmExact { .. } => None, - Instruction::MemoryInit { .. } | - Instruction::MemoryInitTo { .. } | - Instruction::MemoryInitFrom { .. } | - Instruction::MemoryInitFromTo { .. } | - Instruction::MemoryInitExact { .. } | - Instruction::MemoryInitToExact { .. } | - Instruction::MemoryInitFromExact { .. } | - Instruction::MemoryInitFromToExact { .. } => None, - Instruction::GlobalGet { result, .. } => Some(result), - Instruction::GlobalSet { .. } | - Instruction::GlobalSetI32Imm16 { .. } | - Instruction::GlobalSetI64Imm16 { .. } => None, - Instruction::I32Load(instr) => instr.result_mut(), - Instruction::I32LoadAt(instr) => instr.result_mut(), - Instruction::I32LoadOffset16(instr) => instr.result_mut(), - Instruction::I64Load(instr) => instr.result_mut(), - Instruction::I64LoadAt(instr) => instr.result_mut(), - Instruction::I64LoadOffset16(instr) => instr.result_mut(), - Instruction::F32Load(instr) => instr.result_mut(), - Instruction::F32LoadAt(instr) => instr.result_mut(), - Instruction::F32LoadOffset16(instr) => instr.result_mut(), - Instruction::F64Load(instr) => instr.result_mut(), - Instruction::F64LoadAt(instr) => instr.result_mut(), - Instruction::F64LoadOffset16(instr) => instr.result_mut(), - Instruction::I32Load8s(instr) => instr.result_mut(), - Instruction::I32Load8sAt(instr) => instr.result_mut(), - Instruction::I32Load8sOffset16(instr) => instr.result_mut(), - Instruction::I32Load8u(instr) => instr.result_mut(), - Instruction::I32Load8uAt(instr) => instr.result_mut(), - Instruction::I32Load8uOffset16(instr) => instr.result_mut(), - Instruction::I32Load16s(instr) => instr.result_mut(), - Instruction::I32Load16sAt(instr) => instr.result_mut(), - Instruction::I32Load16sOffset16(instr) => instr.result_mut(), - Instruction::I32Load16u(instr) => instr.result_mut(), - Instruction::I32Load16uAt(instr) => instr.result_mut(), - Instruction::I32Load16uOffset16(instr) => instr.result_mut(), - Instruction::I64Load8s(instr) => instr.result_mut(), - Instruction::I64Load8sAt(instr) => instr.result_mut(), - Instruction::I64Load8sOffset16(instr) => instr.result_mut(), - Instruction::I64Load8u(instr) => instr.result_mut(), - Instruction::I64Load8uAt(instr) => instr.result_mut(), - Instruction::I64Load8uOffset16(instr) => instr.result_mut(), - Instruction::I64Load16s(instr) => instr.result_mut(), - Instruction::I64Load16sAt(instr) => instr.result_mut(), - Instruction::I64Load16sOffset16(instr) => instr.result_mut(), - Instruction::I64Load16u(instr) => instr.result_mut(), - Instruction::I64Load16uAt(instr) => instr.result_mut(), - Instruction::I64Load16uOffset16(instr) => instr.result_mut(), - Instruction::I64Load32s(instr) => instr.result_mut(), - Instruction::I64Load32sAt(instr) => instr.result_mut(), - Instruction::I64Load32sOffset16(instr) => instr.result_mut(), - Instruction::I64Load32u(instr) => instr.result_mut(), - Instruction::I64Load32uAt(instr) => instr.result_mut(), - Instruction::I64Load32uOffset16(instr) => instr.result_mut(), - Instruction::I32Store(_) | - Instruction::I32StoreOffset16(_) | - Instruction::I32StoreOffset16Imm16(_) | - Instruction::I32StoreAt(_) | - Instruction::I32StoreAtImm16(_) | - Instruction::I32Store8(_) | - Instruction::I32Store8Offset16(_) | - Instruction::I32Store8Offset16Imm(_) | - Instruction::I32Store8At(_) | - Instruction::I32Store8AtImm(_) | - Instruction::I32Store16(_) | - Instruction::I32Store16Offset16(_) | - Instruction::I32Store16Offset16Imm(_) | - Instruction::I32Store16At(_) | - Instruction::I32Store16AtImm(_) | - Instruction::I64Store(_) | - Instruction::I64StoreOffset16(_) | - Instruction::I64StoreOffset16Imm16(_) | - Instruction::I64StoreAt(_) | - Instruction::I64StoreAtImm16(_) | - Instruction::I64Store8(_) | - Instruction::I64Store8Offset16(_) | - Instruction::I64Store8Offset16Imm(_) | - Instruction::I64Store8At(_) | - Instruction::I64Store8AtImm(_) | - Instruction::I64Store16(_) | - Instruction::I64Store16Offset16(_) | - Instruction::I64Store16Offset16Imm(_) | - Instruction::I64Store16At(_) | - Instruction::I64Store16AtImm(_) | - Instruction::I64Store32(_) | - Instruction::I64Store32Offset16(_) | - Instruction::I64Store32Offset16Imm16(_) | - Instruction::I64Store32At(_) | - Instruction::I64Store32AtImm16(_) | - Instruction::F32Store(_) | - Instruction::F32StoreOffset16(_) | - Instruction::F32StoreAt(_) | - Instruction::F64Store(_) | - Instruction::F64StoreOffset16(_) | - Instruction::F64StoreAt(_) => None, - Instruction::I32Eq(instr) => instr.result_mut(), - Instruction::I32EqImm16(instr) => instr.result_mut(), - Instruction::I64Eq(instr) => instr.result_mut(), - Instruction::I64EqImm16(instr) => instr.result_mut(), - Instruction::I32Ne(instr) => instr.result_mut(), - Instruction::I32NeImm16(instr) => instr.result_mut(), - Instruction::I64Ne(instr) => instr.result_mut(), - Instruction::I64NeImm16(instr) => instr.result_mut(), - Instruction::I32LtS(instr) | - Instruction::I32LtU(instr) => instr.result_mut(), - Instruction::I32LtSImm16(instr) => instr.result_mut(), - Instruction::I32LtUImm16(instr) => instr.result_mut(), - Instruction::I64LtS(instr) | - Instruction::I64LtU(instr) => instr.result_mut(), - Instruction::I64LtSImm16(instr) => instr.result_mut(), - Instruction::I64LtUImm16(instr) => instr.result_mut(), - Instruction::I32GtS(instr) | - Instruction::I32GtU(instr) => instr.result_mut(), - Instruction::I32GtSImm16(instr) => instr.result_mut(), - Instruction::I32GtUImm16(instr) => instr.result_mut(), - Instruction::I64GtS(instr) | - Instruction::I64GtU(instr) => instr.result_mut(), - Instruction::I64GtSImm16(instr) => instr.result_mut(), - Instruction::I64GtUImm16(instr) => instr.result_mut(), - Instruction::I32LeS(instr) | - Instruction::I32LeU(instr) => instr.result_mut(), - Instruction::I32LeSImm16(instr) => instr.result_mut(), - Instruction::I32LeUImm16(instr) => instr.result_mut(), - Instruction::I64LeS(instr) | - Instruction::I64LeU(instr) => instr.result_mut(), - Instruction::I64LeSImm16(instr) => instr.result_mut(), - Instruction::I64LeUImm16(instr) => instr.result_mut(), - Instruction::I32GeS(instr) | - Instruction::I32GeU(instr) => instr.result_mut(), - Instruction::I32GeSImm16(instr) => instr.result_mut(), - Instruction::I32GeUImm16(instr) => instr.result_mut(), - Instruction::I64GeS(instr) | - Instruction::I64GeU(instr) => instr.result_mut(), - Instruction::I64GeSImm16(instr) => instr.result_mut(), - Instruction::I64GeUImm16(instr) => instr.result_mut(), - Instruction::F32Eq(instr) | - Instruction::F64Eq(instr) | - Instruction::F32Ne(instr) | - Instruction::F64Ne(instr) | - Instruction::F32Lt(instr) | - Instruction::F64Lt(instr) | - Instruction::F32Le(instr) | - Instruction::F64Le(instr) | - Instruction::F32Gt(instr) | - Instruction::F64Gt(instr) | - Instruction::F32Ge(instr) | - Instruction::F64Ge(instr) => instr.result_mut(), - Instruction::I32Clz(instr) | - Instruction::I64Clz(instr) | - Instruction::I32Ctz(instr) | - Instruction::I64Ctz(instr) | - Instruction::I32Popcnt(instr) | - Instruction::I64Popcnt(instr) => instr.result_mut(), - Instruction::I32Add(instr) | - Instruction::I64Add(instr) => instr.result_mut(), - Instruction::I32AddImm16(instr) => instr.result_mut(), - Instruction::I64AddImm16(instr) => instr.result_mut(), - Instruction::I32Sub(instr) | - Instruction::I64Sub(instr) => instr.result_mut(), - Instruction::I32SubImm16(instr) => instr.result_mut(), - Instruction::I64SubImm16(instr) => instr.result_mut(), - Instruction::I32SubImm16Rev(instr) => instr.result_mut(), - Instruction::I64SubImm16Rev(instr) => instr.result_mut(), - Instruction::I32Mul(instr) | - Instruction::I64Mul(instr) => instr.result_mut(), - Instruction::I32MulImm16(instr) => instr.result_mut(), - Instruction::I64MulImm16(instr) => instr.result_mut(), - Instruction::I32DivS(instr) | - Instruction::I64DivS(instr) => instr.result_mut(), - Instruction::I32DivSImm16(instr) => instr.result_mut(), - Instruction::I64DivSImm16(instr) => instr.result_mut(), - Instruction::I32DivSImm16Rev(instr) => instr.result_mut(), - Instruction::I64DivSImm16Rev(instr) => instr.result_mut(), - Instruction::I32DivU(instr) | - Instruction::I64DivU(instr) => instr.result_mut(), - Instruction::I32DivUImm16(instr) => instr.result_mut(), - Instruction::I64DivUImm16(instr) => instr.result_mut(), - Instruction::I32DivUImm16Rev(instr) => instr.result_mut(), - Instruction::I64DivUImm16Rev(instr) => instr.result_mut(), - Instruction::I32RemS(instr) | - Instruction::I64RemS(instr) => instr.result_mut(), - Instruction::I32RemSImm16(instr) => instr.result_mut(), - Instruction::I64RemSImm16(instr) => instr.result_mut(), - Instruction::I32RemSImm16Rev(instr) => instr.result_mut(), - Instruction::I64RemSImm16Rev(instr) => instr.result_mut(), - Instruction::I32RemU(instr) | - Instruction::I64RemU(instr) => instr.result_mut(), - Instruction::I32RemUImm16(instr) => instr.result_mut(), - Instruction::I64RemUImm16(instr) => instr.result_mut(), - Instruction::I32RemUImm16Rev(instr) => instr.result_mut(), - Instruction::I64RemUImm16Rev(instr) => instr.result_mut(), - Instruction::I32And(instr) | - Instruction::I64And(instr) => instr.result_mut(), - Instruction::I32AndImm16(instr) => instr.result_mut(), - Instruction::I64AndImm16(instr) => instr.result_mut(), - Instruction::I32Or(instr) | - Instruction::I64Or(instr) => instr.result_mut(), - Instruction::I32OrImm16(instr) => instr.result_mut(), - Instruction::I64OrImm16(instr) => instr.result_mut(), - Instruction::I32Xor(instr) | - Instruction::I64Xor(instr) => instr.result_mut(), - Instruction::I32XorImm16(instr) => instr.result_mut(), - Instruction::I64XorImm16(instr) => instr.result_mut(), - Instruction::I32Shl(instr) | - Instruction::I64Shl(instr) => instr.result_mut(), - Instruction::I32ShlImm(instr) => instr.result_mut(), - Instruction::I64ShlImm(instr) => instr.result_mut(), - Instruction::I32ShlImm16Rev(instr) => instr.result_mut(), - Instruction::I64ShlImm16Rev(instr) => instr.result_mut(), - Instruction::I32ShrU(instr) | - Instruction::I64ShrU(instr) => instr.result_mut(), - Instruction::I32ShrUImm(instr) => instr.result_mut(), - Instruction::I64ShrUImm(instr) => instr.result_mut(), - Instruction::I32ShrUImm16Rev(instr) => instr.result_mut(), - Instruction::I64ShrUImm16Rev(instr) => instr.result_mut(), - Instruction::I32ShrS(instr) | - Instruction::I64ShrS(instr) => instr.result_mut(), - Instruction::I32ShrSImm(instr) => instr.result_mut(), - Instruction::I64ShrSImm(instr) => instr.result_mut(), - Instruction::I32ShrSImm16Rev(instr) => instr.result_mut(), - Instruction::I64ShrSImm16Rev(instr) => instr.result_mut(), - Instruction::I32Rotl(instr) | - Instruction::I64Rotl(instr) => instr.result_mut(), - Instruction::I32RotlImm(instr) => instr.result_mut(), - Instruction::I64RotlImm(instr) => instr.result_mut(), - Instruction::I32RotlImm16Rev(instr) => instr.result_mut(), - Instruction::I64RotlImm16Rev(instr) => instr.result_mut(), - Instruction::I32Rotr(instr) | - Instruction::I64Rotr(instr) => instr.result_mut(), - Instruction::I32RotrImm(instr) => instr.result_mut(), - Instruction::I64RotrImm(instr) => instr.result_mut(), - Instruction::I32RotrImm16Rev(instr) => instr.result_mut(), - Instruction::I64RotrImm16Rev(instr) => instr.result_mut(), - Instruction::F32Abs(instr) | - Instruction::F64Abs(instr) | - Instruction::F32Neg(instr) | - Instruction::F64Neg(instr) | - Instruction::F32Ceil(instr) | - Instruction::F64Ceil(instr) | - Instruction::F32Floor(instr) | - Instruction::F64Floor(instr) | - Instruction::F32Trunc(instr) | - Instruction::F64Trunc(instr) | - Instruction::F32Nearest(instr) | - Instruction::F64Nearest(instr) | - Instruction::F32Sqrt(instr) | - Instruction::F64Sqrt(instr) => instr.result_mut(), - Instruction::F32Add(instr) | - Instruction::F64Add(instr) | - Instruction::F32Sub(instr) | - Instruction::F64Sub(instr) | - Instruction::F32Mul(instr) | - Instruction::F64Mul(instr) | - Instruction::F32Div(instr) | - Instruction::F64Div(instr) | - Instruction::F32Min(instr) | - Instruction::F64Min(instr) | - Instruction::F32Max(instr) | - Instruction::F64Max(instr) | - Instruction::F32Copysign(instr) | - Instruction::F64Copysign(instr) => instr.result_mut(), - Instruction::F32CopysignImm(instr) | - Instruction::F64CopysignImm(instr) => instr.result_mut(), - Instruction::I32WrapI64(instr) | - Instruction::I64ExtendI32S(instr) | - Instruction::I64ExtendI32U(instr) | - Instruction::I32TruncF32S(instr) | - Instruction::I32TruncF32U(instr) | - Instruction::I32TruncF64S(instr) | - Instruction::I32TruncF64U(instr) | - Instruction::I64TruncF32S(instr) | - Instruction::I64TruncF32U(instr) | - Instruction::I64TruncF64S(instr) | - Instruction::I64TruncF64U(instr) | - Instruction::I32TruncSatF32S(instr) | - Instruction::I32TruncSatF32U(instr) | - Instruction::I32TruncSatF64S(instr) | - Instruction::I32TruncSatF64U(instr) | - Instruction::I64TruncSatF32S(instr) | - Instruction::I64TruncSatF32U(instr) | - Instruction::I64TruncSatF64S(instr) | - Instruction::I64TruncSatF64U(instr) | - Instruction::I32Extend8S(instr) | - Instruction::I32Extend16S(instr) | - Instruction::I64Extend8S(instr) | - Instruction::I64Extend16S(instr) | - Instruction::I64Extend32S(instr) | - Instruction::F32DemoteF64(instr) | - Instruction::F64PromoteF32(instr) | - Instruction::F32ConvertI32S(instr) | - Instruction::F32ConvertI32U(instr) | - Instruction::F32ConvertI64S(instr) | - Instruction::F32ConvertI64U(instr) | - Instruction::F64ConvertI32S(instr) | - Instruction::F64ConvertI32U(instr) | - Instruction::F64ConvertI64S(instr) | - Instruction::F64ConvertI64U(instr) => instr.result_mut(), - Instruction::I32EqAssign(instr) | - Instruction::I32NeAssign(instr) | - Instruction::I32LtSAssign(instr) | - Instruction::I32LtUAssign(instr) | - Instruction::I32GtSAssign(instr) | - Instruction::I32GtUAssign(instr) | - Instruction::I32LeSAssign(instr) | - Instruction::I32LeUAssign(instr) | - Instruction::I32GeSAssign(instr) | - Instruction::I32GeUAssign(instr) | - Instruction::I64EqAssign(instr) | - Instruction::I64NeAssign(instr) | - Instruction::I64LtSAssign(instr) | - Instruction::I64LtUAssign(instr) | - Instruction::I64GtSAssign(instr) | - Instruction::I64GtUAssign(instr) | - Instruction::I64LeSAssign(instr) | - Instruction::I64LeUAssign(instr) | - Instruction::I64GeSAssign(instr) | - Instruction::I64GeUAssign(instr) | - Instruction::F32EqAssign(instr) | - Instruction::F32NeAssign(instr) | - Instruction::F32LtAssign(instr) | - Instruction::F32LeAssign(instr) | - Instruction::F32GtAssign(instr) | - Instruction::F32GeAssign(instr) | - Instruction::F64EqAssign(instr) | - Instruction::F64NeAssign(instr) | - Instruction::F64LtAssign(instr) | - Instruction::F64LeAssign(instr) | - Instruction::F64GtAssign(instr) | - Instruction::F64GeAssign(instr) | - Instruction::I32AddAssign(instr) | - Instruction::I32SubAssign(instr) | - Instruction::I32MulAssign(instr) | - Instruction::I32DivSAssign(instr) | - Instruction::I32DivUAssign(instr) | - Instruction::I32RemSAssign(instr) | - Instruction::I32RemUAssign(instr) | - Instruction::I32AndAssign(instr) | - Instruction::I32OrAssign(instr) | - Instruction::I32XorAssign(instr) | - Instruction::I32ShlAssign(instr) | - Instruction::I32ShrUAssign(instr) | - Instruction::I32ShrSAssign(instr) | - Instruction::I32RotlAssign(instr) | - Instruction::I32RotrAssign(instr) | - Instruction::I64AddAssign(instr) | - Instruction::I64SubAssign(instr) | - Instruction::I64MulAssign(instr) | - Instruction::I64DivSAssign(instr) | - Instruction::I64DivUAssign(instr) | - Instruction::I64RemSAssign(instr) | - Instruction::I64RemUAssign(instr) | - Instruction::I64AndAssign(instr) | - Instruction::I64OrAssign(instr) | - Instruction::I64XorAssign(instr) | - Instruction::I64ShlAssign(instr) | - Instruction::I64ShrUAssign(instr) | - Instruction::I64ShrSAssign(instr) | - Instruction::I64RotlAssign(instr) | - Instruction::I64RotrAssign(instr) | - Instruction::F32AddAssign(instr) | - Instruction::F32SubAssign(instr) | - Instruction::F32MulAssign(instr) | - Instruction::F32DivAssign(instr) | - Instruction::F32MinAssign(instr) | - Instruction::F32MaxAssign(instr) | - Instruction::F32CopysignAssign(instr) | - Instruction::F64AddAssign(instr) | - Instruction::F64SubAssign(instr) | - Instruction::F64MulAssign(instr) | - Instruction::F64DivAssign(instr) | - Instruction::F64MinAssign(instr) | - Instruction::F64MaxAssign(instr) | - Instruction::F64CopysignAssign(instr) => instr.result_mut(), - Instruction::I32EqAssignImm(instr) | - Instruction::I32NeAssignImm(instr) | - Instruction::I32LtSAssignImm(instr) | - Instruction::I32GtSAssignImm(instr) | - Instruction::I32LeSAssignImm(instr) | - Instruction::I32GeSAssignImm(instr) => instr.result_mut(), - Instruction::I32LtUAssignImm(instr) | - Instruction::I32GtUAssignImm(instr) | - Instruction::I32LeUAssignImm(instr) | - Instruction::I32GeUAssignImm(instr) => instr.result_mut(), - Instruction::I64EqAssignImm32(instr) | - Instruction::I64NeAssignImm32(instr) | - Instruction::I64LtSAssignImm32(instr) | - Instruction::I64LeSAssignImm32(instr) | - Instruction::I64GtSAssignImm32(instr) | - Instruction::I64GeSAssignImm32(instr) => instr.result_mut(), - Instruction::I64LtUAssignImm32(instr) | - Instruction::I64GtUAssignImm32(instr) | - Instruction::I64LeUAssignImm32(instr) | - Instruction::I64GeUAssignImm32(instr) => instr.result_mut(), - Instruction::F32EqAssignImm(instr) | - Instruction::F32NeAssignImm(instr) | - Instruction::F32LtAssignImm(instr) | - Instruction::F32LeAssignImm(instr) | - Instruction::F32GtAssignImm(instr) | - Instruction::F32GeAssignImm(instr) => instr.result_mut(), - Instruction::F64EqAssignImm32(instr) | - Instruction::F64NeAssignImm32(instr) | - Instruction::F64LtAssignImm32(instr) | - Instruction::F64LeAssignImm32(instr) | - Instruction::F64GtAssignImm32(instr) | - Instruction::F64GeAssignImm32(instr) => instr.result_mut(), - Instruction::I32AddAssignImm(instr) | - Instruction::I32SubAssignImm(instr) | - Instruction::I32MulAssignImm(instr) | - Instruction::I32DivSAssignImm(instr) | - Instruction::I32RemSAssignImm(instr) | - Instruction::I32AndAssignImm(instr) | - Instruction::I32OrAssignImm(instr) | - Instruction::I32XorAssignImm(instr) | - Instruction::I32ShlAssignImm(instr) | - Instruction::I32ShrSAssignImm(instr) | - Instruction::I32ShrUAssignImm(instr) | - Instruction::I32RotlAssignImm(instr) | - Instruction::I32RotrAssignImm(instr) => instr.result_mut(), - Instruction::I32DivUAssignImm(instr) | - Instruction::I32RemUAssignImm(instr) => instr.result_mut(), - Instruction::I64AddAssignImm32(instr) | - Instruction::I64SubAssignImm32(instr) | - Instruction::I64MulAssignImm32(instr) | - Instruction::I64DivSAssignImm32(instr) | - Instruction::I64RemSAssignImm32(instr) | - Instruction::I64AndAssignImm32(instr) | - Instruction::I64OrAssignImm32(instr) | - Instruction::I64XorAssignImm32(instr) | - Instruction::I64ShlAssignImm32(instr) | - Instruction::I64ShrSAssignImm32(instr) | - Instruction::I64ShrUAssignImm32(instr) | - Instruction::I64RotlAssignImm32(instr) | - Instruction::I64RotrAssignImm32(instr) => instr.result_mut(), - Instruction::I64DivUAssignImm32(instr) | - Instruction::I64RemUAssignImm32(instr) => instr.result_mut(), - Instruction::F32AddAssignImm(instr) | - Instruction::F32SubAssignImm(instr) | - Instruction::F32MulAssignImm(instr) | - Instruction::F32DivAssignImm(instr) | - Instruction::F32MinAssignImm(instr) | - Instruction::F32MaxAssignImm(instr) => instr.result_mut(), - Instruction::F32CopysignAssignImm(instr) => instr.result_mut(), - Instruction::F64AddAssignImm32(instr) | - Instruction::F64SubAssignImm32(instr) | - Instruction::F64MulAssignImm32(instr) | - Instruction::F64DivAssignImm32(instr) | - Instruction::F64MinAssignImm32(instr) | - Instruction::F64MaxAssignImm32(instr) => instr.result_mut(), - Instruction::F64CopysignAssignImm(instr) => instr.result_mut(), - } - } -} - -/// Returns the result [`Register`] of `func` if `func` returns a single value. -/// -/// Otherwise returns `None`. -fn call_internal_result_mut<'a>( - results: &'a mut RegisterSpan, - func: CompiledFunc, - res: &ModuleResources, -) -> Option<&'a mut Register> { - let len_results = res - .engine() - .resolve_func_2(func, CompiledFuncEntity::len_results); - if len_results == 1 { - return Some(results.head_mut()); - } - None -} - -/// Returns the result [`Register`] of `func` if `func` returns a single value. -/// -/// Otherwise returns `None`. -fn call_imported_result_mut<'a>( - results: &'a mut RegisterSpan, - func: FuncIdx, - res: &ModuleResources, -) -> Option<&'a mut Register> { - let func_idx = func.to_u32().into(); - let func_type = res.get_type_of_func(func_idx); - let len_results = res - .engine() - .resolve_func_type(func_type, |func_type| func_type.results().len()); - if len_results == 1 { - return Some(results.head_mut()); - } - None -} - -/// Returns the result [`Register`] of `func` if `func_type` returns a single value. -/// -/// Otherwise returns `None`. -fn call_indirect_result_mut<'a>( - results: &'a mut RegisterSpan, - func_type: SignatureIdx, - res: &ModuleResources, -) -> Option<&'a mut Register> { - let func_type_idx = func_type.to_u32().into(); - let func_type = res.get_func_type(func_type_idx); - let len_results = res - .engine() - .resolve_func_type(func_type, |func_type| func_type.results().len()); - if len_results == 1 { - return Some(results.head_mut()); - } - None -} - -impl BinAssignInstr { - /// Returns the single `result` [`Register`] of the [`BinAssignInstr`] if any. - pub fn result_mut(&mut self) -> Option<&mut Register> { - Some(&mut self.inout) - } -} - -impl BinAssignInstrImm { - /// Returns the single `result` [`Register`] of the [`BinAssignInstrImm32`] if any. - pub fn result_mut(&mut self) -> Option<&mut Register> { - Some(&mut self.inout) - } -} - -impl LoadInstr { - /// Returns the single `result` [`Register`] of the [`LoadInstr`] if any. - pub fn result_mut(&mut self) -> Option<&mut Register> { - Some(&mut self.result) - } -} - -impl LoadAtInstr { - /// Returns the single `result` [`Register`] of the [`LoadAtInstr`] if any. - pub fn result_mut(&mut self) -> Option<&mut Register> { - Some(&mut self.result) - } -} - -impl LoadOffset16Instr { - /// Returns the single `result` [`Register`] of the [`LoadOffset16Instr`] if any. - pub fn result_mut(&mut self) -> Option<&mut Register> { - Some(&mut self.result) - } -} - -impl BinInstr { - /// Returns the single `result` [`Register`] of the [`BinInstr`] if any. - pub fn result_mut(&mut self) -> Option<&mut Register> { - Some(&mut self.result) - } -} - -impl BinInstrImm { - /// Returns the single `result` [`Register`] of the [`BinInstrImm16`] if any. - pub fn result_mut(&mut self) -> Option<&mut Register> { - Some(&mut self.result) - } -} - -impl UnaryInstr { - /// Returns the single `result` [`Register`] of the [`UnaryInstr`] if any. - pub fn result_mut(&mut self) -> Option<&mut Register> { - Some(&mut self.result) - } -} From e86b66dbac23ec67f80336d304b0619808307a5e Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 22:19:25 +0100 Subject: [PATCH 16/17] reinstate cmp+branch fusion for new op-assign instructions --- .../regmach/translator/instr_encoder.rs | 167 +++++++++++++++++- 1 file changed, 164 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs b/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs index 9b2f2b2c74..d7f84aa495 100644 --- a/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs +++ b/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs @@ -8,6 +8,8 @@ use crate::{ }, regmach::{ bytecode::{ + BinAssignInstr, + BinAssignInstrImm32, BinInstr, BinInstrImm16, BranchOffset16, @@ -965,6 +967,98 @@ impl InstrEncoder { Ok(instr) } + /// Create a fused cmp+branch instruction and wrap it in a `Some`. + /// + /// We wrap the returned value in `Some` to unify handling of a bunch of cases. + fn fuse_assign( + this: &mut InstrEncoder, + stack: &mut ValueStack, + last_instr: Instr, + instr: BinAssignInstr, + label: LabelRef, + make_instr: BranchCmpConstructor, + ) -> Result, TranslationError> { + if matches!(stack.get_register_space(instr.inout), RegisterSpace::Local) { + // We need to filter out instructions that store their result + // into a local register slot because they introduce observable behavior + // which a fused cmp+branch instruction would remove. + return Ok(None); + } + let offset = this.try_resolve_label_for(label, last_instr)?; + let Some(offset16) = BranchOffset16::new(offset) else { + // The branch offset cannot be 16-bit encoded therefore we have to bail out. + return Ok(None); + }; + Ok(Some(make_instr(instr.inout, instr.rhs, offset16))) + } + + /// Create a fused cmp+branch instruction with a 16-bit immediate and wrap it in a `Some`. + /// + /// We wrap the returned value in `Some` to unify handling of a bunch of cases. + fn fuse_assign_imm( + this: &mut InstrEncoder, + stack: &mut ValueStack, + last_instr: Instr, + instr: BinAssignInstrImm32, + label: LabelRef, + make_instr: BranchCmpConstructor, + make_instr_imm: BranchCmpImmConstructor, + ) -> Result, TranslationError> + where + T: From> + Into, + Const16: TryFrom, + { + if matches!(stack.get_register_space(instr.inout), RegisterSpace::Local) { + // We need to filter out instructions that store their result + // into a local register slot because they introduce observable behavior + // which a fused cmp+branch instruction would remove. + return Ok(None); + } + let offset = this.try_resolve_label_for(label, last_instr)?; + let Some(offset16) = BranchOffset16::new(offset) else { + // The branch offset cannot be 16-bit encoded therefore we have to bail out. + return Ok(None); + }; + let instr = match >::try_from(T::from(instr.rhs)) { + Ok(rhs) => make_instr_imm(instr.inout, rhs, offset16), + Err(_) => { + let rhs = stack.alloc_const(T::from(instr.rhs))?; + make_instr(instr.inout, rhs, offset16) + } + }; + Ok(Some(instr)) + } + + /// Create a fused cmp+branch instruction with a 16-bit immediate and wrap it in a `Some`. + /// + /// We wrap the returned value in `Some` to unify handling of a bunch of cases. + fn fuse_assign_fimm( + this: &mut InstrEncoder, + stack: &mut ValueStack, + last_instr: Instr, + instr: BinAssignInstrImm32, + label: LabelRef, + make_instr: BranchCmpConstructor, + ) -> Result, TranslationError> + where + T: From> + Into, + { + if matches!(stack.get_register_space(instr.inout), RegisterSpace::Local) { + // We need to filter out instructions that store their result + // into a local register slot because they introduce observable behavior + // which a fused cmp+branch instruction would remove. + return Ok(None); + } + let offset = this.try_resolve_label_for(label, last_instr)?; + let Some(offset16) = BranchOffset16::new(offset) else { + // The branch offset cannot be 16-bit encoded therefore we have to bail out. + return Ok(None); + }; + let rhs = stack.alloc_const(T::from(instr.rhs))?; + let instr = make_instr(instr.inout, rhs, offset16); + Ok(Some(instr)) + } + /// Create a fused cmp+branch instruction with a 16-bit immediate and wrap it in a `Some`. /// /// We wrap the returned value in `Some` to unify handling of a bunch of cases. @@ -983,10 +1077,13 @@ impl InstrEncoder { return Ok(None); } let offset = this.try_resolve_label_for(label, last_instr)?; - let instr = BranchOffset16::new(offset) - .map(|offset16| make_instr(instr.reg_in, instr.imm_in, offset16)); - Ok(instr) + let Some(offset16) = BranchOffset16::new(offset) else { + // The branch offset cannot be 16-bit encoded therefore we have to bail out. + return Ok(None); + }; + Ok(Some(make_instr(instr.reg_in, instr.imm_in, offset16))) } + use Instruction as I; let Some(last_instr) = self.last_instr else { @@ -1069,6 +1166,70 @@ impl InstrEncoder { I::I64GtUImm16(instr) => fuse_imm(self, stack, last_instr, instr, label, I::branch_i64_gt_u_imm as _)?, I::I64GeSImm16(instr) => fuse_imm(self, stack, last_instr, instr, label, I::branch_i64_ge_s_imm as _)?, I::I64GeUImm16(instr) => fuse_imm(self, stack, last_instr, instr, label, I::branch_i64_ge_u_imm as _)?, + I::I32EqAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_eq)?, + I::I32NeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_ne)?, + I::I32LtSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_lt_s)?, + I::I32LtUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_lt_u)?, + I::I32LeSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_le_s)?, + I::I32LeUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_le_u)?, + I::I32GtSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_gt_s)?, + I::I32GtUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_gt_u)?, + I::I32GeSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_ge_s)?, + I::I32GeUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_ge_u)?, + I::I64EqAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_eq)?, + I::I64NeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_ne)?, + I::I64LtSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_lt_s)?, + I::I64LtUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_lt_u)?, + I::I64LeSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_le_s)?, + I::I64LeUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_le_u)?, + I::I64GtSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_gt_s)?, + I::I64GtUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_gt_u)?, + I::I64GeSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_ge_s)?, + I::I64GeUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_ge_u)?, + I::F32EqAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f32_eq)?, + I::F32NeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f32_ne)?, + I::F32LtAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f32_lt)?, + I::F32LeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f32_le)?, + I::F32GtAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f32_gt)?, + I::F32GeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f32_ge)?, + I::F64EqAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f64_eq)?, + I::F64NeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f64_ne)?, + I::F64LtAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f64_lt)?, + I::F64LeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f64_le)?, + I::F64GtAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f64_gt)?, + I::F64GeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f64_ge)?, + I::I32EqAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_eq, I::branch_i32_eq_imm)?, + I::I32NeAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_ne, I::branch_i32_ne_imm)?, + I::I32LtSAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_lt_s, I::branch_i32_lt_s_imm)?, + I::I32LtUAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_lt_u, I::branch_i32_lt_u_imm)?, + I::I32LeSAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_le_s, I::branch_i32_le_s_imm)?, + I::I32LeUAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_le_u, I::branch_i32_le_u_imm)?, + I::I32GtSAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_gt_s, I::branch_i32_gt_s_imm)?, + I::I32GtUAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_gt_u, I::branch_i32_gt_u_imm)?, + I::I32GeSAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_ge_s, I::branch_i32_ge_s_imm)?, + I::I32GeUAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_ge_u, I::branch_i32_ge_u_imm)?, + I::I64EqAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_eq, I::branch_i64_eq_imm)?, + I::I64NeAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_ne, I::branch_i64_ne_imm)?, + I::I64LtSAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_lt_s, I::branch_i64_lt_s_imm)?, + I::I64LtUAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_lt_u, I::branch_i64_lt_u_imm)?, + I::I64LeSAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_le_s, I::branch_i64_le_s_imm)?, + I::I64LeUAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_le_u, I::branch_i64_le_u_imm)?, + I::I64GtSAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_gt_s, I::branch_i64_gt_s_imm)?, + I::I64GtUAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_gt_u, I::branch_i64_gt_u_imm)?, + I::I64GeSAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_ge_s, I::branch_i64_ge_s_imm)?, + I::I64GeUAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_ge_u, I::branch_i64_ge_u_imm)?, + I::F32EqAssignImm(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f32_eq)?, + I::F32NeAssignImm(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f32_ne)?, + I::F32LtAssignImm(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f32_lt)?, + I::F32LeAssignImm(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f32_le)?, + I::F32GtAssignImm(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f32_gt)?, + I::F32GeAssignImm(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f32_ge)?, + I::F64EqAssignImm32(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f64_eq)?, + I::F64NeAssignImm32(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f64_ne)?, + I::F64LtAssignImm32(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f64_lt)?, + I::F64LeAssignImm32(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f64_le)?, + I::F64GtAssignImm32(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f64_gt)?, + I::F64GeAssignImm32(instr) => fuse_assign_fimm(self, stack, last_instr, instr, label, I::branch_f64_ge)?, _ => None, }; if let Some(fused_instr) = fused_instr { From 10dca826ae711f4192db02f90f2be0224ee5487a Mon Sep 17 00:00:00 2001 From: Robin Freyler Date: Thu, 23 Nov 2023 22:56:02 +0100 Subject: [PATCH 17/17] add br_eqz+cmp fusion for new op-assign instructions --- .../regmach/translator/instr_encoder.rs | 113 +++++++++++++++++- 1 file changed, 110 insertions(+), 3 deletions(-) diff --git a/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs b/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs index d7f84aa495..c5d97a271b 100644 --- a/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs +++ b/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs @@ -817,6 +817,68 @@ impl InstrEncoder { Ok(instr) } + /// Create a fused cmp+branch instruction and wrap it in a `Some`. + /// + /// We wrap the returned value in `Some` to unify handling of a bunch of cases. + fn fuse_assign( + this: &mut InstrEncoder, + stack: &mut ValueStack, + last_instr: Instr, + instr: BinAssignInstr, + label: LabelRef, + make_instr: BranchCmpConstructor, + ) -> Result, TranslationError> { + if matches!(stack.get_register_space(instr.inout), RegisterSpace::Local) { + // We need to filter out instructions that store their result + // into a local register slot because they introduce observable behavior + // which a fused cmp+branch instruction would remove. + return Ok(None); + } + let offset = this.try_resolve_label_for(label, last_instr)?; + let Some(offset16) = BranchOffset16::new(offset) else { + // The branch offset cannot be 16-bit encoded therefore we have to bail out. + return Ok(None); + }; + Ok(Some(make_instr(instr.inout, instr.rhs, offset16))) + } + + /// Create a fused cmp+branch instruction with a 16-bit immediate and wrap it in a `Some`. + /// + /// We wrap the returned value in `Some` to unify handling of a bunch of cases. + fn fuse_assign_imm( + this: &mut InstrEncoder, + stack: &mut ValueStack, + last_instr: Instr, + instr: BinAssignInstrImm32, + label: LabelRef, + make_instr: BranchCmpConstructor, + make_instr_imm: BranchCmpImmConstructor, + ) -> Result, TranslationError> + where + T: From> + Into, + Const16: TryFrom, + { + if matches!(stack.get_register_space(instr.inout), RegisterSpace::Local) { + // We need to filter out instructions that store their result + // into a local register slot because they introduce observable behavior + // which a fused cmp+branch instruction would remove. + return Ok(None); + } + let offset = this.try_resolve_label_for(label, last_instr)?; + let Some(offset16) = BranchOffset16::new(offset) else { + // The branch offset cannot be 16-bit encoded therefore we have to bail out. + return Ok(None); + }; + let instr = match >::try_from(T::from(instr.rhs)) { + Ok(rhs) => make_instr_imm(instr.inout, rhs, offset16), + Err(_) => { + let rhs = stack.alloc_const(T::from(instr.rhs))?; + make_instr(instr.inout, rhs, offset16) + } + }; + Ok(Some(instr)) + } + /// Create a fused cmp+branch instruction with a 16-bit immediate and wrap it in a `Some`. /// /// We wrap the returned value in `Some` to unify handling of a bunch of cases. @@ -835,10 +897,13 @@ impl InstrEncoder { return Ok(None); } let offset = this.try_resolve_label_for(label, last_instr)?; - let instr = BranchOffset16::new(offset) - .map(|offset16| make_instr(instr.reg_in, instr.imm_in, offset16)); - Ok(instr) + let Some(offset16) = BranchOffset16::new(offset) else { + // The branch offset cannot be 16-bit encoded therefore we have to bail out. + return Ok(None); + }; + Ok(Some(make_instr(instr.reg_in, instr.imm_in, offset16))) } + use Instruction as I; let Some(last_instr) = self.last_instr else { @@ -912,6 +977,48 @@ impl InstrEncoder { I::I64GtUImm16(instr) => fuse_imm(self, stack, last_instr, instr, label, I::branch_i64_le_u_imm as _)?, I::I64GeSImm16(instr) => fuse_imm(self, stack, last_instr, instr, label, I::branch_i64_lt_s_imm as _)?, I::I64GeUImm16(instr) => fuse_imm(self, stack, last_instr, instr, label, I::branch_i64_lt_u_imm as _)?, + + I::I32EqAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_ne as _)?, + I::I32NeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_eq as _)?, + I::I32LtSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_ge_s as _)?, + I::I32LtUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_ge_u as _)?, + I::I32LeSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_gt_s as _)?, + I::I32LeUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_gt_u as _)?, + I::I32GtSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_le_s as _)?, + I::I32GtUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_le_u as _)?, + I::I32GeSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_lt_s as _)?, + I::I32GeUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i32_lt_u as _)?, + I::I64EqAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_ne as _)?, + I::I64NeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_eq as _)?, + I::I64LtSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_ge_s as _)?, + I::I64LtUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_ge_u as _)?, + I::I64LeSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_gt_s as _)?, + I::I64LeUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_gt_u as _)?, + I::I64GtSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_le_s as _)?, + I::I64GtUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_le_u as _)?, + I::I64GeSAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_lt_s as _)?, + I::I64GeUAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_i64_lt_u as _)?, + I::F32EqAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f32_ne as _)?, + I::F32NeAssign(instr) => fuse_assign(self, stack, last_instr, instr, label, I::branch_f32_eq as _)?, + // Note: We cannot fuse cmp+branch for float comparison operators due to how NaN values are treated. + I::I32EqAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_ne, I::branch_i32_ne_imm)?, + I::I32NeAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_eq, I::branch_i32_eq_imm)?, + I::I32LtSAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_lt_s, I::branch_i32_ge_s_imm)?, + I::I32LtUAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_lt_u, I::branch_i32_ge_u_imm)?, + I::I32LeSAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_le_s, I::branch_i32_gt_s_imm)?, + I::I32LeUAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_le_u, I::branch_i32_gt_u_imm)?, + I::I32GtSAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_gt_s, I::branch_i32_le_s_imm)?, + I::I32GtUAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_gt_u, I::branch_i32_le_u_imm)?, + I::I32GeSAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_ge_s, I::branch_i32_lt_s_imm)?, + I::I32GeUAssignImm(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i32_ge_u, I::branch_i32_lt_u_imm)?, + I::I64LtSAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_lt_s, I::branch_i64_ge_s_imm)?, + I::I64LtUAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_lt_u, I::branch_i64_ge_u_imm)?, + I::I64LeSAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_le_s, I::branch_i64_gt_s_imm)?, + I::I64LeUAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_le_u, I::branch_i64_gt_u_imm)?, + I::I64GtSAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_gt_s, I::branch_i64_le_s_imm)?, + I::I64GtUAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_gt_u, I::branch_i64_le_u_imm)?, + I::I64GeSAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_ge_s, I::branch_i64_lt_s_imm)?, + I::I64GeUAssignImm32(instr) => fuse_assign_imm(self, stack, last_instr, instr, label, I::branch_i64_ge_u, I::branch_i64_lt_u_imm)?, _ => None, }; if let Some(fused_instr) = fused_instr {