Implement cmp+branch instruction fusion

crates/wasmi/benches/wat/count_until.wat

@@ -3,21 +3,18 @@
 (module
   (func (export "count_until") (param $limit i32) (result i32)
     (local $counter i32)
-    (block
-        (loop
-            (br_if
-                1
-                (i32.eq
-                    (local.tee $counter
-                        (i32.add
-                            (local.get $counter)
-                            (i32.const 1)
-                        )
+    (loop
+        (br_if
+            0
+            (i32.ne
+                (local.tee $counter
+                    (i32.add
+                        (local.get $counter)
+                        (i32.const 1)
                     )
-                    (local.get $limit)
                 )
+                (local.get $limit)
             )
-            (br 0)
         )
     )
     (return (local.get $counter))

crates/wasmi/src/engine/bytecode/utils.rs

@@ -273,7 +273,6 @@ impl AddressOffset {
 #[derive(Debug, Copy, Clone, PartialEq, Eq)]
 pub struct BranchOffset(i32);
 
-#[cfg(test)]
 impl From<i32> for BranchOffset {
     fn from(index: i32) -> Self {
         Self(index)

crates/wasmi/src/engine/regmach/bytecode/construct.rs

@@ -1,8 +1,10 @@
 use super::{
-    utils::{CopysignImmInstr, Sign},
+    utils::{BranchOffset16, CopysignImmInstr, Sign},
     AnyConst32,
     BinInstr,
     BinInstrImm16,
+    BranchBinOpInstr,
+    BranchBinOpInstrImm,
     CallIndirectParams,
     Const16,
     Const32,
@@ -160,6 +162,92 @@ macro_rules! constructor_for {
     };
 }
 
+macro_rules! constructor_for_branch_binop {
+    ( $( fn $name:ident() -> Self::$op_code:ident; )* ) => {
+        impl Instruction {
+            $(
+                #[doc = concat!("Creates a new [`Instruction::", stringify!($op_code), "`].")]
+                pub fn $name(lhs: Register, rhs: Register, offset: BranchOffset16) -> Self {
+                    Self::$op_code(BranchBinOpInstr::new(lhs, rhs, offset))
+                }
+            )*
+        }
+    }
+}
+constructor_for_branch_binop! {
+    fn branch_i32_eq() -> Self::BranchI32Eq;
+    fn branch_i32_ne() -> Self::BranchI32Ne;
+    fn branch_i32_lt_s() -> Self::BranchI32LtS;
+    fn branch_i32_lt_u() -> Self::BranchI32LtU;
+    fn branch_i32_le_s() -> Self::BranchI32LeS;
+    fn branch_i32_le_u() -> Self::BranchI32LeU;
+    fn branch_i32_gt_s() -> Self::BranchI32GtS;
+    fn branch_i32_gt_u() -> Self::BranchI32GtU;
+    fn branch_i32_ge_s() -> Self::BranchI32GeS;
+    fn branch_i32_ge_u() -> Self::BranchI32GeU;
+
+    fn branch_i64_eq() -> Self::BranchI64Eq;
+    fn branch_i64_ne() -> Self::BranchI64Ne;
+    fn branch_i64_lt_s() -> Self::BranchI64LtS;
+    fn branch_i64_lt_u() -> Self::BranchI64LtU;
+    fn branch_i64_le_s() -> Self::BranchI64LeS;
+    fn branch_i64_le_u() -> Self::BranchI64LeU;
+    fn branch_i64_gt_s() -> Self::BranchI64GtS;
+    fn branch_i64_gt_u() -> Self::BranchI64GtU;
+    fn branch_i64_ge_s() -> Self::BranchI64GeS;
+    fn branch_i64_ge_u() -> Self::BranchI64GeU;
+
+    fn branch_f32_eq() -> Self::BranchF32Eq;
+    fn branch_f32_ne() -> Self::BranchF32Ne;
+    fn branch_f32_lt() -> Self::BranchF32Lt;
+    fn branch_f32_le() -> Self::BranchF32Le;
+    fn branch_f32_gt() -> Self::BranchF32Gt;
+    fn branch_f32_ge() -> Self::BranchF32Ge;
+
+    fn branch_f64_eq() -> Self::BranchF64Eq;
+    fn branch_f64_ne() -> Self::BranchF64Ne;
+    fn branch_f64_lt() -> Self::BranchF64Lt;
+    fn branch_f64_le() -> Self::BranchF64Le;
+    fn branch_f64_gt() -> Self::BranchF64Gt;
+    fn branch_f64_ge() -> Self::BranchF64Ge;
+}
+
+macro_rules! constructor_for_branch_binop_imm {
+    ( $( fn $name:ident($ty:ty) -> Self::$op_code:ident; )* ) => {
+        impl Instruction {
+            $(
+                #[doc = concat!("Creates a new [`Instruction::", stringify!($op_code), "`].")]
+                pub fn $name(lhs: Register, rhs: Const16<$ty>, offset: BranchOffset16) -> Self {
+                    Self::$op_code(BranchBinOpInstrImm::new(lhs, rhs, offset))
+                }
+            )*
+        }
+    }
+}
+constructor_for_branch_binop_imm! {
+    fn branch_i32_eq_imm(i32) -> Self::BranchI32EqImm;
+    fn branch_i32_ne_imm(i32) -> Self::BranchI32NeImm;
+    fn branch_i32_lt_s_imm(i32) -> Self::BranchI32LtSImm;
+    fn branch_i32_lt_u_imm(u32) -> Self::BranchI32LtUImm;
+    fn branch_i32_le_s_imm(i32) -> Self::BranchI32LeSImm;
+    fn branch_i32_le_u_imm(u32) -> Self::BranchI32LeUImm;
+    fn branch_i32_gt_s_imm(i32) -> Self::BranchI32GtSImm;
+    fn branch_i32_gt_u_imm(u32) -> Self::BranchI32GtUImm;
+    fn branch_i32_ge_s_imm(i32) -> Self::BranchI32GeSImm;
+    fn branch_i32_ge_u_imm(u32) -> Self::BranchI32GeUImm;
+
+    fn branch_i64_eq_imm(i64) -> Self::BranchI64EqImm;
+    fn branch_i64_ne_imm(i64) -> Self::BranchI64NeImm;
+    fn branch_i64_lt_s_imm(i64) -> Self::BranchI64LtSImm;
+    fn branch_i64_lt_u_imm(u64) -> Self::BranchI64LtUImm;
+    fn branch_i64_le_s_imm(i64) -> Self::BranchI64LeSImm;
+    fn branch_i64_le_u_imm(u64) -> Self::BranchI64LeUImm;
+    fn branch_i64_gt_s_imm(i64) -> Self::BranchI64GtSImm;
+    fn branch_i64_gt_u_imm(u64) -> Self::BranchI64GtUImm;
+    fn branch_i64_ge_s_imm(i64) -> Self::BranchI64GeSImm;
+    fn branch_i64_ge_u_imm(u64) -> Self::BranchI64GeUImm;
+}
+
 impl Instruction {
     /// Creates a new [`Instruction::Const32`] from the given `value`.
     pub fn const32(value: impl Into<AnyConst32>) -> Self {

crates/wasmi/src/engine/regmach/bytecode/immediate.rs

@@ -14,6 +14,20 @@ pub struct Const16<T> {
     marker: PhantomData<fn() -> T>,
 }
 
+impl Const16<i32> {
+    /// Returns `true` if the [`Const16`]`<i32>` is equal to zero.
+    pub fn is_zero(&self) -> bool {
+        self.inner == AnyConst16::from(0_i16)
+    }
+}
+
+impl Const16<i64> {
+    /// Returns `true` if the [`Const16`]`<i64>` is equal to zero.
+    pub fn is_zero(&self) -> bool {
+        self.inner == AnyConst16::from(0_i16)
+    }
+}
+
 impl<T> Const16<T> {
     /// Crete a new typed [`Const16`] value.
     pub fn new(inner: AnyConst16) -> Self {

crates/wasmi/src/engine/regmach/bytecode/mod.rs

@@ -12,6 +12,9 @@ pub(crate) use self::{
     utils::{
         BinInstr,
         BinInstrImm16,
+        BranchBinOpInstr,
+        BranchBinOpInstrImm,
+        BranchOffset16,
         CallIndirectParams,
         CopysignImmInstr,
         LoadAtInstr,
@@ -387,6 +390,198 @@ pub enum Instruction {
         offset: BranchOffset,
     },
 
+    /// A fused [`Instruction::I32Eq`] and [`Instruction::BranchNez`] instruction.
+    BranchI32Eq(BranchBinOpInstr),
+    /// A fused [`Instruction::I32Eq`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32Eq`] with 16-bit encoded constant `rhs`.
+    BranchI32EqImm(BranchBinOpInstrImm<i32>),
+    /// A fused [`Instruction::I32Ne`] and [`Instruction::BranchNez`] instruction.
+    BranchI32Ne(BranchBinOpInstr),
+    /// A fused [`Instruction::I32Ne`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32Ne`] with 16-bit encoded constant `rhs`.
+    BranchI32NeImm(BranchBinOpInstrImm<i32>),
+
+    /// A fused [`Instruction::I32LtS`] and [`Instruction::BranchNez`] instruction.
+    BranchI32LtS(BranchBinOpInstr),
+    /// A fused [`Instruction::I32LtS`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32LtS`] with 16-bit encoded constant `rhs`.
+    BranchI32LtSImm(BranchBinOpInstrImm<i32>),
+    /// A fused [`Instruction::I32LtU`] and [`Instruction::BranchNez`] instruction.
+    BranchI32LtU(BranchBinOpInstr),
+    /// A fused [`Instruction::I32LtU`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32LtU`] with 16-bit encoded constant `rhs`.
+    BranchI32LtUImm(BranchBinOpInstrImm<u32>),
+    /// A fused [`Instruction::I32LeS`] and [`Instruction::BranchNez`] instruction.
+    BranchI32LeS(BranchBinOpInstr),
+    /// A fused [`Instruction::I32LeS`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32LeS`] with 16-bit encoded constant `rhs`.
+    BranchI32LeSImm(BranchBinOpInstrImm<i32>),
+    /// A fused [`Instruction::I32LeU`] and [`Instruction::BranchNez`] instruction.
+    BranchI32LeU(BranchBinOpInstr),
+    /// A fused [`Instruction::I32LeU`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32LeU`] with 16-bit encoded constant `rhs`.
+    BranchI32LeUImm(BranchBinOpInstrImm<u32>),
+    /// A fused [`Instruction::I32GtS`] and [`Instruction::BranchNez`] instruction.
+    BranchI32GtS(BranchBinOpInstr),
+    /// A fused [`Instruction::I32GtS`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32GtS`] with 16-bit encoded constant `rhs`.
+    BranchI32GtSImm(BranchBinOpInstrImm<i32>),
+    /// A fused [`Instruction::I32GtU`] and [`Instruction::BranchNez`] instruction.
+    BranchI32GtU(BranchBinOpInstr),
+    /// A fused [`Instruction::I32GtU`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32GtU`] with 16-bit encoded constant `rhs`.
+    BranchI32GtUImm(BranchBinOpInstrImm<u32>),
+    /// A fused [`Instruction::I32GeS`] and [`Instruction::BranchNez`] instruction.
+    BranchI32GeS(BranchBinOpInstr),
+    /// A fused [`Instruction::I32GeS`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32GeS`] with 16-bit encoded constant `rhs`.
+    BranchI32GeSImm(BranchBinOpInstrImm<i32>),
+    /// A fused [`Instruction::I32GeU`] and [`Instruction::BranchNez`] instruction.
+    BranchI32GeU(BranchBinOpInstr),
+    /// A fused [`Instruction::I32GeU`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI32GeU`] with 16-bit encoded constant `rhs`.
+    BranchI32GeUImm(BranchBinOpInstrImm<u32>),
+
+    /// A fused [`Instruction::I64Eq`] and [`Instruction::BranchNez`] instruction.
+    BranchI64Eq(BranchBinOpInstr),
+    /// A fused [`Instruction::I64Eq`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64Eq`] with 16-bit encoded constant `rhs`.
+    BranchI64EqImm(BranchBinOpInstrImm<i64>),
+    /// A fused [`Instruction::I64Ne`] and [`Instruction::BranchNez`] instruction.
+    BranchI64Ne(BranchBinOpInstr),
+    /// A fused [`Instruction::I64Ne`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64Ne`] with 16-bit encoded constant `rhs`.
+    BranchI64NeImm(BranchBinOpInstrImm<i64>),
+
+    /// A fused [`Instruction::I64LtS`] and [`Instruction::BranchNez`] instruction.
+    BranchI64LtS(BranchBinOpInstr),
+    /// A fused [`Instruction::I64LtS`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64LtS`] with 16-bit encoded constant `rhs`.
+    BranchI64LtSImm(BranchBinOpInstrImm<i64>),
+    /// A fused [`Instruction::I64LtU`] and [`Instruction::BranchNez`] instruction.
+    BranchI64LtU(BranchBinOpInstr),
+    /// A fused [`Instruction::I64LtU`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64LtU`] with 16-bit encoded constant `rhs`.
+    BranchI64LtUImm(BranchBinOpInstrImm<u64>),
+    /// A fused [`Instruction::I64LeS`] and [`Instruction::BranchNez`] instruction.
+    BranchI64LeS(BranchBinOpInstr),
+    /// A fused [`Instruction::I64LeS`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64LeS`] with 16-bit encoded constant `rhs`.
+    BranchI64LeSImm(BranchBinOpInstrImm<i64>),
+    /// A fused [`Instruction::I64LeU`] and [`Instruction::BranchNez`] instruction.
+    BranchI64LeU(BranchBinOpInstr),
+    /// A fused [`Instruction::I64LeU`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64LeU`] with 16-bit encoded constant `rhs`.
+    BranchI64LeUImm(BranchBinOpInstrImm<u64>),
+    /// A fused [`Instruction::I64GtS`] and [`Instruction::BranchNez`] instruction.
+    BranchI64GtS(BranchBinOpInstr),
+    /// A fused [`Instruction::I64GtS`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64GtS`] with 16-bit encoded constant `rhs`.
+    BranchI64GtSImm(BranchBinOpInstrImm<i64>),
+    /// A fused [`Instruction::I64GtU`] and [`Instruction::BranchNez`] instruction.
+    BranchI64GtU(BranchBinOpInstr),
+    /// A fused [`Instruction::I64GtU`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64GtU`] with 16-bit encoded constant `rhs`.
+    BranchI64GtUImm(BranchBinOpInstrImm<u64>),
+    /// A fused [`Instruction::I64GeS`] and [`Instruction::BranchNez`] instruction.
+    BranchI64GeS(BranchBinOpInstr),
+    /// A fused [`Instruction::I64GeS`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64GeS`] with 16-bit encoded constant `rhs`.
+    BranchI64GeSImm(BranchBinOpInstrImm<i64>),
+    /// A fused [`Instruction::I64GeU`] and [`Instruction::BranchNez`] instruction.
+    BranchI64GeU(BranchBinOpInstr),
+    /// A fused [`Instruction::I64GeU`] and [`Instruction::BranchNez`] instruction.
+    ///
+    /// # Note
+    ///
+    /// Variant of [`Instruction::BranchI64GeU`] with 16-bit encoded constant `rhs`.
+    BranchI64GeUImm(BranchBinOpInstrImm<u64>),
+
+    /// A fused [`Instruction::F32Eq`] and [`Instruction::BranchNez`] instruction.
+    BranchF32Eq(BranchBinOpInstr),
+    /// A fused [`Instruction::F32Ne`] and [`Instruction::BranchNez`] instruction.
+    BranchF32Ne(BranchBinOpInstr),
+
+    /// A fused [`Instruction::F32Lt`] and [`Instruction::BranchNez`] instruction.
+    BranchF32Lt(BranchBinOpInstr),
+    /// A fused [`Instruction::F32Le`] and [`Instruction::BranchNez`] instruction.
+    BranchF32Le(BranchBinOpInstr),
+    /// A fused [`Instruction::F32Gt`] and [`Instruction::BranchNez`] instruction.
+    BranchF32Gt(BranchBinOpInstr),
+    /// A fused [`Instruction::F32Ge`] and [`Instruction::BranchNez`] instruction.
+    BranchF32Ge(BranchBinOpInstr),
+
+    /// A fused [`Instruction::F64Eq`] and [`Instruction::BranchNez`] instruction.
+    BranchF64Eq(BranchBinOpInstr),
+    /// A fused [`Instruction::F64Ne`] and [`Instruction::BranchNez`] instruction.
+    BranchF64Ne(BranchBinOpInstr),
+
+    /// A fused [`Instruction::F64Lt`] and [`Instruction::BranchNez`] instruction.
+    BranchF64Lt(BranchBinOpInstr),
+    /// A fused [`Instruction::F64Le`] and [`Instruction::BranchNez`] instruction.
+    BranchF64Le(BranchBinOpInstr),
+    /// A fused [`Instruction::F64Gt`] and [`Instruction::BranchNez`] instruction.
+    BranchF64Gt(BranchBinOpInstr),
+    /// A fused [`Instruction::F64Ge`] and [`Instruction::BranchNez`] instruction.
+    BranchF64Ge(BranchBinOpInstr),
+
     /// A Wasm `br_table` instruction.
     ///
     /// # Encoding