diff --git a/crates/wasmi/src/engine/regmach/bytecode/construct.rs b/crates/wasmi/src/engine/regmach/bytecode/construct.rs index 3082c3d059..99de76a5dd 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/construct.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/construct.rs @@ -411,14 +411,24 @@ impl Instruction { Self::Branch { offset } } - /// Creates a new [`Instruction::BranchEqz`] for the given `condition` and `offset`. - pub fn branch_eqz(condition: Register, offset: BranchOffset) -> Self { - Self::BranchEqz { condition, offset } + /// Creates a new [`Instruction::BranchI32Eqz`] for the given `condition` and `offset`. + pub fn branch_i32_eqz(condition: Register, offset: BranchOffset) -> Self { + Self::BranchI32Eqz { condition, offset } } - /// Creates a new [`Instruction::BranchNez`] for the given `condition` and `offset`. - pub fn branch_nez(condition: Register, offset: BranchOffset) -> Self { - Self::BranchNez { condition, offset } + /// Creates a new [`Instruction::BranchI32Nez`] for the given `condition` and `offset`. + pub fn branch_i32_nez(condition: Register, offset: BranchOffset) -> Self { + Self::BranchI32Nez { condition, offset } + } + + /// Creates a new [`Instruction::BranchI64Eqz`] for the given `condition` and `offset`. + pub fn branch_i64_eqz(condition: Register, offset: BranchOffset) -> Self { + Self::BranchI64Eqz { condition, offset } + } + + /// Creates a new [`Instruction::BranchI64Nez`] for the given `condition` and `offset`. + pub fn branch_i64_nez(condition: Register, offset: BranchOffset) -> Self { + Self::BranchI64Nez { condition, offset } } /// Creates a new [`Instruction::BranchTable`] for the given `index` and `len_targets`. diff --git a/crates/wasmi/src/engine/regmach/bytecode/mod.rs b/crates/wasmi/src/engine/regmach/bytecode/mod.rs index f7b09d7fb2..e58c2279fb 100644 --- a/crates/wasmi/src/engine/regmach/bytecode/mod.rs +++ b/crates/wasmi/src/engine/regmach/bytecode/mod.rs @@ -372,7 +372,7 @@ pub enum Instruction { /// /// - The branch is taken if `condition` evaluates to zero. /// - Partially translated from negated Wasm `br_if` instructions. - BranchEqz { + BranchI32Eqz { /// The register holding the condition to evaluate against zero. condition: Register, /// The branching offset for the instruction pointer. @@ -383,253 +383,275 @@ pub enum Instruction { /// # Note /// /// The branch is taken if `condition` evaluates to zero. - BranchNez { + BranchI32Nez { + /// The register holding the condition to evaluate against zero. + condition: Register, + /// The branching offset for the instruction pointer. + offset: BranchOffset, + }, + /// A fused Wasm `i64.eqz` + `if` instruction. + /// + /// # Note + /// + /// - The branch is taken if `condition` evaluates to zero. + BranchI64Eqz { + /// The register holding the condition to evaluate against zero. + condition: Register, + /// The branching offset for the instruction pointer. + offset: BranchOffset, + }, + /// A fused Wasm `i64.eqz` + `br_if` instruction. + /// + /// # Note + /// + /// The branch is taken if `condition` evaluates to zero. + BranchI64Nez { /// The register holding the condition to evaluate against zero. condition: Register, /// The branching offset for the instruction pointer. offset: BranchOffset, }, - /// A fused [`Instruction::I32And`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32And`] and [`Instruction::BranchI32Nez`] instruction. BranchI32And(BranchBinOpInstr), - /// A fused [`Instruction::I32And`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32And`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32And`] with 16-bit encoded constant `rhs`. BranchI32AndImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32Or`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32Or`] and [`Instruction::BranchI32Nez`] instruction. BranchI32Or(BranchBinOpInstr), - /// A fused [`Instruction::I32Or`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32Or`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32Or`] with 16-bit encoded constant `rhs`. BranchI32OrImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32Xor`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32Xor`] and [`Instruction::BranchI32Nez`] instruction. BranchI32Xor(BranchBinOpInstr), - /// A fused [`Instruction::I32Xor`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32Xor`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32Xor`] with 16-bit encoded constant `rhs`. BranchI32XorImm(BranchBinOpInstrImm), - /// A fused not-[`Instruction::I32And`] and [`Instruction::BranchNez`] instruction. + /// A fused not-[`Instruction::I32And`] and [`Instruction::BranchI32Nez`] instruction. BranchI32AndEqz(BranchBinOpInstr), - /// A fused not-[`Instruction::I32And`] and [`Instruction::BranchNez`] instruction. + /// A fused not-[`Instruction::I32And`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32AndEqz`] with 16-bit encoded constant `rhs`. BranchI32AndEqzImm(BranchBinOpInstrImm), - /// A fused not-[`Instruction::I32Or`] and [`Instruction::BranchNez`] instruction. + /// A fused not-[`Instruction::I32Or`] and [`Instruction::BranchI32Nez`] instruction. BranchI32OrEqz(BranchBinOpInstr), - /// A fused not-[`Instruction::I32Or`] and [`Instruction::BranchNez`] instruction. + /// A fused not-[`Instruction::I32Or`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32OrEqz`] with 16-bit encoded constant `rhs`. BranchI32OrEqzImm(BranchBinOpInstrImm), - /// A fused not-[`Instruction::I32Xor`] and [`Instruction::BranchNez`] instruction. + /// A fused not-[`Instruction::I32Xor`] and [`Instruction::BranchI32Nez`] instruction. BranchI32XorEqz(BranchBinOpInstr), - /// A fused not-[`Instruction::I32Xor`] and [`Instruction::BranchNez`] instruction. + /// A fused not-[`Instruction::I32Xor`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32XorEqz`] with 16-bit encoded constant `rhs`. BranchI32XorEqzImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32Eq`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32Eq`] and [`Instruction::BranchI32Nez`] instruction. BranchI32Eq(BranchBinOpInstr), - /// A fused [`Instruction::I32Eq`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32Eq`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32Eq`] with 16-bit encoded constant `rhs`. BranchI32EqImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32Ne`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32Ne`] and [`Instruction::BranchI32Nez`] instruction. BranchI32Ne(BranchBinOpInstr), - /// A fused [`Instruction::I32Ne`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32Ne`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32Ne`] with 16-bit encoded constant `rhs`. BranchI32NeImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32LtS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32LtS`] and [`Instruction::BranchI32Nez`] instruction. BranchI32LtS(BranchBinOpInstr), - /// A fused [`Instruction::I32LtS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32LtS`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32LtS`] with 16-bit encoded constant `rhs`. BranchI32LtSImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32LtU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32LtU`] and [`Instruction::BranchI32Nez`] instruction. BranchI32LtU(BranchBinOpInstr), - /// A fused [`Instruction::I32LtU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32LtU`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32LtU`] with 16-bit encoded constant `rhs`. BranchI32LtUImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32LeS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32LeS`] and [`Instruction::BranchI32Nez`] instruction. BranchI32LeS(BranchBinOpInstr), - /// A fused [`Instruction::I32LeS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32LeS`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32LeS`] with 16-bit encoded constant `rhs`. BranchI32LeSImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32LeU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32LeU`] and [`Instruction::BranchI32Nez`] instruction. BranchI32LeU(BranchBinOpInstr), - /// A fused [`Instruction::I32LeU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32LeU`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32LeU`] with 16-bit encoded constant `rhs`. BranchI32LeUImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32GtS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32GtS`] and [`Instruction::BranchI32Nez`] instruction. BranchI32GtS(BranchBinOpInstr), - /// A fused [`Instruction::I32GtS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32GtS`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32GtS`] with 16-bit encoded constant `rhs`. BranchI32GtSImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32GtU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32GtU`] and [`Instruction::BranchI32Nez`] instruction. BranchI32GtU(BranchBinOpInstr), - /// A fused [`Instruction::I32GtU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32GtU`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32GtU`] with 16-bit encoded constant `rhs`. BranchI32GtUImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32GeS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32GeS`] and [`Instruction::BranchI32Nez`] instruction. BranchI32GeS(BranchBinOpInstr), - /// A fused [`Instruction::I32GeS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32GeS`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32GeS`] with 16-bit encoded constant `rhs`. BranchI32GeSImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I32GeU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32GeU`] and [`Instruction::BranchI32Nez`] instruction. BranchI32GeU(BranchBinOpInstr), - /// A fused [`Instruction::I32GeU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I32GeU`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI32GeU`] with 16-bit encoded constant `rhs`. BranchI32GeUImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64Eq`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64Eq`] and [`Instruction::BranchI32Nez`] instruction. BranchI64Eq(BranchBinOpInstr), - /// A fused [`Instruction::I64Eq`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64Eq`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64Eq`] with 16-bit encoded constant `rhs`. BranchI64EqImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64Ne`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64Ne`] and [`Instruction::BranchI32Nez`] instruction. BranchI64Ne(BranchBinOpInstr), - /// A fused [`Instruction::I64Ne`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64Ne`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64Ne`] with 16-bit encoded constant `rhs`. BranchI64NeImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64LtS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64LtS`] and [`Instruction::BranchI32Nez`] instruction. BranchI64LtS(BranchBinOpInstr), - /// A fused [`Instruction::I64LtS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64LtS`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64LtS`] with 16-bit encoded constant `rhs`. BranchI64LtSImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64LtU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64LtU`] and [`Instruction::BranchI32Nez`] instruction. BranchI64LtU(BranchBinOpInstr), - /// A fused [`Instruction::I64LtU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64LtU`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64LtU`] with 16-bit encoded constant `rhs`. BranchI64LtUImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64LeS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64LeS`] and [`Instruction::BranchI32Nez`] instruction. BranchI64LeS(BranchBinOpInstr), - /// A fused [`Instruction::I64LeS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64LeS`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64LeS`] with 16-bit encoded constant `rhs`. BranchI64LeSImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64LeU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64LeU`] and [`Instruction::BranchI32Nez`] instruction. BranchI64LeU(BranchBinOpInstr), - /// A fused [`Instruction::I64LeU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64LeU`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64LeU`] with 16-bit encoded constant `rhs`. BranchI64LeUImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64GtS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64GtS`] and [`Instruction::BranchI32Nez`] instruction. BranchI64GtS(BranchBinOpInstr), - /// A fused [`Instruction::I64GtS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64GtS`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64GtS`] with 16-bit encoded constant `rhs`. BranchI64GtSImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64GtU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64GtU`] and [`Instruction::BranchI32Nez`] instruction. BranchI64GtU(BranchBinOpInstr), - /// A fused [`Instruction::I64GtU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64GtU`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64GtU`] with 16-bit encoded constant `rhs`. BranchI64GtUImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64GeS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64GeS`] and [`Instruction::BranchI32Nez`] instruction. BranchI64GeS(BranchBinOpInstr), - /// A fused [`Instruction::I64GeS`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64GeS`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64GeS`] with 16-bit encoded constant `rhs`. BranchI64GeSImm(BranchBinOpInstrImm), - /// A fused [`Instruction::I64GeU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64GeU`] and [`Instruction::BranchI32Nez`] instruction. BranchI64GeU(BranchBinOpInstr), - /// A fused [`Instruction::I64GeU`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::I64GeU`] and [`Instruction::BranchI32Nez`] instruction. /// /// # Note /// /// Variant of [`Instruction::BranchI64GeU`] with 16-bit encoded constant `rhs`. BranchI64GeUImm(BranchBinOpInstrImm), - /// A fused [`Instruction::F32Eq`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F32Eq`] and [`Instruction::BranchI32Nez`] instruction. BranchF32Eq(BranchBinOpInstr), - /// A fused [`Instruction::F32Ne`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F32Ne`] and [`Instruction::BranchI32Nez`] instruction. BranchF32Ne(BranchBinOpInstr), - /// A fused [`Instruction::F32Lt`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F32Lt`] and [`Instruction::BranchI32Nez`] instruction. BranchF32Lt(BranchBinOpInstr), - /// A fused [`Instruction::F32Le`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F32Le`] and [`Instruction::BranchI32Nez`] instruction. BranchF32Le(BranchBinOpInstr), - /// A fused [`Instruction::F32Gt`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F32Gt`] and [`Instruction::BranchI32Nez`] instruction. BranchF32Gt(BranchBinOpInstr), - /// A fused [`Instruction::F32Ge`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F32Ge`] and [`Instruction::BranchI32Nez`] instruction. BranchF32Ge(BranchBinOpInstr), - /// A fused [`Instruction::F64Eq`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F64Eq`] and [`Instruction::BranchI32Nez`] instruction. BranchF64Eq(BranchBinOpInstr), - /// A fused [`Instruction::F64Ne`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F64Ne`] and [`Instruction::BranchI32Nez`] instruction. BranchF64Ne(BranchBinOpInstr), - /// A fused [`Instruction::F64Lt`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F64Lt`] and [`Instruction::BranchI32Nez`] instruction. BranchF64Lt(BranchBinOpInstr), - /// A fused [`Instruction::F64Le`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F64Le`] and [`Instruction::BranchI32Nez`] instruction. BranchF64Le(BranchBinOpInstr), - /// A fused [`Instruction::F64Gt`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F64Gt`] and [`Instruction::BranchI32Nez`] instruction. BranchF64Gt(BranchBinOpInstr), - /// A fused [`Instruction::F64Ge`] and [`Instruction::BranchNez`] instruction. + /// A fused [`Instruction::F64Ge`] and [`Instruction::BranchI32Nez`] instruction. BranchF64Ge(BranchBinOpInstr), /// A Wasm `br_table` instruction. diff --git a/crates/wasmi/src/engine/regmach/executor/instrs.rs b/crates/wasmi/src/engine/regmach/executor/instrs.rs index 824d308858..ec40a71e7a 100644 --- a/crates/wasmi/src/engine/regmach/executor/instrs.rs +++ b/crates/wasmi/src/engine/regmach/executor/instrs.rs @@ -245,11 +245,17 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { forward_return!(self.execute_return_nez_many(condition, values)) } Instr::Branch { offset } => self.execute_branch(offset), - Instr::BranchEqz { condition, offset } => { - self.execute_branch_eqz(condition, offset) + Instr::BranchI32Eqz { condition, offset } => { + self.execute_branch_i32_eqz(condition, offset) } - Instr::BranchNez { condition, offset } => { - self.execute_branch_nez(condition, offset) + Instr::BranchI32Nez { condition, offset } => { + self.execute_branch_i32_nez(condition, offset) + } + Instr::BranchI64Eqz { condition, offset } => { + self.execute_branch_i64_eqz(condition, offset) + } + Instr::BranchI64Nez { condition, offset } => { + self.execute_branch_i64_nez(condition, offset) } Instr::BranchTable { index, len_targets } => { self.execute_branch_table(index, len_targets) diff --git a/crates/wasmi/src/engine/regmach/executor/instrs/branch.rs b/crates/wasmi/src/engine/regmach/executor/instrs/branch.rs index e05f243162..03071eee6c 100644 --- a/crates/wasmi/src/engine/regmach/executor/instrs/branch.rs +++ b/crates/wasmi/src/engine/regmach/executor/instrs/branch.rs @@ -16,29 +16,39 @@ impl<'ctx, 'engine> Executor<'ctx, 'engine> { } #[inline(always)] - pub fn execute_branch_nez(&mut self, condition: Register, offset: BranchOffset) { - let condition: bool = self.get_register_as(condition); - match condition { - true => { - self.branch_to(offset); - } - false => { - self.next_instr(); - } + pub fn execute_branch_i32_nez(&mut self, condition: Register, offset: BranchOffset) { + let value: i32 = self.get_register_as(condition); + if value != 0 { + return self.branch_to(offset); } + self.next_instr(); } #[inline(always)] - pub fn execute_branch_eqz(&mut self, condition: Register, offset: BranchOffset) { - let condition: bool = self.get_register_as(condition); - match condition { - true => { - self.next_instr(); - } - false => { - self.branch_to(offset); - } + pub fn execute_branch_i32_eqz(&mut self, condition: Register, offset: BranchOffset) { + let value: i32 = self.get_register_as(condition); + if value == 0 { + return self.branch_to(offset); } + self.next_instr(); + } + + #[inline(always)] + pub fn execute_branch_i64_nez(&mut self, condition: Register, offset: BranchOffset) { + let value: i64 = self.get_register_as(condition); + if value != 0 { + return self.branch_to(offset); + } + self.next_instr(); + } + + #[inline(always)] + pub fn execute_branch_i64_eqz(&mut self, condition: Register, offset: BranchOffset) { + let value: i64 = self.get_register_as(condition); + if value == 0 { + return self.branch_to(offset); + } + self.next_instr(); } #[inline(always)] diff --git a/crates/wasmi/src/engine/regmach/tests/op/block.rs b/crates/wasmi/src/engine/regmach/tests/op/block.rs index 25cd58b302..5b89a53d39 100644 --- a/crates/wasmi/src/engine/regmach/tests/op/block.rs +++ b/crates/wasmi/src/engine/regmach/tests/op/block.rs @@ -346,7 +346,7 @@ fn branch_if_block_0() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_nez(Register::from_i16(0), BranchOffset::from(1)), + Instruction::branch_i32_nez(Register::from_i16(0), BranchOffset::from(1)), Instruction::Return, ]) .run() @@ -369,7 +369,7 @@ fn branch_if_block_1() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(1), BranchOffset::from(3)), + Instruction::branch_i32_eqz(Register::from_i16(1), BranchOffset::from(3)), Instruction::copy(Register::from_i16(2), Register::from_i16(0)), Instruction::branch(BranchOffset::from(2)), Instruction::copy(Register::from_i16(2), Register::from_i16(0)), diff --git a/crates/wasmi/src/engine/regmach/tests/op/br_if.rs b/crates/wasmi/src/engine/regmach/tests/op/br_if.rs index caa4fc69be..e4259c8c08 100644 --- a/crates/wasmi/src/engine/regmach/tests/op/br_if.rs +++ b/crates/wasmi/src/engine/regmach/tests/op/br_if.rs @@ -844,7 +844,7 @@ fn branch_if_results_0() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_nez(Register::from_i16(0), BranchOffset::from(1)), + Instruction::branch_i32_nez(Register::from_i16(0), BranchOffset::from(1)), Instruction::Return, ]) .run() @@ -867,7 +867,7 @@ fn branch_if_results_1() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(1), BranchOffset::from(3)), + Instruction::branch_i32_eqz(Register::from_i16(1), BranchOffset::from(3)), Instruction::copy(Register::from_i16(2), Register::from_i16(0)), Instruction::branch(BranchOffset::from(2)), Instruction::copy(Register::from_i16(2), Register::from_i16(0)), @@ -902,7 +902,7 @@ fn branch_if_results_1_avoid_copy() { .expect_func_instrs([ Instruction::i32_clz(Register::from_i16(2), Register::from_i16(0)), Instruction::i32_ctz(Register::from_i16(3), Register::from_i16(1)), - Instruction::branch_nez(Register::from_i16(3), BranchOffset::from(1)), + Instruction::branch_i32_nez(Register::from_i16(3), BranchOffset::from(1)), Instruction::return_reg(Register::from_i16(2)), ]) .run() @@ -927,7 +927,7 @@ fn branch_if_results_2() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(2), BranchOffset::from(3)), + Instruction::branch_i32_eqz(Register::from_i16(2), BranchOffset::from(3)), Instruction::copy2(RegisterSpan::new(Register::from_i16(3)), 0, 1), Instruction::branch(BranchOffset::from(2)), Instruction::copy2(RegisterSpan::new(Register::from_i16(3)), 0, 1), @@ -968,7 +968,7 @@ fn branch_if_results_2_avoid_copy() { .expect_func_instrs([ Instruction::i32_clz(Register::from_i16(3), Register::from_i16(0)), Instruction::i32_ctz(Register::from_i16(4), Register::from_i16(1)), - Instruction::branch_nez(Register::from_i16(2), BranchOffset::from(1)), + Instruction::branch_i32_nez(Register::from_i16(2), BranchOffset::from(1)), Instruction::i32_add( Register::from_i16(3), Register::from_i16(3), @@ -1002,7 +1002,7 @@ fn branch_if_results_4_mixed_1() { TranslationTest::new(wasm) .expect_func( ExpectedFunc::new([ - Instruction::branch_eqz(Register::from_i16(2), BranchOffset::from(4)), + Instruction::branch_i32_eqz(Register::from_i16(2), BranchOffset::from(4)), Instruction::copy_many_non_overlapping( RegisterSpan::new(Register::from_i16(3)), -1, @@ -1045,7 +1045,7 @@ fn branch_if_results_4_mixed_2() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(2), BranchOffset::from(4)), + Instruction::branch_i32_eqz(Register::from_i16(2), BranchOffset::from(4)), Instruction::copy_many_non_overlapping(RegisterSpan::new(Register::from_i16(3)), 0, 0), Instruction::register2(1, 1), Instruction::branch(BranchOffset::from(3)), diff --git a/crates/wasmi/src/engine/regmach/tests/op/cmp_br.rs b/crates/wasmi/src/engine/regmach/tests/op/cmp_br.rs index f1bb88cfe6..6a628bc9c7 100644 --- a/crates/wasmi/src/engine/regmach/tests/op/cmp_br.rs +++ b/crates/wasmi/src/engine/regmach/tests/op/cmp_br.rs @@ -171,8 +171,8 @@ fn loop_backward_imm_eqz() { ]) .run() } - test_for("eq", Instruction::branch_eqz); - test_for("ne", Instruction::branch_nez); + test_for("eq", Instruction::branch_i32_eqz); + test_for("ne", Instruction::branch_i32_nez); } #[test] @@ -522,3 +522,47 @@ fn if_i32_eqz_fuse() { test_for("or", Instruction::branch_i32_or); test_for("xor", Instruction::branch_i32_xor); } + +#[test] +#[cfg_attr(miri, ignore)] +fn block_i64_eqz_fuse() { + let wasm = wat2wasm( + r" + (module + (func (param i64) + (block + (i64.eqz (local.get 0)) + (br_if 0) + ) + ) + )", + ); + TranslationTest::new(wasm) + .expect_func_instrs([ + Instruction::branch_i64_eqz(Register::from_i16(0), BranchOffset::from(1)), + Instruction::Return, + ]) + .run() +} + +#[test] +#[cfg_attr(miri, ignore)] +fn if_i64_eqz_fuse() { + let wasm = wat2wasm( + r" + (module + (func (param i64) + (if + (i64.eqz (local.get 0)) + (then) + ) + ) + )", + ); + TranslationTest::new(wasm) + .expect_func_instrs([ + Instruction::branch_i64_nez(Register::from_i16(0), BranchOffset::from(1)), + Instruction::Return, + ]) + .run() +} diff --git a/crates/wasmi/src/engine/regmach/tests/op/if_.rs b/crates/wasmi/src/engine/regmach/tests/op/if_.rs index c24d7c9c5e..3f496ad747 100644 --- a/crates/wasmi/src/engine/regmach/tests/op/if_.rs +++ b/crates/wasmi/src/engine/regmach/tests/op/if_.rs @@ -21,7 +21,7 @@ fn simple_if_then() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(1)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(1)), Instruction::Return, ]) .run() @@ -46,8 +46,8 @@ fn simple_if_then_nested() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(2)), - Instruction::branch_eqz(Register::from_i16(1), BranchOffset::from(1)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(2)), + Instruction::branch_i32_eqz(Register::from_i16(1), BranchOffset::from(1)), Instruction::Return, ]) .run() @@ -72,7 +72,7 @@ fn if_then_global_set() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(2)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(2)), Instruction::global_set(GlobalIdx::from(0), Register::from_i16(1)), Instruction::return_imm32(AnyConst32::from(10_i32)), ]) @@ -102,7 +102,7 @@ fn if_then_return() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(3)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(3)), Instruction::i32_add( Register::from_i16(3), Register::from_i16(1), @@ -135,7 +135,7 @@ fn if_then_else_return() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(2)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(2)), Instruction::return_imm32(AnyConst32::from(10_i32)), Instruction::return_imm32(AnyConst32::from(20_i32)), ]) @@ -163,7 +163,7 @@ fn if_then_br_else() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(2)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(2)), Instruction::branch(BranchOffset::from(2)), Instruction::return_imm32(AnyConst32::from(10_i32)), Instruction::return_imm32(AnyConst32::from(20_i32)), @@ -192,7 +192,7 @@ fn if_then_else_br() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(2)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(2)), Instruction::return_imm32(AnyConst32::from(10_i32)), Instruction::branch(BranchOffset::from(1)), Instruction::return_imm32(AnyConst32::from(20_i32)), @@ -218,7 +218,7 @@ fn simple_if_then_else() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(2)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(2)), Instruction::branch(BranchOffset::from(1)), Instruction::Return, ]) @@ -251,11 +251,11 @@ fn simple_if_then_else_nested() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(4)), - Instruction::branch_eqz(Register::from_i16(1), BranchOffset::from(2)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(4)), + Instruction::branch_i32_eqz(Register::from_i16(1), BranchOffset::from(2)), Instruction::branch(BranchOffset::from(1)), Instruction::branch(BranchOffset::from(3)), - Instruction::branch_eqz(Register::from_i16(1), BranchOffset::from(2)), + Instruction::branch_i32_eqz(Register::from_i16(1), BranchOffset::from(2)), Instruction::branch(BranchOffset::from(1)), Instruction::Return, ]) @@ -280,7 +280,7 @@ fn if_then_else_with_params() { ); TranslationTest::new(wasm) .expect_func_instrs([ - Instruction::branch_eqz(Register::from_i16(0), BranchOffset::from(3)), + Instruction::branch_i32_eqz(Register::from_i16(0), BranchOffset::from(3)), Instruction::i32_add( Register::from_i16(3), Register::from_i16(1), @@ -445,7 +445,7 @@ fn const_condition_br_if_then() { test_for( false, [ - Instruction::branch_nez(Register::from_i16(0), BranchOffset::from(2)), + Instruction::branch_i32_nez(Register::from_i16(0), BranchOffset::from(2)), Instruction::Trap(TrapCode::UnreachableCodeReached), Instruction::return_imm32(AnyConst32::from(1_i32)), ], @@ -484,7 +484,7 @@ fn const_condition_br_if_else() { test_for( true, [ - Instruction::branch_nez(Register::from_i16(0), BranchOffset::from(2)), + Instruction::branch_i32_nez(Register::from_i16(0), BranchOffset::from(2)), Instruction::Trap(TrapCode::UnreachableCodeReached), Instruction::return_imm32(AnyConst32::from(1_i32)), ], @@ -569,7 +569,7 @@ fn test_if_without_else_has_result() { RegisterSpan::new(Register::from_i16(0)), CompiledFunc::from_u32(0), ), - Instruction::branch_eqz(Register::from_i16(1), BranchOffset::from(3)), + Instruction::branch_i32_eqz(Register::from_i16(1), BranchOffset::from(3)), Instruction::copy_i64imm32(Register::from_i16(0), -1), Instruction::branch(BranchOffset::from(1)), Instruction::return_reg(Register::from_i16(0)), diff --git a/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs b/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs index 3c7f5f3ab9..90ca2e3a03 100644 --- a/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs +++ b/crates/wasmi/src/engine/regmach/translator/instr_encoder.rs @@ -851,7 +851,7 @@ impl InstrEncoder { label: LabelRef, ) -> Result<(), TranslationError> { let offset = this.try_resolve_label(label)?; - this.push_instr(Instruction::branch_eqz(condition, offset))?; + this.push_instr(Instruction::branch_i32_eqz(condition, offset))?; Ok(()) } @@ -912,10 +912,17 @@ impl InstrEncoder { match stack.get_register_space(instr.result) { RegisterSpace::Local => None, _ => { - // Note: unfortunately we cannot apply this optimization for `i64` variants - // since the standard `branch_eqz` assumes its operands to be of type `i32`. let offset32 = self.try_resolve_label_for(label, last_instr)?; - Some(Instruction::branch_nez(instr.reg_in, offset32)) + Some(Instruction::branch_i32_nez(instr.reg_in, offset32)) + } + } + } + I::I64EqImm16(instr) if instr.imm_in.is_zero() => { + match stack.get_register_space(instr.result) { + RegisterSpace::Local => None, + _ => { + let offset32 = self.try_resolve_label_for(label, last_instr)?; + Some(Instruction::branch_i64_nez(instr.reg_in, offset32)) } } } @@ -923,10 +930,17 @@ impl InstrEncoder { match stack.get_register_space(instr.result) { RegisterSpace::Local => None, _ => { - // Note: unfortunately we cannot apply this optimization for `i64` variants - // since the standard `branch_nez` assumes its operands to be of type `i32`. let offset32 = self.try_resolve_label_for(label, last_instr)?; - Some(Instruction::branch_eqz(instr.reg_in, offset32)) + Some(Instruction::branch_i32_eqz(instr.reg_in, offset32)) + } + } + } + I::I64NeImm16(instr) if instr.imm_in.is_zero() => { + match stack.get_register_space(instr.result) { + RegisterSpace::Local => None, + _ => { + let offset32 = self.try_resolve_label_for(label, last_instr)?; + Some(Instruction::branch_i64_eqz(instr.reg_in, offset32)) } } } @@ -1013,7 +1027,7 @@ impl InstrEncoder { label: LabelRef, ) -> Result<(), TranslationError> { let offset = this.try_resolve_label(label)?; - this.push_instr(Instruction::branch_nez(condition, offset))?; + this.push_instr(Instruction::branch_i32_nez(condition, offset))?; Ok(()) } @@ -1074,10 +1088,17 @@ impl InstrEncoder { match stack.get_register_space(instr.result) { RegisterSpace::Local => None, _ => { - // Note: unfortunately we cannot apply this optimization for `i64` variants - // since the standard `branch_eqz` assumes its operands to be of type `i32`. let offset32 = self.try_resolve_label_for(label, last_instr)?; - Some(Instruction::branch_eqz(instr.reg_in, offset32)) + Some(Instruction::branch_i32_eqz(instr.reg_in, offset32)) + } + } + } + I::I64EqImm16(instr) if instr.imm_in.is_zero() => { + match stack.get_register_space(instr.result) { + RegisterSpace::Local => None, + _ => { + let offset32 = self.try_resolve_label_for(label, last_instr)?; + Some(Instruction::branch_i64_eqz(instr.reg_in, offset32)) } } } @@ -1085,10 +1106,17 @@ impl InstrEncoder { match stack.get_register_space(instr.result) { RegisterSpace::Local => None, _ => { - // Note: unfortunately we cannot apply this optimization for `i64` variants - // since the standard `branch_nez` assumes its operands to be of type `i32`. let offset32 = self.try_resolve_label_for(label, last_instr)?; - Some(Instruction::branch_nez(instr.reg_in, offset32)) + Some(Instruction::branch_i32_nez(instr.reg_in, offset32)) + } + } + } + I::I64NeImm16(instr) if instr.imm_in.is_zero() => { + match stack.get_register_space(instr.result) { + RegisterSpace::Local => None, + _ => { + let offset32 = self.try_resolve_label_for(label, last_instr)?; + Some(Instruction::branch_i64_nez(instr.reg_in, offset32)) } } } @@ -1178,8 +1206,10 @@ impl Instruction { ) -> Result<(), TranslationError> { match self { Instruction::Branch { offset } - | Instruction::BranchEqz { offset, .. } - | Instruction::BranchNez { offset, .. } => { + | Instruction::BranchI32Eqz { offset, .. } + | Instruction::BranchI32Nez { offset, .. } + | Instruction::BranchI64Eqz { offset, .. } + | Instruction::BranchI64Nez { offset, .. } => { offset.init(new_offset); Ok(()) } diff --git a/crates/wasmi/src/engine/regmach/translator/result_mut.rs b/crates/wasmi/src/engine/regmach/translator/result_mut.rs index 4e4547958e..71fa2f9926 100644 --- a/crates/wasmi/src/engine/regmach/translator/result_mut.rs +++ b/crates/wasmi/src/engine/regmach/translator/result_mut.rs @@ -67,8 +67,10 @@ impl Instruction { Instruction::ReturnNezSpan { .. } | Instruction::ReturnNezMany { .. } | Instruction::Branch { .. } | - Instruction::BranchEqz { .. } | - Instruction::BranchNez { .. } | + Instruction::BranchI32Eqz { .. } | + Instruction::BranchI32Nez { .. } | + Instruction::BranchI64Eqz { .. } | + Instruction::BranchI64Nez { .. } | Instruction::BranchTable { .. } => None, Instruction::BranchI32And(_) | Instruction::BranchI32AndImm(_) | diff --git a/crates/wasmi/src/engine/regmach/translator/visit_register.rs b/crates/wasmi/src/engine/regmach/translator/visit_register.rs index 731e7302d7..84d6840103 100644 --- a/crates/wasmi/src/engine/regmach/translator/visit_register.rs +++ b/crates/wasmi/src/engine/regmach/translator/visit_register.rs @@ -79,8 +79,10 @@ impl VisitInputRegisters for Instruction { values.visit_input_registers(f); } Instruction::Branch { .. } => {}, - Instruction::BranchEqz { condition, .. } | - Instruction::BranchNez { condition, .. } => f(condition), + Instruction::BranchI32Eqz { condition, .. } | + Instruction::BranchI32Nez { condition, .. } | + Instruction::BranchI64Eqz { condition, .. } | + Instruction::BranchI64Nez { condition, .. } => f(condition), Instruction::BranchTable { index, .. } => f(index), Instruction::BranchI32And(instr) => instr.visit_input_registers(f),