Merge pull request #556 from 0xPolygonMiden/next

Tracking PR for v0.4 release

.github/workflows/ci.yml

@@ -7,14 +7,37 @@ on:
     types: [opened, repoened, synchronize]
 
 jobs:
+  check:
+    name: Check Rust ${{matrix.toolchain}} on ${{matrix.os}} with ${{matrix.args}}
+    runs-on: ${{matrix.os}}-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        toolchain: [stable, nightly]
+        os: [ubuntu]
+        args: [--all-targets --no-default-features, --all-targets, --all-targets --all-features]
+    steps:
+      - uses: actions/checkout@main
+      - name: Install rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{matrix.toolchain}}
+          override: true
+      - name: Check
+        uses: actions-rs/cargo@v1
+        with:
+          command: check
+          args: ${{matrix.args}}
+
   test:
-    name: Test Rust ${{matrix.toolchain}} on ${{matrix.os}}
+    name: Test Rust ${{matrix.toolchain}} on ${{matrix.os}} with ${{matrix.args}}
     runs-on: ${{matrix.os}}-latest
     strategy:
       fail-fast: false
       matrix:
         toolchain: [stable, nightly]
         os: [ubuntu]
+        args: [--release, --doc]
     steps:
       - uses: actions/checkout@main
       - name: Install rust
@@ -28,7 +51,31 @@ jobs:
           RUSTFLAGS: -C debug-assertions
         with:
           command: test
-          args: --release
+          args: ${{matrix.args}} --features "internals"
+
+  # we separate the script so the CI will not require the same runner to have
+  # both windows and linux capabilities
+  test-windows:
+    name: Test Rust nightly on windows-2022
+    # run windows check only when the target is `main`. will execute for release, push or PR.
+    if: github.ref_name == 'main'
+    runs-on: windows-2022
+    strategy:
+      fail-fast: false
+    steps:
+      - uses: actions/checkout@main
+      - name: Install rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: nightly
+          override: true
+      - name: Test
+        uses: actions-rs/cargo@v1
+        env:
+          RUSTFLAGS: -C debug-assertions
+        with:
+          command: test
+          args: --release --features "internals"
 
   clippy:
     name: Clippy

.gitignore

@@ -11,3 +11,9 @@ Cargo.lock
 
 # These are failure persistance files generated by proptest
 proptest-regressions/
+
+# This is a file generated by the build process:
+stdlib/assets/std.masl
+
+# These are files generated by MacOS
+**/.DS_Store

CHANGELOG.md

@@ -1,5 +1,37 @@
 # Changelog
 
+## 0.4.0 (2023-02-27)
+
+#### Advice provider
+- [BREAKING] Converted `AdviceProvider` into a trait which can be provided to the processor.
+- Added a decorator for interpolating polynomials over degree 2 extension field (`ext2intt`).
+- Added `AdviceSource` enum for greater future flexibility of advice injectors.
+
+#### CLI
+- Added `debug` subcommand to enable stepping through program execution forward/backward.
+- Added cycle count to the output of program execution.
+
+#### Assembly
+- Added support for constant declarations.
+- Added new instructions: `clk`, `ext2*`, `fri_ext2fold4`, `hash`, `u32checked_popcnt`, `u32unchecked_popcnt`.
+- [BREAKING] Renamed `rpperm` to `hperm` and `rphash` to `hmerge`.
+- Removed requirement that code blocks must be non-empty (i.e., allowed empty blocks).
+- [BREAKING] Refactored `mtree_set` and `mtree_cwm` instructions to leave the old value on the stack.
+- [BREAKING] Replaced `ModuleProvider` with `Library` to improve 3rd party library support.
+
+#### Processor, Prover, and Verifier
+- [BREAKING] Refactored `execute()`, `prove()`, `verify()` functions to take `StackInputs` as one of the parameters.
+- [BREAKING] Refactored `prove()` function to return `ExecutionProof` (which is a wrapper for `StarkProof`).
+- [BREAKING] Refactored `verify()` function to take `ProgramInfo`, `StackInputs`, and `ExecutionProof` as parameters and return a `u32` indicating security level of the verified proof.
+
+#### Stdlib
+- Added `std::mem::memcopy` procedure for copying regions of memory.
+- Added `std::crypto::fri::frie2f4::verify` for verifying FRI proofs over degree 2 extension field.
+
+#### VM Internals
+- [BREAKING] Migrated to Rescue Prime Optimized hash function.
+- Updated Winterfell backend to v0.5.1
+
 ## 0.3.0 (2022-11-23)
 
 - Implemented `call` operation for context-isolated function calls.

CONTRIBUTING.md

@@ -74,7 +74,7 @@ For example, a new change to the AIR crate might have the following message: `fe
     // ================================================================================
     ```
 
-- [Rustfmt](https://github.com/rust-lang/rustfmt) and [Clippy](https://github.com/rust-lang/rust-clippy) linting is included in CI pipeline. Anyways it's prefferable to run linting locally before push:
+- [Rustfmt](https://github.com/rust-lang/rustfmt) and [Clippy](https://github.com/rust-lang/rust-clippy) linting is included in CI pipeline. Anyways it's preferable to run linting locally before push:
     ```
     cargo fix --allow-staged --allow-dirty --all-targets --all-features; cargo fmt; cargo clippy --workspace --all-targets --all-features -- -D warnings
     ```
@@ -107,4 +107,4 @@ We use [semver](https://semver.org/) naming convention.
  
 
 ## Any contributions you make will be under the MIT Software License
-In short, when you submit code changes, your submissions are understood to be under the same [MIT License](http://choosealicense.com/licenses/mit/) that covers the project. Feel free to contact the maintainers if that's a concern.
+In short, when you submit code changes, your submissions are understood to be under the same [MIT License](http://choosealicense.com/licenses/mit/) that covers the project. Feel free to contact the maintainers if that's a concern.

LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 Polygon (previously Matic)
+Copyright (c) 2023 Polygon (previously Matic)
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

Makefile

@@ -0,0 +1,5 @@
+exec:
+	cargo build --release --features concurrent,executable
+
+test:
+	RUSTFLAGS="-C debug-assertions -C overflow-checks -C debuginfo=2" cargo test --release --features internals

README.md

@@ -1,4 +1,4 @@
-# Polygon Miden
+# Miden Virtual Machine
 
 <a href="https://github.com/0xPolygonMiden/miden-vm/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-MIT-blue.svg"></a>
 <img src="https://github.com/0xPolygonMiden/miden-vm/workflows/CI/badge.svg?branch=main">
@@ -16,7 +16,7 @@ Miden VM is a zero-knowledge virtual machine written in Rust. For any program ex
 * If you'd like to learn more about STARKs, check out the [references](#references) section.
 
 ### Status and features
-Miden VM is currently on release v0.3. In this release, most of the core features of the VM have been stabilized, and most of the STARK proof generation has been implemented. While we expect to keep making changes to the VM internals, the external interfaces should remain relatively stable, and we will do our best to minimize the amount of breaking changes going forward.
+Miden VM is currently on release v0.4. In this release, most of the core features of the VM have been stabilized, and most of the STARK proof generation has been implemented. While we expect to keep making changes to the VM internals, the external interfaces should remain relatively stable, and we will do our best to minimize the amount of breaking changes going forward.
 
 The next version of the VM is being developed in the [next](https://github.com/0xPolygonMiden/miden-vm/tree/next) branch. There is also a documentation for the latest features and changes in the next branch [documentation next branch](https://0xpolygonmiden.github.io/miden-vm/intro/main.html).
 
@@ -28,15 +28,16 @@ Miden VM is a fully-featured virtual machine. Despite being optimized for zero-k
 * **Execution contexts.** Miden VM program execution can span multiple isolated contexts, each with its own dedicated memory space. The contexts are separated into the *root context* and *user contexts*. The root context can be accessed from user contexts via customizable kernel calls.
 * **Memory.** Miden VM supports read-write random-access memory. Procedures can reserve portions of global memory for easier management of local variables.
 * **u32 operations.** Miden VM supports native operations with 32-bit unsigned integers. This includes basic arithmetic, comparison, and bitwise operations.
-* **Cryptographic operations.** Miden assembly provides built-in instructions for computing hashes and verifying Merkle paths. These instructions use Rescue Prime hash function (which is the native hash function of the VM).
-* **Standard library.** Miden VM ships with a standard library which expands the core functionality of the VM (e.g., by adding support for 64-bit unsigned integers). Currently, the standard library is quite limited, but we plan to expand it significantly in the future.
+* **Cryptographic operations.** Miden assembly provides built-in instructions for computing hashes and verifying Merkle paths. These instructions use the Rescue Prime Optimized hash function (which is the native hash function of the VM).
+* **External libraries.** Miden VM supports compiling programs against pre-defined libraries. The VM ships with one such library: Miden `stdlib` which adds support for such things as 64-bit unsigned integers. Developers can build other similar libraries to extend the VM's functionality in ways which fit their use cases.
 * **Nondeterminism**. Unlike traditional virtual machines, Miden VM supports nondeterministic programming. This means a prover may do additional work outside of the VM and then provide execution *hints* to the VM. These hints can be used to dramatically speed up certain types of computations, as well as to supply secret inputs to the VM.
+* **Custom advice providers.** Miden VM can be instantiated with user-defined advice providers. These advice providers are used to supply external data to the VM during execution/proof generation (via nondeterministic inputs) and can connect the VM to arbitrary data sources (e.g., a database or RPC calls).
 
 #### Planned features
 In the coming months we plan to finalize the design of the VM and implement support for the following features:
 
-* **Custom advice providers.** It will be possible to instantiate the VM with custom advice providers. These providers can be used to supply external data to the VM (e.g., from a database or RPC calls).
-* **User-provided libraries.** It will be possible to compile Miden VM programs against arbitrary 3rd-party libraries (not just Miden `stdlib`). Together with execution context isolation and custom advice providers, this will enable flexible ways to extend the VM with core features such as persistent storage.
+* **Recursive proofs.** Miden VM will soon be able to verify a proof of its own execution. This will enable infinitely recursive proofs, an extremely useful tool for real-world applications.
+* **Better debugging.** Miden VM will provide a better debugging experience including the ability to place breakpoints, better source mapping, and more complete program analysis info.
 * **Faulty execution.** Miden VM will support generating proofs for programs with faulty execution (a notoriously complex task in ZK context). That is, it will be possible to prove that execution of some program resulted in an error.
 
 #### Compilation to WebAssembly.

air/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "miden-air"
-version = "0.3.0"
+version = "0.4.0"
 description = "Algebraic intermediate representation of Miden VM processor"
 authors = ["miden contributors"]
 readme = "README.md"
@@ -9,7 +9,7 @@ repository = "https://github.com/0xPolygonMiden/miden-vm"
 categories = ["cryptography", "no-std"]
 keywords = ["air", "arithmetization", "crypto", "miden"]
 edition = "2021"
-rust-version = "1.62"
+rust-version = "1.67"
 
 [lib]
 bench = false
@@ -28,10 +28,10 @@ default = ["std"]
 std = ["vm-core/std", "winter-air/std"]
 
 [dependencies]
-vm-core = { package = "miden-core", path = "../core", version = "0.3", default-features = false }
-winter-air = { package = "winter-air", version = "0.4.2", default-features = false }
+vm-core = { package = "miden-core", path = "../core", version = "0.4", default-features = false }
+winter-air = { package = "winter-air", version = "0.5", default-features = false }
 
 [dev-dependencies]
 criterion = "0.4"
 proptest = "1.0"
-rand-utils = { package = "winter-rand-utils", version = "0.4.2" }
+rand-utils = { package = "winter-rand-utils", version = "0.5" }

air/src/chiplets/bitwise/mod.rs

@@ -137,11 +137,7 @@ fn enforce_input_decomposition<E: FieldElement>(
     constraint_offset += NUM_DECOMP_BITS;
 
     // Values in bit decomposition columns b0..b3 should be binary.
-    for (idx, result) in result[constraint_offset..]
-        .iter_mut()
-        .take(NUM_DECOMP_BITS)
-        .enumerate()
-    {
+    for (idx, result) in result[constraint_offset..].iter_mut().take(NUM_DECOMP_BITS).enumerate() {
         *result = processor_flag * is_binary(frame.b_bit(idx));
     }
     constraint_offset += NUM_DECOMP_BITS;
@@ -307,46 +303,57 @@ impl<E: FieldElement> EvaluationFrameExt<E> for &EvaluationFrame<E> {
     fn selector(&self) -> E {
         self.current()[BITWISE_SELECTOR_COL_IDX]
     }
+
     #[inline(always)]
     fn selector_next(&self) -> E {
         self.next()[BITWISE_SELECTOR_COL_IDX]
     }
+
     #[inline(always)]
     fn a(&self) -> E {
         self.current()[BITWISE_A_COL_IDX]
     }
+
     #[inline(always)]
     fn a_next(&self) -> E {
         self.next()[BITWISE_A_COL_IDX]
     }
+
     #[inline(always)]
     fn a_bit(&self, index: usize) -> E {
         self.current()[BITWISE_A_COL_RANGE.start + index]
     }
+
     #[inline(always)]
     fn b(&self) -> E {
         self.current()[BITWISE_B_COL_IDX]
     }
+
     #[inline(always)]
     fn b_next(&self) -> E {
         self.next()[BITWISE_B_COL_IDX]
     }
+
     #[inline(always)]
     fn b_bit(&self, index: usize) -> E {
         self.current()[BITWISE_B_COL_RANGE.start + index]
     }
+
     #[inline(always)]
     fn bit_decomp(&self) -> &[E] {
         &self.current()[BITWISE_A_COL_RANGE.start..BITWISE_B_COL_RANGE.end]
     }
+
     #[inline(always)]
     fn output_prev(&self) -> E {
         self.current()[BITWISE_PREV_OUTPUT_COL_IDX]
     }
+
     #[inline(always)]
     fn output_prev_next(&self) -> E {
         self.next()[BITWISE_PREV_OUTPUT_COL_IDX]
     }
+
     #[inline(always)]
     fn output(&self) -> E {
         self.current()[BITWISE_OUTPUT_COL_IDX]
@@ -357,14 +364,17 @@ impl<E: FieldElement> EvaluationFrameExt<E> for &EvaluationFrame<E> {
     fn a_agg_bits(&self) -> E {
         agg_bits(self.current(), BITWISE_A_COL_RANGE.start)
     }
+
     #[inline(always)]
     fn a_agg_bits_next(&self) -> E {
         agg_bits(self.next(), BITWISE_A_COL_RANGE.start)
     }
+
     #[inline(always)]
     fn b_agg_bits(&self) -> E {
         agg_bits(self.current(), BITWISE_B_COL_RANGE.start)
     }
+
     #[inline(always)]
     fn b_agg_bits_next(&self) -> E {
         agg_bits(self.next(), BITWISE_B_COL_RANGE.start)
@@ -376,6 +386,7 @@ impl<E: FieldElement> EvaluationFrameExt<E> for &EvaluationFrame<E> {
     fn bitwise_and_flag(&self) -> E {
         binary_not(self.current()[BITWISE_SELECTOR_COL_IDX])
     }
+
     #[inline(always)]
     fn bitwise_xor_flag(&self) -> E {
         self.current()[BITWISE_SELECTOR_COL_IDX]