Merge branch 'master' into feat/das-google-cloud

OffchainLabs · Oct 2, 2024 · 2644a8e · 2644a8e
2 parents bee2534 + 7a3d6e6
commit 2644a8e
Show file tree

Hide file tree

Showing 135 changed files with 2,483 additions and 1,921 deletions.
diff --git a/.github/workflows/arbitrator-ci.yml b/.github/workflows/arbitrator-ci.yml
@@ -50,15 +50,13 @@ jobs:
       - name: Install go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.21.x
+          go-version: 1.23.x
 
       - name: Install custom go-ethereum
         run: |
           cd /tmp
-          git clone --branch v1.13.8 --depth 1 https://github.com/ethereum/go-ethereum.git
+          git clone --branch v1.14.11 --depth 1 https://github.com/ethereum/go-ethereum.git
           cd go-ethereum
-          # Enable KZG point evaluation precompile early
-          sed -i 's#var PrecompiledContractsBerlin = map\[common.Address\]PrecompiledContract{#\0 common.BytesToAddress([]byte{0x0a}): \&kzgPointEvaluation{},#g' core/vm/contracts.go
           go build -o /usr/local/bin/geth ./cmd/geth
 
       - name: Setup nodejs

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -46,7 +46,7 @@ jobs:
       - name: Install go
         uses: actions/setup-go@v4
         with:
-          go-version: 1.21.x
+          go-version: 1.23.x
 
       - name: Install wasm-ld
         run: |
@@ -87,12 +87,12 @@ jobs:
         uses: actions/cache@v3
         with:
           path: |
-            ~/.cargo/registry/
-            ~/.cargo/git/
+            ~/.cargo/
             arbitrator/target/
             arbitrator/wasm-libraries/target/
-            arbitrator/wasm-libraries/soft-float/SoftFloat/build
+            arbitrator/wasm-libraries/soft-float/
             target/etc/initial-machine-cache/
+            /home/runner/.rustup/toolchains/
           key: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-min-${{ hashFiles('arbitrator/Cargo.lock') }}-${{ matrix.test-mode }}
           restore-keys: ${{ runner.os }}-cargo-${{ steps.install-rust.outputs.rustc_hash }}-
 
@@ -145,89 +145,42 @@ jobs:
         env:
           TEST_STATE_SCHEME: path
         run: |
-          packages=`go list ./...`
-          for package in $packages; do
-            echo running tests for $package
-            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -timeout 20m -tags=cionly > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
-              exit 1
-            fi
-          done
+          echo "Running tests with Path Scheme" >> full.log
+          ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m --cover
 
       - name: run tests without race detection and hash state scheme
         if: matrix.test-mode == 'defaults'
         env:
           TEST_STATE_SCHEME: hash
         run: |
-          packages=`go list ./...`
-          for package in $packages; do
-            echo running tests for $package
-            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 20m -tags=cionly; then
-              exit 1
-            fi
-          done
-
-      - name: run tests with race detection and path state scheme
-        if: matrix.test-mode == 'race'
-        env:
-          TEST_STATE_SCHEME: path
-        run: |
-          packages=`go list ./...`
-          for package in $packages; do
-            echo running tests for $package
-            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
-              exit 1
-            fi
-          done
+          echo "Running tests with Hash Scheme" >> full.log
+          ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags cionly --timeout 20m
 
       - name: run tests with race detection and hash state scheme
         if: matrix.test-mode == 'race'
         env:
           TEST_STATE_SCHEME: hash
         run: |
-          packages=`go list ./...`
-          for package in $packages; do
-            echo running tests for $package
-            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -race -timeout 30m; then
-              exit 1
-            fi
-          done
+          echo "Running tests with Hash Scheme" >> full.log
+          ${{ github.workspace }}/.github/workflows/gotestsum.sh --race --timeout 30m
 
       - name: run redis tests
         if: matrix.test-mode == 'defaults'
-        run: TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./...
+        run: |
+          echo "Running redis tests" >> full.log
+          TEST_REDIS=redis://localhost:6379/0 gotestsum --format short-verbose -- -p 1 -run TestRedis ./arbnode/... ./system_tests/... -coverprofile=coverage-redis.txt -covermode=atomic -coverpkg=./...
 
       - name: run challenge tests
         if: matrix.test-mode == 'challenge'
-        run: |
-          packages=`go list ./...`
-          for package in $packages; do
-            echo running tests for $package
-            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=challengetest -run=TestChallenge > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
-              exit 1
-            fi
-          done
+        run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags challengetest --run TestChallenge --cover
 
       - name: run stylus tests
         if: matrix.test-mode == 'stylus'
-        run: |
-          packages=`go list ./...`
-          for package in $packages; do
-            echo running tests for $package
-            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramArbitrator" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
-              exit 1
-            fi
-          done
+        run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramArbitrator --timeout 60m --cover
 
       - name: run long stylus tests
         if: matrix.test-mode == 'long'
-        run: |
-          packages=`go list ./...`
-          for package in $packages; do
-            echo running tests for $package
-            if ! stdbuf -oL gotestsum --format short-verbose --packages="$package" --rerun-fails=2 --no-color=false -- -timeout 60m -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/... -tags=stylustest -run="TestProgramLong" > >(stdbuf -oL tee -a full.log | grep -vE "INFO|seal"); then
-              exit 1
-            fi
-          done
+        run: ${{ github.workspace }}/.github/workflows/gotestsum.sh --tags stylustest --run TestProgramLong --timeout 60m --cover
 
       - name: Archive detailed run log
         uses: actions/upload-artifact@v3

diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -73,7 +73,7 @@ jobs:
     - name: Install go
       uses: actions/setup-go@v4
       with:
-        go-version: 1.21.x
+        go-version: 1.23.x
 
     - name: Install rust stable
       uses: dtolnay/rust-toolchain@stable

diff --git a/.github/workflows/gotestsum.sh b/.github/workflows/gotestsum.sh
@@ -0,0 +1,83 @@
+#!/bin/bash
+
+check_missing_value() {
+    if [[ $1 -eq 0 || $2 == -* ]]; then
+        echo "missing $3 argument value"
+        exit 1
+    fi
+}
+
+timeout=""
+tags=""
+run=""
+race=false
+cover=false
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --timeout)
+      shift
+      check_missing_value $# "$1" "--timeout"
+      timeout=$1
+      shift
+      ;;
+    --tags)
+      shift
+      check_missing_value $# "$1" "--tags"
+      tags=$1
+      shift
+      ;;
+    --run)
+      shift
+      check_missing_value $# "$1" "--run"
+      run=$1
+      shift
+      ;;
+    --race)
+      race=true
+      shift
+      ;;
+    --cover)
+      cover=true
+      shift
+      ;;
+    *)
+      echo "Invalid argument: $1"
+      exit 1
+      ;;
+  esac
+done
+
+packages=$(go list ./...)
+for package in $packages; do
+  cmd="stdbuf -oL gotestsum --format short-verbose --packages=\"$package\" --rerun-fails=2 --no-color=false --"
+
+  if [ "$timeout" != "" ]; then
+    cmd="$cmd -timeout $timeout"
+  fi
+
+  if [ "$tags" != "" ]; then
+    cmd="$cmd -tags=$tags"
+  fi
+
+  if [ "$run" != "" ]; then
+    cmd="$cmd -run=$run"
+  fi
+
+  if [ "$race" == true ]; then
+    cmd="$cmd -race"
+  fi
+
+  if [ "$cover" == true ]; then
+    cmd="$cmd -coverprofile=coverage.txt -covermode=atomic -coverpkg=./...,./go-ethereum/..."
+  fi
+
+  cmd="$cmd > >(stdbuf -oL tee -a full.log | grep -vE \"INFO|seal\")"
+
+  echo ""
+  echo running tests for "$package"
+  echo "$cmd"
+
+  if ! eval "$cmd"; then
+    exit 1
+  fi
+done
diff --git a/Dockerfile b/Dockerfile
@@ -66,7 +66,7 @@ COPY --from=wasm-libs-builder /workspace/ /
 
 FROM wasm-base AS wasm-bin-builder
 # pinned go version
-RUN curl -L https://golang.org/dl/go1.21.10.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf -
+RUN curl -L https://golang.org/dl/go1.23.1.linux-`dpkg --print-architecture`.tar.gz | tar -C /usr/local -xzf -
 COPY ./Makefile ./go.mod ./go.sum ./
 COPY ./arbcompress ./arbcompress
 COPY ./arbos ./arbos
@@ -218,8 +218,9 @@ COPY ./scripts/download-machine.sh .
 #RUN ./download-machine.sh consensus-v20 0x8b104a2e80ac6165dc58b9048de12f301d70b02a0ab51396c22b4b4b802a16a4
 RUN ./download-machine.sh consensus-v30 0xb0de9cb89e4d944ae6023a3b62276e54804c242fd8c4c2d8e6cc4450f5fa8b1b && true
 RUN ./download-machine.sh consensus-v31 0x260f5fa5c3176a856893642e149cf128b5a8de9f828afec8d11184415dd8dc69
+RUN ./download-machine.sh consensus-v32 0x184884e1eb9fefdc158f6c8ac912bb183bf3cf83f0090317e0bc4ac5860baa39
 
-FROM golang:1.21.10-bookworm AS node-builder
+FROM golang:1.23.1-bookworm AS node-builder
 WORKDIR /workspace
 ARG version=""
 ARG datetime=""

diff --git a/LICENSE.md b/LICENSE.md
@@ -22,7 +22,7 @@ Additional Use Grant: You may use the Licensed Work in a production environment
                       Expansion Program Term of Use](https://docs.arbitrum.foundation/assets/files/Arbitrum%20Expansion%20Program%20Jan182024-4f08b0c2cb476a55dc153380fa3e64b0.pdf). For purposes of this
                       Additional Use Grant, the "Covered Arbitrum Chains" are 
                       (a) Arbitrum One (chainid:42161), Arbitrum Nova (chainid:42170), 
-                      rbitrum Rinkeby testnet/Rinkarby (chainid:421611),Arbitrum Nitro
+                      Arbitrum Rinkeby testnet/Rinkarby (chainid:421611),Arbitrum Nitro
                       Goerli testnet (chainid:421613), and Arbitrum Sepolia Testnet
                       (chainid:421614); (b) any future blockchains authorized to be
                       designated as Covered Arbitrum Chains by the decentralized autonomous

diff --git a/Makefile b/Makefile
@@ -60,7 +60,7 @@ replay_wasm=$(output_latest)/replay.wasm
 arb_brotli_files = $(wildcard arbitrator/brotli/src/*.* arbitrator/brotli/src/*/*.* arbitrator/brotli/*.toml arbitrator/brotli/*.rs) .make/cbrotli-lib .make/cbrotli-wasm
 
 arbitrator_generated_header=$(output_root)/include/arbitrator.h
-arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, wasi_stub host_io soft-float arbcompress user_host program_exec)
+arbitrator_wasm_libs=$(patsubst %, $(output_root)/machines/latest/%.wasm, forward wasi_stub host_io soft-float arbcompress user_host program_exec)
 arbitrator_stylus_lib=$(output_root)/lib/libstylus.a
 prover_bin=$(output_root)/bin/prover
 arbitrator_jit=$(output_root)/bin/jit
@@ -75,22 +75,29 @@ arbitrator_test_wasms=$(patsubst %.wat,%.wasm, $(arbitrator_tests_wat)) $(patsub
 arbitrator_tests_link_info = $(shell cat $(arbitrator_cases)/link.txt | xargs)
 arbitrator_tests_link_deps = $(patsubst %,$(arbitrator_cases)/%.wasm, $(arbitrator_tests_link_info))
 
+arbitrator_tests_forward_wats = $(wildcard $(arbitrator_cases)/forward/*.wat)
+arbitrator_tests_forward_deps = $(arbitrator_tests_forward_wats:wat=wasm)
+
 WASI_SYSROOT?=/opt/wasi-sdk/wasi-sysroot
 
 arbitrator_wasm_lib_flags=$(patsubst %, -l %, $(arbitrator_wasm_libs))
 
 rust_arbutil_files = $(wildcard arbitrator/arbutil/src/*.* arbitrator/arbutil/src/*/*.* arbitrator/arbutil/*.toml arbitrator/caller-env/src/*.* arbitrator/caller-env/src/*/*.* arbitrator/caller-env/*.toml) .make/cbrotli-lib
 
+prover_direct_includes = $(patsubst %,$(output_latest)/%.wasm, forward forward_stub)
 prover_dir = arbitrator/prover/
 rust_prover_files = $(wildcard $(prover_dir)/src/*.* $(prover_dir)/src/*/*.* $(prover_dir)/*.toml $(prover_dir)/*.rs) $(rust_arbutil_files) $(prover_direct_includes) $(arb_brotli_files)
 
 wasm_lib = arbitrator/wasm-libraries
 wasm_lib_cargo = $(wasm_lib)/.cargo/config.toml
 wasm_lib_deps = $(wildcard $(wasm_lib)/$(1)/*.toml $(wasm_lib)/$(1)/src/*.rs $(wasm_lib)/$(1)/*.rs) $(wasm_lib_cargo) $(rust_arbutil_files) $(arb_brotli_files) .make/machines
 wasm_lib_go_abi = $(call wasm_lib_deps,go-abi)
+wasm_lib_forward = $(call wasm_lib_deps,forward)
 wasm_lib_user_host_trait = $(call wasm_lib_deps,user-host-trait)
 wasm_lib_user_host = $(call wasm_lib_deps,user-host) $(wasm_lib_user_host_trait)
 
+forward_dir = $(wasm_lib)/forward
+
 stylus_files = $(wildcard $(stylus_dir)/*.toml $(stylus_dir)/src/*.rs) $(wasm_lib_user_host_trait) $(rust_prover_files)
 
 jit_dir = arbitrator/jit
@@ -274,7 +281,9 @@ clean:
 	rm -f arbitrator/wasm-libraries/soft-float/*.o
 	rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.o
 	rm -f arbitrator/wasm-libraries/soft-float/SoftFloat/build/Wasm-Clang/*.a
+	rm -f arbitrator/wasm-libraries/forward/*.wat
 	rm -rf arbitrator/stylus/tests/*/target/ arbitrator/stylus/tests/*/*.wasm
+	rm -rf brotli/buildfiles
 	@rm -rf contracts/build contracts/cache solgen/go/
 	@rm -f .make/*
 
@@ -391,7 +400,7 @@ $(output_latest)/host_io.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,host-io) $(
 	cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package host-io
 	install arbitrator/wasm-libraries/$(wasm32_wasi)/host_io.wasm $@
 
-$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) .make/machines
+$(output_latest)/user_host.wasm: $(DEP_PREDICATE) $(wasm_lib_user_host) $(rust_prover_files) $(output_latest)/forward_stub.wasm .make/machines
 	cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package user-host
 	install arbitrator/wasm-libraries/$(wasm32_wasi)/user_host.wasm $@
 
@@ -407,9 +416,17 @@ $(output_latest)/arbcompress.wasm: $(DEP_PREDICATE) $(call wasm_lib_deps,brotli)
 	cargo build --manifest-path arbitrator/wasm-libraries/Cargo.toml --release --target wasm32-wasi --config $(wasm_lib_cargo) --package arbcompress
 	install arbitrator/wasm-libraries/$(wasm32_wasi)/arbcompress.wasm $@
 
+$(output_latest)/forward.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines
+	cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward.wat
+	wat2wasm $(wasm_lib)/forward/forward.wat -o $@
+
+$(output_latest)/forward_stub.wasm: $(DEP_PREDICATE) $(wasm_lib_forward) .make/machines
+	cargo run --manifest-path $(forward_dir)/Cargo.toml -- --path $(forward_dir)/forward_stub.wat --stub
+	wat2wasm $(wasm_lib)/forward/forward_stub.wat -o $@
+
 $(output_latest)/machine.wavm.br: $(DEP_PREDICATE) $(prover_bin) $(arbitrator_wasm_libs) $(replay_wasm)
-	$(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) --with-forwarder \
-	$(patsubst %,-l $(output_latest)/%.wasm, soft-float wasi_stub host_io user_host arbcompress program_exec)
+	$(prover_bin) $(replay_wasm) --generate-binaries $(output_latest) \
+	$(patsubst %,-l $(output_latest)/%.wasm, forward soft-float wasi_stub host_io user_host arbcompress program_exec)
 
 $(arbitrator_cases)/%.wasm: $(arbitrator_cases)/%.wat
 	wat2wasm $< -o $@
@@ -476,10 +493,10 @@ target/testdata/preimages.bin:
 	python3 scripts/create-test-preimages.py $@
 
 contracts/test/prover/proofs/rust-%.json: $(arbitrator_cases)/rust/$(wasm32_wasi)/%.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin
-	$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin --with-forwarder
+	$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --allow-hostapi --require-success --inbox-add-stub-headers --inbox $(arbitrator_cases)/rust/data/msg0.bin --inbox $(arbitrator_cases)/rust/data/msg1.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg0.bin --delayed-inbox $(arbitrator_cases)/rust/data/msg1.bin --preimages target/testdata/preimages.bin
 
 contracts/test/prover/proofs/go.json: $(arbitrator_cases)/go/testcase.wasm $(prover_bin) $(arbitrator_wasm_libs) target/testdata/preimages.bin $(arbitrator_tests_link_deps) $(arbitrator_cases)/user.wasm
-	$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin --stylus-modules $(arbitrator_cases)/user.wasm --with-forwarder
+	$(prover_bin) $< $(arbitrator_wasm_lib_flags) -o $@ -b --require-success --preimages target/testdata/preimages.bin  --stylus-modules $(arbitrator_cases)/user.wasm
 
 # avoid testing user.wasm in onestepproofs. It can only run as stylus program.
 contracts/test/prover/proofs/user.json:
@@ -492,6 +509,9 @@ contracts/test/prover/proofs/read-inboxmsg-10.json:
 contracts/test/prover/proofs/global-state.json:
 	echo "[]" > $@
 
+contracts/test/prover/proofs/forward-test.json: $(arbitrator_cases)/forward-test.wasm $(arbitrator_tests_forward_deps) $(prover_bin)
+	$(prover_bin) $< -o $@ --allow-hostapi $(patsubst %,-l %, $(arbitrator_tests_forward_deps))
+
 contracts/test/prover/proofs/link.json: $(arbitrator_cases)/link.wasm $(arbitrator_tests_link_deps) $(prover_bin)
 	$(prover_bin) $< -o $@ --allow-hostapi --stylus-modules $(arbitrator_tests_link_deps) --require-success
 

diff --git a/arbcompress/native.go b/arbcompress/native.go
@@ -7,7 +7,7 @@
 package arbcompress
 
 /*
-#cgo CFLAGS: -g -Wall -I${SRCDIR}/../target/include/
+#cgo CFLAGS: -g -I${SRCDIR}/../target/include/
 #cgo LDFLAGS: ${SRCDIR}/../target/lib/libstylus.a -lm
 #include "arbitrator.h"
 */