Merge pull request #2605 from OffchainLabs/cross-compile

support cross-compilation of stylus programs

arbnode/inbox_test.go

@@ -72,7 +72,9 @@ func NewTransactionStreamerForTest(t *testing.T, ownerAddress common.Address) (*
 	if err != nil {
 		Fail(t, err)
 	}
-	if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, &gethexec.DefaultStylusTargetConfig); err != nil {
+	stylusTargetConfig := &gethexec.DefaultStylusTargetConfig
+	Require(t, stylusTargetConfig.Validate()) // pre-processes config (i.a. parses wasmTargets)
+	if err := execEngine.Initialize(gethexec.DefaultCachingConfig.StylusLRUCache, stylusTargetConfig); err != nil {
 		Fail(t, err)
 	}
 	execSeq := &execClientWrapper{execEngine, t}

arbos/programs/native.go

@@ -27,6 +27,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/vm"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/offchainlabs/nitro/arbos/burn"
 	"github.com/offchainlabs/nitro/arbos/util"
@@ -71,7 +72,7 @@ func activateProgramInternal(
 	version uint16,
 	debug bool,
 	gasLeft *uint64,
-) (*activationInfo, map[rawdb.Target][]byte, error) {
+) (*activationInfo, map[ethdb.WasmTarget][]byte, error) {
 	output := &rustBytes{}
 	moduleHash := &bytes32{}
 	stylusData := &C.StylusData{}
@@ -99,25 +100,50 @@ func activateProgramInternal(
 		}
 		return nil, nil, err
 	}
-	target := rawdb.LocalTarget()
-	status_asm := C.stylus_compile(
-		goSlice(wasm),
-		u16(version),
-		cbool(debug),
-		goSlice([]byte(target)),
-		output,
-	)
-	asm := output.intoBytes()
-	if status_asm != 0 {
-		return nil, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))
+	targets := db.Database().WasmTargets()
+	type result struct {
+		target ethdb.WasmTarget
+		asm    []byte
+		err    error
 	}
-	asmMap := map[rawdb.Target][]byte{
-		rawdb.TargetWavm: module,
-		target:           asm,
+	results := make(chan result, len(targets))
+	for _, target := range targets {
+		if target == rawdb.TargetWavm {
+			results <- result{target, module, nil}
+		} else {
+			target := target
+			go func() {
+				output := &rustBytes{}
+				status_asm := C.stylus_compile(
+					goSlice(wasm),
+					u16(version),
+					cbool(debug),
+					goSlice([]byte(target)),
+					output,
+				)
+				asm := output.intoBytes()
+				if status_asm != 0 {
+					results <- result{target, nil, fmt.Errorf("%w: %s", ErrProgramActivation, string(asm))}
+					return
+				}
+				results <- result{target, asm, nil}
+			}()
+		}
+	}
+	asmMap := make(map[ethdb.WasmTarget][]byte, len(targets))
+	for range targets {
+		res := <-results
+		if res.err != nil {
+			err = errors.Join(res.err, err)
+		} else {
+			asmMap[res.target] = res.asm
+		}
+	}
+	if err != nil {
+		return nil, nil, fmt.Errorf("compilation failed for one or more targets: %w", err)
 	}
 
 	hash := moduleHash.toHash()
-
 	info := &activationInfo{
 		moduleHash:    hash,
 		initGas:       uint16(stylusData.init_cost),
@@ -171,7 +197,7 @@ func getLocalAsm(statedb vm.StateDB, moduleHash common.Hash, addressForLogging c
 	}
 	asm, exists := asmMap[localTarget]
 	if !exists {
-		var availableTargets []rawdb.Target
+		var availableTargets []ethdb.WasmTarget
 		for target := range asmMap {
 			availableTargets = append(availableTargets, target)
 		}
@@ -202,12 +228,8 @@ func callProgram(
 		panic("missing asm")
 	}
 
-	if db, ok := db.(*state.StateDB); ok {
-		targets := []rawdb.Target{
-			rawdb.TargetWavm,
-			rawdb.LocalTarget(),
-		}
-		db.RecordProgram(targets, moduleHash)
+	if stateDb, ok := db.(*state.StateDB); ok {
+		stateDb.RecordProgram(db.Database().WasmTargets(), moduleHash)
 	}
 
 	evmApi := newApi(interpreter, tracingInfo, scope, memoryModel)
@@ -291,7 +313,7 @@ func ResizeWasmLruCache(size uint32) {
 const DefaultTargetDescriptionArm = "arm64-linux-unknown+neon"
 const DefaultTargetDescriptionX86 = "x86_64-linux-unknown+sse4.2+lzcnt+bmi"
 
-func SetTarget(name rawdb.Target, description string, native bool) error {
+func SetTarget(name ethdb.WasmTarget, description string, native bool) error {
 	output := &rustBytes{}
 	status := userStatus(C.stylus_target_set(
 		goSlice([]byte(name)),

arbos/programs/wasmstorehelper.go

@@ -43,8 +43,9 @@ func (p Programs) SaveActiveProgramToWasmStore(statedb *state.StateDB, codeHash
 		return err
 	}
 
+	targets := statedb.Database().WasmTargets()
 	// If already in wasm store then return early
-	_, err = statedb.TryGetActivatedAsmMap([]rawdb.Target{rawdb.TargetWavm, rawdb.LocalTarget()}, moduleHash)
+	_, err = statedb.TryGetActivatedAsmMap(targets, moduleHash)
 	if err == nil {
 		return nil
 	}

cmd/nitro/init.go

@@ -560,7 +560,7 @@ func rebuildLocalWasm(ctx context.Context, config *gethexec.Config, l2BlockChain
 	return chainDb, l2BlockChain, nil
 }
 
-func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) {
+func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeConfig, chainId *big.Int, cacheConfig *core.CacheConfig, targetConfig *gethexec.StylusTargetConfig, persistentConfig *conf.PersistentConfig, l1Client arbutil.L1Interface, rollupAddrs chaininfo.RollupAddresses) (ethdb.Database, *core.BlockChain, error) {
 	if !config.Init.Force {
 		if readOnlyDb, err := stack.OpenDatabaseWithFreezerWithExtraOptions("l2chaindata", 0, 0, config.Persistent.Ancient, "l2chaindata/", true, persistentConfig.Pebble.ExtraOptions("l2chaindata")); err == nil {
 			if chainConfig := gethexec.TryReadStoredChainConfig(readOnlyDb); chainConfig != nil {
@@ -585,7 +585,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo
 				if err := dbutil.UnfinishedConversionCheck(wasmDb); err != nil {
 					return nil, nil, fmt.Errorf("wasm unfinished database conversion check error: %w", err)
 				}
-				chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1)
+				chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets())
 				_, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb)
 				if err != nil {
 					return nil, nil, err
@@ -660,7 +660,7 @@ func openInitializeChainDb(ctx context.Context, stack *node.Node, config *NodeCo
 	if err := validateOrUpgradeWasmStoreSchemaVersion(wasmDb); err != nil {
 		return nil, nil, err
 	}
-	chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1)
+	chainDb := rawdb.WrapDatabaseWithWasm(chainData, wasmDb, 1, targetConfig.WasmTargets())
 	_, err = rawdb.ParseStateScheme(cacheConfig.StateScheme, chainDb)
 	if err != nil {
 		return nil, nil, err

cmd/nitro/init_test.go

@@ -354,6 +354,12 @@ func TestEmptyDatabaseDir(t *testing.T) {
 	}
 }
 
+func defaultStylusTargetConfigForTest(t *testing.T) *gethexec.StylusTargetConfig {
+	targetConfig := gethexec.DefaultStylusTargetConfig
+	Require(t, targetConfig.Validate())
+	return &targetConfig
+}
+
 func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) {
 	t.Parallel()
 
@@ -381,6 +387,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) {
 		&nodeConfig,
 		new(big.Int).SetUint64(nodeConfig.Chain.ID),
 		gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching),
+		defaultStylusTargetConfigForTest(t),
 		&nodeConfig.Persistent,
 		l1Client,
 		chaininfo.RollupAddresses{},
@@ -397,6 +404,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) {
 		&nodeConfig,
 		new(big.Int).SetUint64(nodeConfig.Chain.ID),
 		gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching),
+		defaultStylusTargetConfigForTest(t),
 		&nodeConfig.Persistent,
 		l1Client,
 		chaininfo.RollupAddresses{},
@@ -414,6 +422,7 @@ func TestOpenInitializeChainDbIncompatibleStateScheme(t *testing.T) {
 		&nodeConfig,
 		new(big.Int).SetUint64(nodeConfig.Chain.ID),
 		gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching),
+		defaultStylusTargetConfigForTest(t),
 		&nodeConfig.Persistent,
 		l1Client,
 		chaininfo.RollupAddresses{},
@@ -549,6 +558,7 @@ func TestOpenInitializeChainDbEmptyInit(t *testing.T) {
 		&nodeConfig,
 		new(big.Int).SetUint64(nodeConfig.Chain.ID),
 		gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching),
+		defaultStylusTargetConfigForTest(t),
 		&nodeConfig.Persistent,
 		l1Client,
 		chaininfo.RollupAddresses{},

cmd/nitro/nitro.go

@@ -495,7 +495,7 @@ func mainImpl() int {
 		}
 	}
 
-	chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Persistent, l1Client, rollupAddrs)
+	chainDb, l2BlockChain, err := openInitializeChainDb(ctx, stack, nodeConfig, new(big.Int).SetUint64(nodeConfig.Chain.ID), gethexec.DefaultCacheConfigFor(stack, &nodeConfig.Execution.Caching), &nodeConfig.Execution.StylusTarget, &nodeConfig.Persistent, l1Client, rollupAddrs)
 	if l2BlockChain != nil {
 		deferFuncs = append(deferFuncs, func() { l2BlockChain.Stop() })
 	}

execution/gethexec/executionengine.go

@@ -151,19 +151,33 @@ func (s *ExecutionEngine) MarkFeedStart(to arbutil.MessageIndex) {
 }
 
 func populateStylusTargetCache(targetConfig *StylusTargetConfig) error {
-	var effectiveStylusTarget string
-	target := rawdb.LocalTarget()
-	switch target {
-	case rawdb.TargetArm64:
-		effectiveStylusTarget = targetConfig.Arm64
-	case rawdb.TargetAmd64:
-		effectiveStylusTarget = targetConfig.Amd64
-	case rawdb.TargetHost:
-		effectiveStylusTarget = targetConfig.Host
-	}
-	err := programs.SetTarget(target, effectiveStylusTarget, true)
-	if err != nil {
-		return fmt.Errorf("Failed to set stylus target: %w", err)
+	localTarget := rawdb.LocalTarget()
+	targets := targetConfig.WasmTargets()
+	var nativeSet bool
+	for _, target := range targets {
+		var effectiveStylusTarget string
+		switch target {
+		case rawdb.TargetWavm:
+			// skip wavm target
+			continue
+		case rawdb.TargetArm64:
+			effectiveStylusTarget = targetConfig.Arm64
+		case rawdb.TargetAmd64:
+			effectiveStylusTarget = targetConfig.Amd64
+		case rawdb.TargetHost:
+			effectiveStylusTarget = targetConfig.Host
+		default:
+			return fmt.Errorf("unsupported stylus target: %v", target)
+		}
+		isNative := target == localTarget
+		err := programs.SetTarget(target, effectiveStylusTarget, isNative)
+		if err != nil {
+			return fmt.Errorf("failed to set stylus target: %w", err)
+		}
+		nativeSet = nativeSet || isNative
+	}
+	if !nativeSet {
+		return fmt.Errorf("local target %v missing in list of archs %v", localTarget, targets)
 	}
 	return nil
 }

execution/gethexec/node.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ethereum/go-ethereum/arbitrum"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
+	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/eth"
 	"github.com/ethereum/go-ethereum/eth/filters"
@@ -29,21 +30,51 @@ import (
 )
 
 type StylusTargetConfig struct {
-	Arm64 string `koanf:"arm64"`
-	Amd64 string `koanf:"amd64"`
-	Host  string `koanf:"host"`
+	Arm64      string   `koanf:"arm64"`
+	Amd64      string   `koanf:"amd64"`
+	Host       string   `koanf:"host"`
+	ExtraArchs []string `koanf:"extra-archs"`
+
+	wasmTargets []ethdb.WasmTarget
+}
+
+func (c *StylusTargetConfig) WasmTargets() []ethdb.WasmTarget {
+	return c.wasmTargets
+}
+
+func (c *StylusTargetConfig) Validate() error {
+	targetsSet := make(map[ethdb.WasmTarget]bool, len(c.ExtraArchs))
+	for _, arch := range c.ExtraArchs {
+		target := ethdb.WasmTarget(arch)
+		if !rawdb.IsSupportedWasmTarget(target) {
+			return fmt.Errorf("unsupported architecture: %v, possible values: %s, %s, %s, %s", arch, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost)
+		}
+		targetsSet[target] = true
+	}
+	if !targetsSet[rawdb.TargetWavm] {
+		return fmt.Errorf("%s target not found in archs list, archs: %v", rawdb.TargetWavm, c.ExtraArchs)
+	}
+	targetsSet[rawdb.LocalTarget()] = true
+	targets := make([]ethdb.WasmTarget, 0, len(c.ExtraArchs)+1)
+	for target := range targetsSet {
+		targets = append(targets, target)
+	}
+	c.wasmTargets = targets
+	return nil
 }
 
 var DefaultStylusTargetConfig = StylusTargetConfig{
-	Arm64: programs.DefaultTargetDescriptionArm,
-	Amd64: programs.DefaultTargetDescriptionX86,
-	Host:  "",
+	Arm64:      programs.DefaultTargetDescriptionArm,
+	Amd64:      programs.DefaultTargetDescriptionX86,
+	Host:       "",
+	ExtraArchs: []string{string(rawdb.TargetWavm)},
 }
 
 func StylusTargetConfigAddOptions(prefix string, f *flag.FlagSet) {
 	f.String(prefix+".arm64", DefaultStylusTargetConfig.Arm64, "stylus programs compilation target for arm64 linux")
 	f.String(prefix+".amd64", DefaultStylusTargetConfig.Amd64, "stylus programs compilation target for amd64 linux")
 	f.String(prefix+".host", DefaultStylusTargetConfig.Host, "stylus programs compilation target for system other than 64-bit ARM or 64-bit x86")
+	f.StringSlice(prefix+".extra-archs", DefaultStylusTargetConfig.ExtraArchs, fmt.Sprintf("Comma separated list of extra architectures to cross-compile stylus program to and cache in wasm store (additionally to local target). Currently must include at least %s. (supported targets: %s, %s, %s, %s)", rawdb.TargetWavm, rawdb.TargetWavm, rawdb.TargetArm64, rawdb.TargetAmd64, rawdb.TargetHost))
 }
 
 type Config struct {
@@ -82,6 +113,9 @@ func (c *Config) Validate() error {
 	if c.forwardingTarget != "" && c.Sequencer.Enable {
 		return errors.New("ForwardingTarget set and sequencer enabled")
 	}
+	if err := c.StylusTarget.Validate(); err != nil {
+		return err
+	}
 	return nil
 }
 

staker/block_validator.go

@@ -500,7 +500,7 @@ func (v *BlockValidator) sendRecord(s *validationStatus) error {
 
 //nolint:gosec
 func (v *BlockValidator) writeToFile(validationEntry *validationEntry, moduleRoot common.Hash) error {
-	input, err := validationEntry.ToInput([]rawdb.Target{rawdb.TargetWavm})
+	input, err := validationEntry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm})
 	if err != nil {
 		return err
 	}

staker/challenge_manager.go

@@ -16,6 +16,7 @@ import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/offchainlabs/nitro/arbutil"
@@ -468,7 +469,7 @@ func (m *ChallengeManager) createExecutionBackend(ctx context.Context, step uint
 	if err != nil {
 		return fmt.Errorf("error creating validation entry for challenge %v msg %v for execution challenge: %w", m.challengeIndex, initialCount, err)
 	}
-	input, err := entry.ToInput([]rawdb.Target{rawdb.TargetWavm})
+	input, err := entry.ToInput([]ethdb.WasmTarget{rawdb.TargetWavm})
 	if err != nil {
 		return fmt.Errorf("error getting validation entry input of challenge %v msg %v: %w", m.challengeIndex, initialCount, err)
 	}

staker/stateless_block_validator.go

@@ -12,7 +12,6 @@ import (
 	"github.com/offchainlabs/nitro/arbstate/daprovider"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/log"
@@ -134,7 +133,7 @@ type validationEntry struct {
 	DelayedMsg []byte
 }
 
-func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.ValidationInput, error) {
+func (e *validationEntry) ToInput(stylusArchs []ethdb.WasmTarget) (*validator.ValidationInput, error) {
 	if e.Stage != Ready {
 		return nil, errors.New("cannot create input from non-ready entry")
 	}
@@ -143,7 +142,7 @@ func (e *validationEntry) ToInput(stylusArchs []rawdb.Target) (*validator.Valida
 		HasDelayedMsg: e.HasDelayedMsg,
 		DelayedMsgNr:  e.DelayedMsgNr,
 		Preimages:     e.Preimages,
-		UserWasms:     make(map[rawdb.Target]map[common.Hash][]byte, len(e.UserWasms)),
+		UserWasms:     make(map[ethdb.WasmTarget]map[common.Hash][]byte, len(e.UserWasms)),
 		BatchInfo:     e.BatchInfo,
 		DelayedMsg:    e.DelayedMsg,
 		StartState:    e.Start,