Enables Out of Order Execution for E2E Tests

.github/workflows/ccip-load-tests.yml

@@ -14,7 +14,6 @@ on:
         description: 'Key to run tests with custom test secrets'
         required: false
         type: string        
-  pull_request: # DEBUG: Checking run conditions
 
 # Only run 1 of this workflow at a time per PR
 concurrency:

.github/workflows/integration-tests.yml

@@ -373,21 +373,21 @@ jobs:
             file: ccip
             run: -run ^TestSmokeCCIPForBidirectionalLane$
             config_path: ./integration-tests/ccip-tests/testconfig/tomls/leader-lane.toml
-          - name: ccip-smoke-reorg
+          - name: ccip-smoke-reorg-bidirectional
             nodes: 1
             dir: ccip-tests/smoke
             os: ubuntu-latest
             file: ccip
             run: -run ^TestSmokeCCIPReorgBelowFinality$ -v
             config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml
-          - name: ccip-smoke-reorg
+          - name: ccip-smoke-reorg-below-finality
             nodes: 1
             dir: ccip-tests/smoke
             os: ubuntu-latest
             file: ccip
             run: -run ^TestSmokeCCIPReorgAboveFinalityAtDestination$ -v
             config_path: ./integration-tests/ccip-tests/testconfig/tomls/ccip-reorg.toml
-          - name: ccip-smoke-reorg
+          - name: ccip-smoke-reorg-above-finality
             nodes: 1
             dir: ccip-tests/smoke
             os: ubuntu-latest

core/services/ocr2/plugins/ccip/testhelpers/ccip_contracts.go

@@ -150,13 +150,15 @@ func NewExecOffchainConfig(
 	RelativeBoostPerWaitHour float64,
 	InflightCacheExpiry config.Duration,
 	RootSnoozeTime config.Duration,
+	BatchingStrategyID uint32, // 0 = Standard, 1 = Out of Order
 ) ExecOffchainConfig {
 	return ExecOffchainConfig{v1_2_0.JSONExecOffchainConfig{
 		DestOptimisticConfirmations: DestOptimisticConfirmations,
 		BatchGasLimit:               BatchGasLimit,
 		RelativeBoostPerWaitHour:    RelativeBoostPerWaitHour,
 		InflightCacheExpiry:         InflightCacheExpiry,
 		RootSnoozeTime:              RootSnoozeTime,
+		BatchingStrategyID:          BatchingStrategyID,
 	}}
 }
 

core/services/ocr2/plugins/ccip/testhelpers/config.go

@@ -70,6 +70,7 @@ func (c *CCIPContracts) createExecOffchainConfig(t *testing.T, inflightCacheExpi
 		0.07,
 		*config.MustNewDuration(inflightCacheExpiry),
 		*config.MustNewDuration(rootSnoozeTime),
+		uint32(0),
 	).Encode()
 	require.NoError(t, err)
 	return config

core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/ccip_contracts_1_4_0.go

@@ -157,13 +157,15 @@ func NewExecOffchainConfig(
 	RelativeBoostPerWaitHour float64,
 	InflightCacheExpiry config.Duration,
 	RootSnoozeTime config.Duration,
+	BatchingStrategyID uint32,
 ) ExecOffchainConfig {
 	return ExecOffchainConfig{v1_2_0.JSONExecOffchainConfig{
 		DestOptimisticConfirmations: DestOptimisticConfirmations,
 		BatchGasLimit:               BatchGasLimit,
 		RelativeBoostPerWaitHour:    RelativeBoostPerWaitHour,
 		InflightCacheExpiry:         InflightCacheExpiry,
 		RootSnoozeTime:              RootSnoozeTime,
+		BatchingStrategyID:          BatchingStrategyID,
 	}}
 }
 

core/services/ocr2/plugins/ccip/testhelpers/testhelpers_1_4_0/config_1_4_0.go

@@ -68,6 +68,7 @@ func (c *CCIPContracts) createExecOffchainConfig(t *testing.T, inflightCacheExpi
 		0.07,
 		*config.MustNewDuration(inflightCacheExpiry),
 		*config.MustNewDuration(rootSnoozeTime),
+		uint32(0),
 	).Encode()
 	require.NoError(t, err)
 	return config

integration-tests/ccip-tests/actions/ccip_helpers.go

@@ -61,6 +61,7 @@ import (
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/rmn_contract"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/router"
 	"github.com/smartcontractkit/chainlink/v2/core/gethwrappers/ccip/generated/token_pool"
+	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/ccipexec"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/config"
 	"github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/testhelpers"
 	integrationtesthelpers "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/testhelpers/integration"
@@ -185,6 +186,7 @@ type CCIPCommon struct {
 	gasUpdateWatcherMu        *sync.Mutex
 	gasUpdateWatcher          map[uint64]*big.Int // key - destchain id; value - timestamp of update
 	GasUpdateEvents           []contracts.GasUpdateEvent
+	AllowOutOfOrder           bool
 }
 
 // FreeUpUnusedSpace sets nil to various elements of ccipModule which are only used
@@ -273,6 +275,9 @@ func (ccipModule *CCIPCommon) CurseARM() (*types.Transaction, error) {
 
 func (ccipModule *CCIPCommon) LoadContractAddresses(conf *laneconfig.LaneConfig, noOfTokens *int) {
 	if conf != nil {
+		if conf.AllowOutOfOrder {
+			ccipModule.AllowOutOfOrder = true
+		}
 		if common.IsHexAddress(conf.FeeToken) {
 			ccipModule.FeeToken = &contracts.LinkToken{
 				EthAddress: common.HexToAddress(conf.FeeToken),
@@ -725,17 +730,16 @@ func (ccipModule *CCIPCommon) WriteLaneConfig(conf *laneconfig.LaneConfig) {
 	for k, v := range ccipModule.PriceAggregators {
 		priceAggrs[k.Hex()] = v.ContractAddress.Hex()
 	}
-	conf.CommonContracts = laneconfig.CommonContracts{
-		FeeToken:         ccipModule.FeeToken.Address(),
-		BridgeTokens:     btAddresses,
-		BridgeTokenPools: btpAddresses,
-		ARM:              ccipModule.RMNContract.Hex(),
-		Router:           ccipModule.Router.Address(),
-		PriceRegistry:    ccipModule.PriceRegistry.Address(),
-		PriceAggregators: priceAggrs,
-		WrappedNative:    ccipModule.WrappedNative.Hex(),
-		Multicall:        ccipModule.MulticallContract.Hex(),
-	}
+	conf.CommonContracts.FeeToken = ccipModule.FeeToken.Address()
+	conf.CommonContracts.BridgeTokens = btAddresses
+	conf.CommonContracts.BridgeTokenPools = btpAddresses
+	conf.CommonContracts.ARM = ccipModule.RMNContract.Hex()
+	conf.CommonContracts.Router = ccipModule.Router.Address()
+	conf.CommonContracts.PriceRegistry = ccipModule.PriceRegistry.Address()
+	conf.CommonContracts.PriceAggregators = priceAggrs
+	conf.CommonContracts.WrappedNative = ccipModule.WrappedNative.Hex()
+	conf.CommonContracts.Multicall = ccipModule.MulticallContract.Hex()
+
 	if ccipModule.TokenAdminRegistry != nil {
 		conf.CommonContracts.TokenAdminRegistry = ccipModule.TokenAdminRegistry.Address()
 	}
@@ -1626,7 +1630,7 @@ func (sourceCCIP *SourceCCIPModule) IsRequestTriggeredWithinTimeframe(timeframe
 		if sendRequestedEvents, exists := value.([]*evm_2_evm_onramp.EVM2EVMOnRampCCIPSendRequested); exists {
 			for _, sendRequestedEvent := range sendRequestedEvents {
 				raw := sendRequestedEvent.Raw
-				hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(raw.BlockNumber)))
+				hdr, err := sourceCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(raw.BlockNumber))
 				if err == nil {
 					if hdr.Timestamp.After(lastSeenTimestamp) {
 						foundAt = pointer.ToTime(hdr.Timestamp)
@@ -1709,6 +1713,7 @@ func (sourceCCIP *SourceCCIPModule) AssertEventCCIPSendRequested(
 // CCIPMsg constructs the message for a CCIP request
 func (sourceCCIP *SourceCCIPModule) CCIPMsg(
 	receiver common.Address,
+	allowOutOfOrder bool,
 	gasLimit *big.Int,
 ) (router.ClientEVM2AnyMessage, error) {
 	length := sourceCCIP.MsgDataLength
@@ -1750,7 +1755,15 @@ func (sourceCCIP *SourceCCIPModule) CCIPMsg(
 		return router.ClientEVM2AnyMessage{}, fmt.Errorf("failed encoding the receiver address: %w", err)
 	}
 
-	extraArgsV1, err := testhelpers.GetEVMExtraArgsV1(gasLimit, false)
+	var extraArgs []byte
+	matchErr := contracts.MatchContractVersionsOrAbove(map[contracts.Name]contracts.Version{
+		contracts.OnRampContract: contracts.V1_5_0,
+	})
+	if matchErr != nil {
+		extraArgs, err = testhelpers.GetEVMExtraArgsV1(gasLimit, false)
+	} else {
+		extraArgs, err = testhelpers.GetEVMExtraArgsV2(gasLimit, allowOutOfOrder)
+	}
 	if err != nil {
 		return router.ClientEVM2AnyMessage{}, fmt.Errorf("failed encoding the options field: %w", err)
 	}
@@ -1760,22 +1773,19 @@ func (sourceCCIP *SourceCCIPModule) CCIPMsg(
 		Data:         []byte(data),
 		TokenAmounts: tokenAndAmounts,
 		FeeToken:     common.HexToAddress(sourceCCIP.Common.FeeToken.Address()),
-		ExtraArgs:    extraArgsV1,
+		ExtraArgs:    extraArgs,
 	}, nil
 }
 
 // SendRequest sends a CCIP request to the source chain's router contract
-func (sourceCCIP *SourceCCIPModule) SendRequest(
-	receiver common.Address,
-	gasLimit *big.Int,
-) (common.Hash, time.Duration, *big.Int, error) {
+func (sourceCCIP *SourceCCIPModule) SendRequest(receiver common.Address, gasLimit *big.Int) (common.Hash, time.Duration, *big.Int, error) {
 	var d time.Duration
 	destChainSelector, err := chainselectors.SelectorFromChainId(sourceCCIP.DestinationChainId)
 	if err != nil {
 		return common.Hash{}, d, nil, fmt.Errorf("failed getting the chain selector: %w", err)
 	}
 	// form the message for transfer
-	msg, err := sourceCCIP.CCIPMsg(receiver, gasLimit)
+	msg, err := sourceCCIP.CCIPMsg(receiver, sourceCCIP.Common.AllowOutOfOrder, gasLimit)
 	if err != nil {
 		return common.Hash{}, d, nil, fmt.Errorf("failed forming the ccip msg: %w", err)
 	}
@@ -2218,7 +2228,7 @@ func (destCCIP *DestCCIPModule) AssertNoReportAcceptedEventReceived(lggr *zerolo
 				e, exists := value.(*evm_2_evm_offramp.EVM2EVMOffRampExecutionStateChanged)
 				if exists {
 					vLogs := e.Raw
-					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, big.NewInt(int64(vLogs.BlockNumber)))
+					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, new(big.Int).SetUint64(vLogs.BlockNumber))
 					if err != nil {
 						return true
 					}
@@ -2259,7 +2269,7 @@ func (destCCIP *DestCCIPModule) AssertNoExecutionStateChangedEventReceived(
 				e, exists := value.(*contracts.EVM2EVMOffRampExecutionStateChanged)
 				if exists {
 					vLogs := e.LogInfo
-					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, big.NewInt(int64(vLogs.BlockNumber)))
+					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(ctx, new(big.Int).SetUint64(vLogs.BlockNumber))
 					if err != nil {
 						return true
 					}
@@ -2288,7 +2298,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged(
 	reqStat *testreporters.RequestStat,
 	execState testhelpers.MessageExecutionState,
 ) (uint8, error) {
-	lggr.Info().Int64("seqNum", int64(seqNum)).Str("Timeout", timeout.String()).Msg("Waiting for ExecutionStateChanged event")
+	lggr.Info().Uint64("seqNum", seqNum).Str("Timeout", timeout.String()).Msg("Waiting for ExecutionStateChanged event")
 	timer := time.NewTimer(timeout)
 	defer timer.Stop()
 	ticker := time.NewTicker(time.Second)
@@ -2306,7 +2316,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged(
 					destCCIP.ExecStateChangedWatcher.Delete(seqNum)
 					vLogs := e.LogInfo
 					receivedAt := time.Now().UTC()
-					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(vLogs.BlockNumber)))
+					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(vLogs.BlockNumber))
 					if err == nil {
 						receivedAt = hdr.Timestamp
 					}
@@ -2319,7 +2329,7 @@ func (destCCIP *DestCCIPModule) AssertEventExecutionStateChanged(
 						gasUsed = receipt.GasUsed
 					}
 					if testhelpers.MessageExecutionState(e.State) == execState {
-						lggr.Info().Int64("seqNum", int64(seqNum)).Uint8("ExecutionState", e.State).Msg("ExecutionStateChanged event received")
+						lggr.Info().Uint64("seqNum", seqNum).Uint8("ExecutionState", e.State).Msg("ExecutionStateChanged event received")
 						reqStat.UpdateState(lggr, seqNum, testreporters.ExecStateChanged, receivedAt.Sub(timeNow),
 							testreporters.Success,
 							&testreporters.TransactionStats{
@@ -2363,7 +2373,7 @@ func (destCCIP *DestCCIPModule) AssertEventReportAccepted(
 	prevEventAt time.Time,
 	reqStat *testreporters.RequestStat,
 ) (*contracts.CommitStoreReportAccepted, time.Time, error) {
-	lggr.Info().Int64("seqNum", int64(seqNum)).Str("Timeout", timeout.String()).Msg("Waiting for ReportAccepted event")
+	lggr.Info().Uint64("seqNum", seqNum).Str("Timeout", timeout.String()).Msg("Waiting for ReportAccepted event")
 	timer := time.NewTimer(timeout)
 	defer timer.Stop()
 	resetTimerCount := 0
@@ -2379,7 +2389,7 @@ func (destCCIP *DestCCIPModule) AssertEventReportAccepted(
 					// if the value is processed, delete it from the map
 					destCCIP.ReportAcceptedWatcher.Delete(seqNum)
 					receivedAt := time.Now().UTC()
-					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(reportAccepted.LogInfo.BlockNumber)))
+					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(reportAccepted.LogInfo.BlockNumber))
 					if err == nil {
 						receivedAt = hdr.Timestamp
 					}
@@ -2488,7 +2498,7 @@ func (destCCIP *DestCCIPModule) AssertReportBlessed(
 						// if the value is processed, delete it from the map
 						destCCIP.ReportBlessedBySeqNum.Delete(seqNum)
 					}
-					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), big.NewInt(int64(vLogs.BlockNumber)))
+					hdr, err := destCCIP.Common.ChainClient.HeaderByNumber(context.Background(), new(big.Int).SetUint64(vLogs.BlockNumber))
 					if err == nil {
 						receivedAt = hdr.Timestamp
 					}
@@ -2536,7 +2546,7 @@ func (destCCIP *DestCCIPModule) AssertSeqNumberExecuted(
 	timeNow time.Time,
 	reqStat *testreporters.RequestStat,
 ) error {
-	lggr.Info().Int64("seqNum", int64(seqNumberBefore)).Str("Timeout", timeout.String()).Msg("Waiting to be processed by commit store")
+	lggr.Info().Uint64("seqNum", seqNumberBefore).Str("Timeout", timeout.String()).Msg("Waiting to be processed by commit store")
 	timer := time.NewTimer(timeout)
 	defer timer.Stop()
 	resetTimerCount := 0
@@ -2793,7 +2803,8 @@ func (lane *CCIPLane) AddToSentReqs(txHash common.Hash, reqStats []*testreporter
 func (lane *CCIPLane) Multicall(noOfRequests int, multiSendAddr common.Address) error {
 	var ccipMultipleMsg []contracts.CCIPMsgData
 	feeToken := common.HexToAddress(lane.Source.Common.FeeToken.Address())
-	genericMsg, err := lane.Source.CCIPMsg(lane.Dest.ReceiverDapp.EthAddress, big.NewInt(DefaultDestinationGasLimit))
+	genericMsg, err := lane.Source.CCIPMsg(lane.Dest.ReceiverDapp.EthAddress, lane.Source.Common.AllowOutOfOrder,
+		big.NewInt(DefaultDestinationGasLimit))
 	if err != nil {
 		return fmt.Errorf("failed to form the ccip message: %w", err)
 	}
@@ -2885,10 +2896,7 @@ func (lane *CCIPLane) Multicall(noOfRequests int, multiSendAddr common.Address)
 func (lane *CCIPLane) SendRequests(noOfRequests int, gasLimit *big.Int) error {
 	for i := 1; i <= noOfRequests; i++ {
 		stat := testreporters.NewCCIPRequestStats(int64(lane.NumberOfReq+i), lane.SourceNetworkName, lane.DestNetworkName)
-		txHash, txConfirmationDur, fee, err := lane.Source.SendRequest(
-			lane.Dest.ReceiverDapp.EthAddress,
-			gasLimit,
-		)
+		txHash, txConfirmationDur, fee, err := lane.Source.SendRequest(lane.Dest.ReceiverDapp.EthAddress, gasLimit)
 		if err != nil {
 			stat.UpdateState(lane.Logger, 0, testreporters.TX, txConfirmationDur, testreporters.Failure, nil)
 			return fmt.Errorf("could not send request: %w", err)
@@ -3522,6 +3530,7 @@ func (lane *CCIPLane) DeployNewCCIPLane(
 		srcConf                = lane.SrcNetworkLaneCfg
 		destConf               = lane.DstNetworkLaneCfg
 		commitAndExecOnSameDON = pointer.GetBool(testConf.CommitAndExecuteOnSameDON)
+		allowOutOfOrder        = pointer.GetBool(testConf.AllowOutOfOrder)
 		withPipeline           = pointer.GetBool(testConf.TokenConfig.WithPipeline)
 		configureCLNodes       = !pointer.GetBool(testConf.ExistingDeployment)
 	)
@@ -3536,6 +3545,13 @@ func (lane *CCIPLane) DeployNewCCIPLane(
 	if err != nil {
 		return fmt.Errorf("failed to create source module: %w", err)
 	}
+
+	// If AllowOutOfOrder is set globally in test config, then assumption is to set it for every lane.
+	//However, if this is set as false, and set as true for specific chain in lane_configs then apply it
+	//only for a lane where source network is of that chain.
+	if allowOutOfOrder {
+		lane.Source.Common.AllowOutOfOrder = true
+	}
 	lane.Dest, err = DefaultDestinationCCIPModule(
 		lane.Logger, testConf,
 		destChainClient, sourceChainClient.GetChainID().Uint64(),
@@ -3787,13 +3803,20 @@ func SetOCR2Config(
 	if len(execNodes) > 0 {
 		nodes = execNodes
 	}
+
+	// Use out of order batching strategy if we expect to be sending out of order messages
+	batchingStrategyID := ccipexec.BestEffortBatchingStrategyID
+	if pointer.GetBool(testConf.AllowOutOfOrder) {
+		batchingStrategyID = ccipexec.ZKOverflowBatchingStrategyID
+	}
 	if destCCIP.OffRamp != nil {
 		execOffchainCfg, err := contracts.NewExecOffchainConfig(
 			1,
 			BatchGasLimit,
 			0.7,
 			*inflightExpiryExec,
 			*commonconfig.MustNewDuration(RootSnoozeTime),
+			batchingStrategyID,
 		)
 		if err != nil {
 			return fmt.Errorf("failed to create exec offchain config: %w", err)

integration-tests/ccip-tests/contracts/contract_deployer.go

@@ -77,7 +77,7 @@ func MatchContractVersionsOrAbove(requiredContractVersions map[Name]Version) err
 // if the version is less than 1.5.0, then token admin registry is not needed
 func NeedTokenAdminRegistry() bool {
 	return MatchContractVersionsOrAbove(map[Name]Version{
-		TokenPoolContract: V1_5_0_dev,
+		TokenPoolContract: V1_5_0,
 	}) == nil
 }
 
@@ -1044,6 +1044,7 @@ func (e *CCIPContractsDeployer) DeployOnRamp(
 					MaxPerMsgGasLimit:                 4_000_000,
 					DefaultTokenFeeUSDCents:           50,
 					DefaultTokenDestGasOverhead:       125_000,
+					EnforceOutOfOrder:                 false,
 				},
 				evm_2_evm_onramp.RateLimiterConfig{
 					Capacity: opts.Capacity,
@@ -1465,12 +1466,14 @@ func NewExecOnchainConfig(
 	}
 }
 
+// NewExecOffchainConfig creates a config for the OffChain portion of how CCIP operates
 func NewExecOffchainConfig(
 	destOptimisticConfirmations uint32,
 	batchGasLimit uint32,
 	relativeBoostPerWaitHour float64,
 	inflightCacheExpiry config.Duration,
 	rootSnoozeTime config.Duration,
+	batchingStrategyID uint32, // See ccipexec package
 ) (ccipconfig.OffchainConfig, error) {
 	switch VersionMap[OffRampContract] {
 	case Latest:
@@ -1480,6 +1483,7 @@ func NewExecOffchainConfig(
 			relativeBoostPerWaitHour,
 			inflightCacheExpiry,
 			rootSnoozeTime,
+			batchingStrategyID,
 		), nil
 	case V1_2_0:
 		return testhelpers_1_4_0.NewExecOffchainConfig(
@@ -1488,6 +1492,7 @@ func NewExecOffchainConfig(
 			relativeBoostPerWaitHour,
 			inflightCacheExpiry,
 			rootSnoozeTime,
+			batchingStrategyID,
 		), nil
 	default:
 		return nil, fmt.Errorf("version not supported: %s", VersionMap[OffRampContract])