improve solana tx broadcasting in e2e tests (#3378)

e2e/e2etests/test_solana_whitelist_spl.go

@@ -2,6 +2,7 @@ package e2etests
 
 import (
 	"github.com/gagliardetto/solana-go"
+	"github.com/gagliardetto/solana-go/rpc"
 	"github.com/stretchr/testify/require"
 
 	"github.com/zeta-chain/node/e2e/runner"
@@ -26,7 +27,11 @@ func TestSolanaWhitelistSPL(r *runner.E2ERunner, _ []string) {
 	whitelistEntryPDA, _, err := solana.FindProgramAddress(seed, r.GatewayProgram)
 	require.NoError(r, err)
 
-	whitelistEntryInfo, err := r.SolanaClient.GetAccountInfo(r.Ctx, whitelistEntryPDA)
+	whitelistEntryInfo, err := r.SolanaClient.GetAccountInfoWithOpts(
+		r.Ctx,
+		whitelistEntryPDA,
+		&rpc.GetAccountInfoOpts{Commitment: rpc.CommitmentConfirmed},
+	)
 	require.Error(r, err)
 	require.Nil(r, whitelistEntryInfo)
 
@@ -62,7 +67,11 @@ func TestSolanaWhitelistSPL(r *runner.E2ERunner, _ []string) {
 	r.WaitForMinedCCTXFromIndex(whitelistCCTXIndex)
 
 	// check that whitelist entry exists for this spl
-	whitelistEntryInfo, err = r.SolanaClient.GetAccountInfo(r.Ctx, whitelistEntryPDA)
+	whitelistEntryInfo, err = r.SolanaClient.GetAccountInfoWithOpts(
+		r.Ctx,
+		whitelistEntryPDA,
+		&rpc.GetAccountInfoOpts{Commitment: rpc.CommitmentConfirmed},
+	)
 	require.NoError(r, err)
 	require.NotNil(r, whitelistEntryInfo)
 }

e2e/e2etests/test_spl_deposit.go

@@ -23,11 +23,11 @@ func TestSPLDeposit(r *runner.E2ERunner, args []string) {
 	pda := r.ComputePdaAddress()
 	pdaAta := r.ResolveSolanaATA(privKey, pda, r.SPLAddr)
 
-	pdaBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, pdaAta, rpc.CommitmentFinalized)
+	pdaBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, pdaAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
 	senderAta := r.ResolveSolanaATA(privKey, privKey.PublicKey(), r.SPLAddr)
-	senderBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, senderAta, rpc.CommitmentFinalized)
+	senderBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, senderAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
 	// get zrc20 balance for recipient
@@ -45,10 +45,10 @@ func TestSPLDeposit(r *runner.E2ERunner, args []string) {
 	require.Equal(r, cctx.GetCurrentOutboundParam().Receiver, r.EVMAddress().Hex())
 
 	// verify balances are updated
-	pdaBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, pdaAta, rpc.CommitmentFinalized)
+	pdaBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, pdaAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
-	senderBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, senderAta, rpc.CommitmentFinalized)
+	senderBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, senderAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
 	zrc20BalanceAfter, err := r.SPLZRC20.BalanceOf(&bind.CallOpts{}, r.EVMAddress())

e2e/e2etests/test_spl_deposit_and_call.go

@@ -29,11 +29,11 @@ func TestSPLDepositAndCall(r *runner.E2ERunner, args []string) {
 	pda := r.ComputePdaAddress()
 	pdaAta := r.ResolveSolanaATA(privKey, pda, r.SPLAddr)
 
-	pdaBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, pdaAta, rpc.CommitmentFinalized)
+	pdaBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, pdaAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
 	senderAta := r.ResolveSolanaATA(privKey, privKey.PublicKey(), r.SPLAddr)
-	senderBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, senderAta, rpc.CommitmentFinalized)
+	senderBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, senderAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
 	// get zrc20 balance for recipient
@@ -55,10 +55,10 @@ func TestSPLDepositAndCall(r *runner.E2ERunner, args []string) {
 	utils.MustHaveCalledExampleContractWithMsg(r, contract, big.NewInt(int64(amount)), data)
 
 	// verify balances are updated
-	pdaBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, pdaAta, rpc.CommitmentFinalized)
+	pdaBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, pdaAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
-	senderBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, senderAta, rpc.CommitmentFinalized)
+	senderBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, senderAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
 	zrc20BalanceAfter, err := r.SPLZRC20.BalanceOf(&bind.CallOpts{}, contractAddr)

e2e/e2etests/test_spl_withdraw.go

@@ -40,7 +40,7 @@ func TestSPLWithdraw(r *runner.E2ERunner, args []string) {
 
 	// get receiver ata balance before withdraw
 	receiverAta := r.ResolveSolanaATA(privkey, privkey.PublicKey(), r.SPLAddr)
-	receiverBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, receiverAta, rpc.CommitmentFinalized)
+	receiverBalanceBefore, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, receiverAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 	r.Logger.Info("receiver balance of SPL before withdraw: %s", receiverBalanceBefore.Value.Amount)
 
@@ -57,7 +57,7 @@ func TestSPLWithdraw(r *runner.E2ERunner, args []string) {
 	r.Logger.Info("runner balance of SPL after withdraw: %d", zrc20BalanceAfter)
 
 	// verify balances are updated
-	receiverBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, receiverAta, rpc.CommitmentFinalized)
+	receiverBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, receiverAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 	r.Logger.Info("receiver balance of SPL after withdraw: %s", receiverBalanceAfter.Value.Amount)
 

e2e/e2etests/test_spl_withdraw_and_create_receiver_ata.go

@@ -45,7 +45,11 @@ func TestSPLWithdrawAndCreateReceiverAta(r *runner.E2ERunner, args []string) {
 	receiverAta, _, err := solana.FindAssociatedTokenAddress(receiverPrivKey.PublicKey(), r.SPLAddr)
 	require.NoError(r, err)
 
-	receiverAtaAcc, err := r.SolanaClient.GetAccountInfo(r.Ctx, receiverAta)
+	receiverAtaAcc, err := r.SolanaClient.GetAccountInfoWithOpts(
+		r.Ctx,
+		receiverAta,
+		&rpc.GetAccountInfoOpts{Commitment: rpc.CommitmentConfirmed},
+	)
 	require.Error(r, err)
 	require.Nil(r, receiverAtaAcc)
 
@@ -62,12 +66,16 @@ func TestSPLWithdrawAndCreateReceiverAta(r *runner.E2ERunner, args []string) {
 	r.Logger.Info("runner balance of SPL after withdraw: %d", zrc20BalanceAfter)
 
 	// verify receiver ata was created
-	receiverAtaAcc, err = r.SolanaClient.GetAccountInfo(r.Ctx, receiverAta)
+	receiverAtaAcc, err = r.SolanaClient.GetAccountInfoWithOpts(
+		r.Ctx,
+		receiverAta,
+		&rpc.GetAccountInfoOpts{Commitment: rpc.CommitmentConfirmed},
+	)
 	require.NoError(r, err)
 	require.NotNil(r, receiverAtaAcc)
 
 	// verify balances are updated
-	receiverBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, receiverAta, rpc.CommitmentFinalized)
+	receiverBalanceAfter, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, receiverAta, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 	r.Logger.Info("receiver balance of SPL after withdraw: %s", receiverBalanceAfter.Value.Amount)
 

e2e/runner/balances.go

@@ -104,7 +104,7 @@ func (r *E2ERunner) GetAccountBalances(skipBTC bool) (AccountBalances, error) {
 		solSOLBalance, err := r.SolanaClient.GetBalance(
 			r.Ctx,
 			solanaAddr,
-			rpc.CommitmentFinalized,
+			rpc.CommitmentConfirmed,
 		)
 		if err != nil {
 			return AccountBalances{}, fmt.Errorf("get sol balance: %w", err)
@@ -119,7 +119,7 @@ func (r *E2ERunner) GetAccountBalances(skipBTC bool) (AccountBalances, error) {
 				solanaAddr,
 				r.SPLAddr,
 			)
-			splBalance, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, ata, rpc.CommitmentFinalized)
+			splBalance, err := r.SolanaClient.GetTokenAccountBalance(r.Ctx, ata, rpc.CommitmentConfirmed)
 			if err != nil {
 				return AccountBalances{}, fmt.Errorf("get spl balance: %w", err)
 			}

e2e/runner/setup_solana.go

@@ -36,7 +36,7 @@ func (r *E2ERunner) SetupSolana(gatewayID, deployerPrivateKey string) {
 	// get deployer account balance
 	privkey, err := solana.PrivateKeyFromBase58(deployerPrivateKey)
 	require.NoError(r, err)
-	bal, err := r.SolanaClient.GetBalance(r.Ctx, privkey.PublicKey(), rpc.CommitmentFinalized)
+	bal, err := r.SolanaClient.GetBalance(r.Ctx, privkey.PublicKey(), rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 	r.Logger.Info("deployer address: %s, balance: %f SOL", privkey.PublicKey().String(), float64(bal.Value)/1e9)
 
@@ -68,7 +68,9 @@ func (r *E2ERunner) SetupSolana(gatewayID, deployerPrivateKey string) {
 	r.Logger.Info("initialize logs: %v", out.Meta.LogMessages)
 
 	// retrieve the PDA account info
-	pdaInfo, err := r.SolanaClient.GetAccountInfo(r.Ctx, pdaComputed)
+	pdaInfo, err := r.SolanaClient.GetAccountInfoWithOpts(r.Ctx, pdaComputed, &rpc.GetAccountInfoOpts{
+		Commitment: rpc.CommitmentConfirmed,
+	})
 	require.NoError(r, err)
 
 	// deserialize the PDA info

e2e/runner/solana.go

@@ -8,6 +8,7 @@ import (
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
 	"github.com/gagliardetto/solana-go"
 	associatedtokenaccount "github.com/gagliardetto/solana-go/programs/associated-token-account"
+	computebudget "github.com/gagliardetto/solana-go/programs/compute-budget"
 	"github.com/gagliardetto/solana-go/programs/system"
 	"github.com/gagliardetto/solana-go/programs/token"
 	"github.com/gagliardetto/solana-go/rpc"
@@ -141,9 +142,11 @@ func (r *E2ERunner) CreateSignedTransaction(
 	additionalPrivateKeys []solana.PrivateKey,
 ) *solana.Transaction {
 	// get a recent blockhash
-	recent, err := r.SolanaClient.GetLatestBlockhash(r.Ctx, rpc.CommitmentFinalized)
+	recent, err := r.SolanaClient.GetLatestBlockhash(r.Ctx, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
+	r.Logger.Info("Latest valid block height for tx %d", recent.Value.LastValidBlockHeight)
+
 	// create the initialize transaction
 	tx, err := solana.NewTransaction(
 		instructions,
@@ -180,7 +183,11 @@ func (r *E2ERunner) ResolveSolanaATA(
 	pdaAta, _, err := solana.FindAssociatedTokenAddress(owner, mintAccount)
 	require.NoError(r, err)
 
-	info, _ := r.SolanaClient.GetAccountInfo(r.Ctx, pdaAta)
+	info, _ := r.SolanaClient.GetAccountInfoWithOpts(
+		r.Ctx,
+		pdaAta,
+		&rpc.GetAccountInfoOpts{Commitment: rpc.CommitmentConfirmed},
+	)
 	if info != nil {
 		// already exists
 		return pdaAta
@@ -228,8 +235,12 @@ func (r *E2ERunner) SPLDepositAndCall(
 		receiver,
 		data,
 	)
+
+	limit := computebudget.NewSetComputeUnitLimitInstruction(50000).Build() // 50k compute unit limit
+	feesInit := computebudget.NewSetComputeUnitPriceInstructionBuilder().
+		SetMicroLamports(100000).Build() // 0.1 lamports per compute unit
 	signedTx := r.CreateSignedTransaction(
-		[]solana.Instruction{depositSPLInstruction},
+		[]solana.Instruction{limit, feesInit, depositSPLInstruction},
 		*privateKey,
 		[]solana.PrivateKey{},
 	)
@@ -241,7 +252,7 @@ func (r *E2ERunner) SPLDepositAndCall(
 }
 
 func (r *E2ERunner) DeploySPL(privateKey *solana.PrivateKey, whitelist bool) *solana.Wallet {
-	lamport, err := r.SolanaClient.GetMinimumBalanceForRentExemption(r.Ctx, token.MINT_SIZE, rpc.CommitmentFinalized)
+	lamport, err := r.SolanaClient.GetMinimumBalanceForRentExemption(r.Ctx, token.MINT_SIZE, rpc.CommitmentConfirmed)
 	require.NoError(r, err)
 
 	// to deploy new spl token, create account instruction and initialize mint instruction have to be in the same transaction
@@ -292,7 +303,11 @@ func (r *E2ERunner) DeploySPL(privateKey *solana.PrivateKey, whitelist bool) *so
 		whitelistEntryPDA, _, err := solana.FindProgramAddress(seed, r.GatewayProgram)
 		require.NoError(r, err)
 
-		whitelistEntryInfo, err := r.SolanaClient.GetAccountInfo(r.Ctx, whitelistEntryPDA)
+		whitelistEntryInfo, err := r.SolanaClient.GetAccountInfoWithOpts(
+			r.Ctx,
+			whitelistEntryPDA,
+			&rpc.GetAccountInfoOpts{Commitment: rpc.CommitmentConfirmed},
+		)
 		require.Error(r, err)
 
 		// already whitelisted
@@ -313,39 +328,82 @@ func (r *E2ERunner) DeploySPL(privateKey *solana.PrivateKey, whitelist bool) *so
 		_, out := r.BroadcastTxSync(signedTx)
 		r.Logger.Info("whitelist spl mint logs: %v", out.Meta.LogMessages)
 
-		whitelistEntryInfo, err = r.SolanaClient.GetAccountInfo(r.Ctx, whitelistEntryPDA)
+		whitelistEntryInfo, err = r.SolanaClient.GetAccountInfoWithOpts(
+			r.Ctx,
+			whitelistEntryPDA,
+			&rpc.GetAccountInfoOpts{
+				Commitment: rpc.CommitmentConfirmed,
+			},
+		)
 		require.NoError(r, err)
 		require.NotNil(r, whitelistEntryInfo)
 	}
 
 	return mintAccount
 }
 
-// BroadcastTxSync broadcasts a transaction and waits for it to be finalized
-func (r *E2ERunner) BroadcastTxSync(tx *solana.Transaction) (solana.Signature, *rpc.GetTransactionResult) {
+// BroadcastTxSync broadcasts a transaction once and checks if it's confirmed
+func (r *E2ERunner) BroadcastTxSyncOnce(tx *solana.Transaction) (solana.Signature, *rpc.GetTransactionResult, bool) {
 	// broadcast the transaction
-	sig, err := r.SolanaClient.SendTransactionWithOpts(r.Ctx, tx, rpc.TransactionOpts{})
-	require.NoError(r, err)
-	r.Logger.Info("broadcast success! tx sig %s; waiting for confirmation...", sig)
+	r.Logger.Info("Broadcast once start")
+	maxRetries := uint(1)
+	sig, err := r.SolanaClient.SendTransactionWithOpts(r.Ctx, tx, rpc.TransactionOpts{
+		SkipPreflight:       true,
+		MaxRetries:          &maxRetries,
+		PreflightCommitment: rpc.CommitmentConfirmed,
+	})
+	if err != nil { // try to fetch tx to see if error is not because it is already broadcasted, since we manually retry
+		r.Logger.Info("Error sending tx %s, check if it's already broadcasted, err: %s", sig, err.Error())
 
-	var (
-		start   = time.Now()
-		timeout = 2 * time.Minute // Solana tx expires automatically after 2 minutes
-	)
+		out, errGet := r.SolanaClient.GetTransaction(r.Ctx, sig, &rpc.GetTransactionOpts{
+			Commitment: rpc.CommitmentConfirmed,
+		})
+
+		if errGet == nil {
+			return sig, out, true
+		}
+
+		r.Logger.Info("Error getting tx %s", errGet.Error())
+		require.NoError(r, err) // fail the test with send tx error
+	}
+	r.Logger.Info("Broadcast success! tx sig %s; waiting for confirmation...", sig)
 
 	// wait for the transaction to be finalized
 	var out *rpc.GetTransactionResult
+	time.Sleep(5 * time.Second) // wait a bit and check if its confirmed
+	blockHeight, err := r.SolanaClient.GetBlockHeight(r.Ctx, rpc.CommitmentConfirmed)
+	require.NoError(r, err)
+	r.Logger.Info("Current block height %d", blockHeight)
+
+	out, err = r.SolanaClient.GetTransaction(r.Ctx, sig, &rpc.GetTransactionOpts{
+		Commitment: rpc.CommitmentConfirmed,
+	})
+	if err != nil {
+		r.Logger.Info("Error getting tx %s", err.Error())
+	}
+
+	isConfirmed := err == nil
+	r.Logger.Info("Broadcast once finished, tx: %s, confirmed: %t", sig, isConfirmed)
+	return sig, out, isConfirmed
+}
+
+// BroadcastTxSync broadcasts a transaction and waits for it to be finalized
+func (r *E2ERunner) BroadcastTxSync(tx *solana.Transaction) (solana.Signature, *rpc.GetTransactionResult) {
+	r.Logger.Info("Broadcast start")
+	start := time.Now()
+	timeout := 2 * time.Minute // Expires after 2 mins
+	sig, out, isConfirmed := r.BroadcastTxSyncOnce(tx)
 	for {
-		require.False(r, time.Since(start) > timeout, "waiting solana tx timeout")
+		require.False(r, time.Since(start) > timeout, "solana tx timeout")
 
-		time.Sleep(1 * time.Second)
-		out, err = r.SolanaClient.GetTransaction(r.Ctx, sig, &rpc.GetTransactionOpts{})
-		if err == nil {
-			break
+		if isConfirmed {
+			r.Logger.Info("Tx broadcasted and confirmed")
+			return sig, out
 		}
-	}
 
-	return sig, out
+		r.Logger.Info("Manually retrying tx")
+		sig, out, isConfirmed = r.BroadcastTxSyncOnce(tx)
+	}
 }
 
 // SOLDepositAndCall deposits an amount of ZRC20 SOL tokens (in lamports) and calls a contract (if data is provided)
@@ -365,7 +423,14 @@ func (r *E2ERunner) SOLDepositAndCall(
 	instruction := r.CreateDepositInstruction(signerPrivKey.PublicKey(), receiver, data, amount.Uint64())
 
 	// create and sign the transaction
-	signedTx := r.CreateSignedTransaction([]solana.Instruction{instruction}, *signerPrivKey, []solana.PrivateKey{})
+	limit := computebudget.NewSetComputeUnitLimitInstruction(50000).Build() // 50k compute unit limit
+	feesInit := computebudget.NewSetComputeUnitPriceInstructionBuilder().
+		SetMicroLamports(100000).Build() // 0.1 lamports per compute unit
+	signedTx := r.CreateSignedTransaction(
+		[]solana.Instruction{limit, feesInit, instruction},
+		*signerPrivKey,
+		[]solana.PrivateKey{},
+	)
 
 	// broadcast the transaction and wait for finalization
 	sig, out := r.BroadcastTxSync(signedTx)

e2e/runner/verify.go

@@ -42,7 +42,9 @@ func (r *E2ERunner) VerifySolanaWithdrawalAmountFromCCTX(cctx *crosschaintypes.C
 	require.NoError(r, err)
 
 	// query transaction by signature
-	txResult, err := r.SolanaClient.GetTransaction(r.Ctx, sig, &rpc.GetTransactionOpts{})
+	txResult, err := r.SolanaClient.GetTransaction(r.Ctx, sig, &rpc.GetTransactionOpts{
+		Commitment: rpc.CommitmentConfirmed,
+	})
 	require.NoError(r, err)
 
 	// unmarshal transaction