From f14f9c60ec0f747ee0a3ba47234f99c3dc954ab6 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Mon, 4 Sep 2023 18:09:07 +0300 Subject: [PATCH 1/8] Fill bundle_uuid field on bundle insertions --- core/types/transaction.go | 20 ++++++++++++++++++++ flashbotsextra/database.go | 14 ++++++++------ flashbotsextra/database_types.go | 9 +++++---- 3 files changed, 33 insertions(+), 10 deletions(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 03eb018326..91b75b438e 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -19,9 +19,11 @@ package types import ( "bytes" "container/heap" + "encoding/binary" "errors" "io" "math/big" + "sort" "sync/atomic" "time" @@ -30,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/google/uuid" + "golang.org/x/crypto/sha3" ) var ( @@ -784,6 +787,23 @@ type MevBundle struct { Hash common.Hash } +func (b *MevBundle) UniquePayload() []byte { + var buf []byte + buf = binary.AppendVarint(buf, b.BlockNumber.Int64()) + buf = append(buf, b.Hash[:]...) + sort.Slice(b.RevertingTxHashes, func(i, j int) bool { + return bytes.Compare(b.RevertingTxHashes[i][:], b.RevertingTxHashes[j][:]) <= 0 + }) + for _, txHash := range b.RevertingTxHashes { + buf = append(buf, txHash[:]...) + } + return buf +} + +func (b *MevBundle) ComputeUUID() uuid.UUID { + return uuid.NewHash(sha3.NewLegacyKeccak256(), uuid.Nil, b.UniquePayload(), 5) +} + func (b *MevBundle) RevertingHash(hash common.Hash) bool { for _, revHash := range b.RevertingTxHashes { if revHash == hash { diff --git a/flashbotsextra/database.go b/flashbotsextra/database.go index d39274e974..3700eb20fe 100644 --- a/flashbotsextra/database.go +++ b/flashbotsextra/database.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/google/uuid" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" ) @@ -61,7 +62,7 @@ func NewDatabaseService(postgresDSN string) (*DatabaseService, error) { return nil, err } - insertMissingBundleStmt, err := db.PrepareNamed("insert into bundles (bundle_hash, param_signed_txs, param_block_number, param_timestamp, received_timestamp, param_reverting_tx_hashes, coinbase_diff, total_gas_used, state_block_number, gas_fees, eth_sent_to_coinbase) values (:bundle_hash, :param_signed_txs, :param_block_number, :param_timestamp, :received_timestamp, :param_reverting_tx_hashes, :coinbase_diff, :total_gas_used, :state_block_number, :gas_fees, :eth_sent_to_coinbase) on conflict (bundle_hash, param_block_number) do nothing returning id") + insertMissingBundleStmt, err := db.PrepareNamed("insert into bundles (bundle_hash, param_signed_txs, param_block_number, param_timestamp, received_timestamp, param_reverting_tx_hashes, coinbase_diff, total_gas_used, state_block_number, gas_fees, eth_sent_to_coinbase, bundle_uuid) values (:bundle_hash, :param_signed_txs, :param_block_number, :param_timestamp, :received_timestamp, :param_reverting_tx_hashes, :coinbase_diff, :total_gas_used, :state_block_number, :gas_fees, :eth_sent_to_coinbase, :bundle_uuid) on conflict do nothing returning id") if err != nil { return nil, err } @@ -111,21 +112,22 @@ func (ds *DatabaseService) getBundleIds(ctx context.Context, blockNumber uint64, } for _, request := range requestsToMake { - query, args, err := sqlx.In("select id, bundle_hash from bundles where param_block_number = ? and bundle_hash in (?)", blockNumber, request) + query, args, err := sqlx.In("select id, bundle_hash, bundle_uuid from bundles where param_block_number = ? and bundle_hash in (?)", blockNumber, request) if err != nil { return nil, err } query = ds.db.Rebind(query) queryRes := []struct { - Id uint64 `db:"id"` - BundleHash string `db:"bundle_hash"` + Id uint64 `db:"id"` + BundleHash string `db:"bundle_hash"` + BundleUUID uuid.UUID `db:"bundle_uuid"` }{} + err = ds.db.SelectContext(ctx, &queryRes, query, args...) if err != nil { return nil, err } - for _, row := range queryRes { bundleIdsMap[row.BundleHash] = row.Id } @@ -250,7 +252,7 @@ func (ds *DatabaseService) ConsumeBuiltBlock(block *types.Block, blockValue *big bidTrace *apiv1.BidTrace) { ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) defer cancel() - + bundleIdsMap, err := ds.getBundleIdsAndInsertMissingBundles(ctx, block.NumberU64(), allBundles) if err != nil { log.Error("could not insert bundles", "err", err) diff --git a/flashbotsextra/database_types.go b/flashbotsextra/database_types.go index 8bff4d8cd2..74c51b60f7 100644 --- a/flashbotsextra/database_types.go +++ b/flashbotsextra/database_types.go @@ -38,8 +38,9 @@ type BuiltBlockBundle struct { } type DbBundle struct { - DbId uint64 `db:"id"` - BundleHash string `db:"bundle_hash"` + DbId uint64 `db:"id"` + BundleHash string `db:"bundle_hash"` + BundleUUID uuid.UUID `db:"bundle_uuid"` ParamSignedTxs string `db:"param_signed_txs"` ParamBlockNumber uint64 `db:"param_block_number"` @@ -93,8 +94,8 @@ func SimulatedBundleToDbBundle(bundle *types.SimulatedBundle) DbBundle { } return DbBundle{ - BundleHash: bundle.OriginalBundle.Hash.String(), - + BundleHash: bundle.OriginalBundle.Hash.String(), + BundleUUID: bundle.OriginalBundle.ComputeUUID(), ParamSignedTxs: strings.Join(signedTxsStrings, ","), ParamBlockNumber: bundle.OriginalBundle.BlockNumber.Uint64(), ParamTimestamp: &bundle.OriginalBundle.MinTimestamp, From 7c929ec202bbd088aa533e42e1f9f80b2b3a6152 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Tue, 5 Sep 2023 12:33:07 +0300 Subject: [PATCH 2/8] fix db ID determination --- flashbotsextra/database.go | 56 ++++++++++++++++++++++++--------- flashbotsextra/database_test.go | 2 +- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/flashbotsextra/database.go b/flashbotsextra/database.go index 3700eb20fe..e5146638a8 100644 --- a/flashbotsextra/database.go +++ b/flashbotsextra/database.go @@ -93,12 +93,12 @@ func Min(l int, r int) int { return r } -func (ds *DatabaseService) getBundleIds(ctx context.Context, blockNumber uint64, bundles []types.SimulatedBundle) (map[string]uint64, error) { +func (ds *DatabaseService) getBundleIds(ctx context.Context, blockNumber uint64, bundles []uuidBundle) (map[uuid.UUID]uint64, error) { if len(bundles) == 0 { return nil, nil } - bundleIdsMap := make(map[string]uint64, len(bundles)) + bundleIdsMap := make(map[uuid.UUID]uint64, len(bundles)) // Batch by 500 requestsToMake := [][]string{make([]string, 0, Min(500, len(bundles)))} @@ -108,7 +108,7 @@ func (ds *DatabaseService) getBundleIds(ctx context.Context, blockNumber uint64, cRequestInd += 1 requestsToMake = append(requestsToMake, make([]string, 0, Min(500, len(bundles)-i))) } - requestsToMake[cRequestInd] = append(requestsToMake[cRequestInd], bundle.OriginalBundle.Hash.String()) + requestsToMake[cRequestInd] = append(requestsToMake[cRequestInd], bundle.SimulatedBundle.OriginalBundle.Hash.String()) } for _, request := range requestsToMake { @@ -128,8 +128,19 @@ func (ds *DatabaseService) getBundleIds(ctx context.Context, blockNumber uint64, if err != nil { return nil, err } + RowLoop: for _, row := range queryRes { - bundleIdsMap[row.BundleHash] = row.Id + for _, b := range bundles { + // if UUID agree it's same exact bundle we stop searching + if b.UUID == row.BundleUUID { + bundleIdsMap[b.UUID] = row.Id + continue RowLoop + } + // we can have multiple bundles with same hash eventually, so we fall back on getting row with same hash + if b.SimulatedBundle.OriginalBundle.Hash.String() == row.BundleHash { + bundleIdsMap[b.UUID] = row.Id + } + } } } @@ -137,24 +148,23 @@ func (ds *DatabaseService) getBundleIds(ctx context.Context, blockNumber uint64, } // TODO: cache locally for current block! -func (ds *DatabaseService) getBundleIdsAndInsertMissingBundles(ctx context.Context, blockNumber uint64, bundles []types.SimulatedBundle) (map[string]uint64, error) { +func (ds *DatabaseService) getBundleIdsAndInsertMissingBundles(ctx context.Context, blockNumber uint64, bundles []uuidBundle) (map[uuid.UUID]uint64, error) { bundleIdsMap, err := ds.getBundleIds(ctx, blockNumber, bundles) if err != nil { return nil, err } - toRetry := []types.SimulatedBundle{} + toRetry := make([]uuidBundle, 0) for _, bundle := range bundles { - bundleHashString := bundle.OriginalBundle.Hash.String() - if _, found := bundleIdsMap[bundleHashString]; found { + if _, found := bundleIdsMap[bundle.UUID]; found { continue } var bundleId uint64 - missingBundleData := SimulatedBundleToDbBundle(&bundle) // nolint: gosec + missingBundleData := SimulatedBundleToDbBundle(&bundle.SimulatedBundle) // nolint: gosec err = ds.insertMissingBundleStmt.GetContext(ctx, &bundleId, missingBundleData) // not using the tx as it relies on the unique constraint! if err == nil { - bundleIdsMap[bundleHashString] = bundleId + bundleIdsMap[bundle.UUID] = bundleId } else if err == sql.ErrNoRows /* conflict, someone else inserted the bundle before we could */ { toRetry = append(toRetry, bundle) } else { @@ -215,7 +225,7 @@ func (ds *DatabaseService) insertBuildBlockBundleIds(tx *sqlx.Tx, ctx context.Co return err } -func (ds *DatabaseService) insertAllBlockBundleIds(tx *sqlx.Tx, ctx context.Context, blockId uint64, bundleIdsMap map[string]uint64) error { +func (ds *DatabaseService) insertAllBlockBundleIds(tx *sqlx.Tx, ctx context.Context, blockId uint64, bundleIdsMap map[uuid.UUID]uint64) error { if len(bundleIdsMap) == 0 { return nil } @@ -246,14 +256,30 @@ func (ds *DatabaseService) insertUsedSBundleIds(tx *sqlx.Tx, ctx context.Context return err } +type uuidBundle struct { + SimulatedBundle types.SimulatedBundle + UUID uuid.UUID +} + func (ds *DatabaseService) ConsumeBuiltBlock(block *types.Block, blockValue *big.Int, ordersClosedAt time.Time, sealedAt time.Time, commitedBundles []types.SimulatedBundle, allBundles []types.SimulatedBundle, usedSbundles []types.UsedSBundle, bidTrace *apiv1.BidTrace) { + + var allUUIDBundles = make([]uuidBundle, 0, len(allBundles)) + for _, bundle := range allBundles { + allUUIDBundles = append(allUUIDBundles, uuidBundle{bundle, bundle.OriginalBundle.ComputeUUID()}) + } + + var commitedUUIDBundles = make([]uuidBundle, 0, len(commitedBundles)) + for _, bundle := range commitedBundles { + commitedUUIDBundles = append(commitedUUIDBundles, uuidBundle{bundle, bundle.OriginalBundle.ComputeUUID()}) + } + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) defer cancel() - - bundleIdsMap, err := ds.getBundleIdsAndInsertMissingBundles(ctx, block.NumberU64(), allBundles) + + bundleIdsMap, err := ds.getBundleIdsAndInsertMissingBundles(ctx, block.NumberU64(), allUUIDBundles) if err != nil { log.Error("could not insert bundles", "err", err) } @@ -272,8 +298,8 @@ func (ds *DatabaseService) ConsumeBuiltBlock(block *types.Block, blockValue *big } commitedBundlesIds := make([]uint64, 0, len(commitedBundles)) - for _, bundle := range commitedBundles { - if id, found := bundleIdsMap[bundle.OriginalBundle.Hash.String()]; found { + for _, bundle := range commitedUUIDBundles { + if id, found := bundleIdsMap[bundle.UUID]; found { commitedBundlesIds = append(commitedBundlesIds, id) } } diff --git a/flashbotsextra/database_test.go b/flashbotsextra/database_test.go index 232ab4c3b1..8dd16dd218 100644 --- a/flashbotsextra/database_test.go +++ b/flashbotsextra/database_test.go @@ -151,7 +151,7 @@ func TestDatabaseBlockInsertion(t *testing.T) { require.Equal(t, sealedAt.Truncate(time.Millisecond), dbBlock.SealedAt.UTC().Truncate(time.Millisecond)) var bundles []DbBundle - ds.db.Select(&bundles, "select bundle_hash, param_signed_txs, param_block_number, param_timestamp, param_reverting_tx_hashes, coinbase_diff, total_gas_used, state_block_number, gas_fees, eth_sent_to_coinbase from bundles order by param_timestamp") + ds.db.Select(&bundles, "select bundle_hash, param_signed_txs, param_block_number, param_timestamp, param_reverting_tx_hashes, coinbase_diff, total_gas_used, state_block_number, gas_fees, eth_sent_to_coinbase, bundle_uuid from bundles order by param_timestamp") require.Len(t, bundles, 4) require.Equal(t, []DbBundle{SimulatedBundleToDbBundle(&simBundle1), SimulatedBundleToDbBundle(&simBundle2), SimulatedBundleToDbBundle(&simBundle3), SimulatedBundleToDbBundle(&simBundle4)}, bundles) From 0d6d2a47c04cc848b64803828d5996ba2b288adf Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Tue, 5 Sep 2023 13:10:46 +0300 Subject: [PATCH 3/8] add bundle uuid hash test --- flashbotsextra/database.go | 1 - flashbotsextra/database_test.go | 46 +++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/flashbotsextra/database.go b/flashbotsextra/database.go index e5146638a8..20bc209d82 100644 --- a/flashbotsextra/database.go +++ b/flashbotsextra/database.go @@ -265,7 +265,6 @@ func (ds *DatabaseService) ConsumeBuiltBlock(block *types.Block, blockValue *big commitedBundles []types.SimulatedBundle, allBundles []types.SimulatedBundle, usedSbundles []types.UsedSBundle, bidTrace *apiv1.BidTrace) { - var allUUIDBundles = make([]uuidBundle, 0, len(allBundles)) for _, bundle := range allBundles { allUUIDBundles = append(allUUIDBundles, uuidBundle{bundle, bundle.OriginalBundle.ComputeUUID()}) diff --git a/flashbotsextra/database_test.go b/flashbotsextra/database_test.go index 8dd16dd218..e1f18ac456 100644 --- a/flashbotsextra/database_test.go +++ b/flashbotsextra/database_test.go @@ -173,3 +173,49 @@ func TestDatabaseBlockInsertion(t *testing.T) { Inserted: usedSbundle.Success, }, usedSbundles[0]) } + +func simpleTx(nonce uint64) *types.Transaction { + value := big.NewInt(1000000000000000) // in wei (0.001 eth) + gasLimit := uint64(21000) // in units + gasPrice := big.NewInt(1000000000) + + toAddress := common.HexToAddress("0x7777492a736CD894Cb12DFE5e944047499AEF7a0") + var data []byte + return types.NewTx(&types.LegacyTx{ + Nonce: nonce, + To: &toAddress, + Value: value, + Gas: gasLimit, + GasPrice: gasPrice, + Data: data, + }) +} + +func TestBundleUUIDHash(t *testing.T) { + tx1 := simpleTx(1) + tx2 := simpleTx(2) + bts1, err := tx1.MarshalBinary() + require.Nil(t, err) + bts2, err := tx2.MarshalBinary() + require.Nil(t, err) + _, _ = bts1, bts2 + t.Run("no reverts", func(t *testing.T) { + b := types.MevBundle{ + BlockNumber: big.NewInt(1), + Hash: common.HexToHash("0x135a7f22459b2102d51de2d6704512a03e1e2d2059c34bcbb659f4ba65e9f92c"), + } + + require.Equal(t, "82624e95-741e-5a60-9198-b2f7b2ed973f", b.ComputeUUID().String()) + }) + t.Run("one revert", func(t *testing.T) { + b := types.MevBundle{ + BlockNumber: big.NewInt(1), + Hash: common.HexToHash("0x135a7f22459b2102d51de2d6704512a03e1e2d2059c34bcbb659f4ba65e9f92c"), + RevertingTxHashes: []common.Hash{ + tx1.Hash(), + }, + } + + require.Equal(t, "19f6e9a2-04a1-5616-9bb3-5c9f81653e4f", b.ComputeUUID().String()) + }) +} From 2dc2e0e393713f320620984ab18895b0b28d63d6 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Tue, 12 Sep 2023 09:52:29 +0200 Subject: [PATCH 4/8] change to sha256 --- core/types/transaction.go | 4 ++-- flashbotsextra/database_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/core/types/transaction.go b/core/types/transaction.go index 91b75b438e..9fc95e3df5 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -19,6 +19,7 @@ package types import ( "bytes" "container/heap" + "crypto/sha256" "encoding/binary" "errors" "io" @@ -32,7 +33,6 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/google/uuid" - "golang.org/x/crypto/sha3" ) var ( @@ -801,7 +801,7 @@ func (b *MevBundle) UniquePayload() []byte { } func (b *MevBundle) ComputeUUID() uuid.UUID { - return uuid.NewHash(sha3.NewLegacyKeccak256(), uuid.Nil, b.UniquePayload(), 5) + return uuid.NewHash(sha256.New(), uuid.Nil, b.UniquePayload(), 5) } func (b *MevBundle) RevertingHash(hash common.Hash) bool { diff --git a/flashbotsextra/database_test.go b/flashbotsextra/database_test.go index e1f18ac456..f05c2a2da9 100644 --- a/flashbotsextra/database_test.go +++ b/flashbotsextra/database_test.go @@ -205,7 +205,7 @@ func TestBundleUUIDHash(t *testing.T) { Hash: common.HexToHash("0x135a7f22459b2102d51de2d6704512a03e1e2d2059c34bcbb659f4ba65e9f92c"), } - require.Equal(t, "82624e95-741e-5a60-9198-b2f7b2ed973f", b.ComputeUUID().String()) + require.Equal(t, "5171315f-6ba4-52b2-866e-e2390d422d81", b.ComputeUUID().String()) }) t.Run("one revert", func(t *testing.T) { b := types.MevBundle{ @@ -216,6 +216,6 @@ func TestBundleUUIDHash(t *testing.T) { }, } - require.Equal(t, "19f6e9a2-04a1-5616-9bb3-5c9f81653e4f", b.ComputeUUID().String()) + require.Equal(t, "49dada39-6db2-500e-ae59-6cc18b2c19e0", b.ComputeUUID().String()) }) } From 903df5415c2aae5238c8f132443f179c6e3978e0 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Tue, 12 Sep 2023 13:25:52 +0200 Subject: [PATCH 5/8] align replacement uuid logic with new bundle_uuid field --- core/txpool/txpool.go | 13 +++++++++++++ core/types/transaction.go | 1 + flashbotsextra/database.go | 3 ++- flashbotsextra/database_test.go | 1 - flashbotsextra/database_types.go | 1 + 5 files changed, 17 insertions(+), 2 deletions(-) diff --git a/core/txpool/txpool.go b/core/txpool/txpool.go index eb591fcc72..ec2a010b46 100644 --- a/core/txpool/txpool.go +++ b/core/txpool/txpool.go @@ -645,6 +645,7 @@ func resolveCancellableBundles(lubCh chan []types.LatestUuidBundle, errCh chan e log.Trace("Processing uuid bundles", "uuidBundles", uuidBundles) lubs := <-lubCh +LubLoop: for _, lub := range lubs { ubk := uuidBundleKey{lub.Uuid, lub.SigningAddress} bundles, found := uuidBundles[ubk] @@ -652,6 +653,18 @@ func resolveCancellableBundles(lubCh chan []types.LatestUuidBundle, errCh chan e log.Trace("missing uuid bundle", "ubk", ubk) continue } + + // If lub has bundle_uuid set, and we can find corresponding bundle we prefer it, if not we fallback to bundle_hash equivalence + if lub.BundleUUID != types.EmptyUUID { + for _, bundle := range bundles { + if bundle.ComputeUUID() == lub.BundleUUID { + log.Trace("adding uuid bundle", "bundle hash", bundle.Hash.String(), "lub", lub) + currentCancellableBundles = append(currentCancellableBundles, bundle) + continue LubLoop + } + } + } + for _, bundle := range bundles { if bundle.Hash == lub.BundleHash { log.Trace("adding uuid bundle", "bundle hash", bundle.Hash.String(), "lub", lub) diff --git a/core/types/transaction.go b/core/types/transaction.go index 9fc95e3df5..a58814b2bd 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -774,6 +774,7 @@ type LatestUuidBundle struct { Uuid uuid.UUID SigningAddress common.Address BundleHash common.Hash + BundleUUID uuid.UUID } type MevBundle struct { diff --git a/flashbotsextra/database.go b/flashbotsextra/database.go index 20bc209d82..1a27b2ac1c 100644 --- a/flashbotsextra/database.go +++ b/flashbotsextra/database.go @@ -72,7 +72,7 @@ func NewDatabaseService(postgresDSN string) (*DatabaseService, error) { return nil, err } - fetchGetLatestUuidBundlesStmt, err := db.PrepareNamed("select replacement_uuid, signing_address, bundle_hash from latest_uuid_bundle where target_block_number = :target_block_number") + fetchGetLatestUuidBundlesStmt, err := db.PrepareNamed("select replacement_uuid, signing_address, bundle_hash, bundle_uuid from latest_uuid_bundle where target_block_number = :target_block_number") if err != nil { return nil, err } @@ -353,6 +353,7 @@ func (ds *DatabaseService) GetLatestUuidBundles(ctx context.Context, blockNum in Uuid: dbLub.Uuid, SigningAddress: common.HexToAddress(dbLub.SigningAddress), BundleHash: common.HexToHash(dbLub.BundleHash), + BundleUUID: dbLub.BundleUUID, }) } return latestBundles, nil diff --git a/flashbotsextra/database_test.go b/flashbotsextra/database_test.go index f05c2a2da9..a9c2c79337 100644 --- a/flashbotsextra/database_test.go +++ b/flashbotsextra/database_test.go @@ -17,7 +17,6 @@ func TestDatabaseBlockInsertion(t *testing.T) { if dsn == "" { t.Skip() } - ds, err := NewDatabaseService(dsn) require.NoError(t, err) diff --git a/flashbotsextra/database_types.go b/flashbotsextra/database_types.go index 74c51b60f7..c034d6cfe8 100644 --- a/flashbotsextra/database_types.go +++ b/flashbotsextra/database_types.go @@ -59,6 +59,7 @@ type DbLatestUuidBundle struct { Uuid uuid.UUID `db:"replacement_uuid"` SigningAddress string `db:"signing_address"` BundleHash string `db:"bundle_hash"` + BundleUUID uuid.UUID `db:"bundle_uuid"` } type blockAndBundleId struct { From f8fcf323a3db37bea0d335f90a1015c6d4662665 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Thu, 14 Sep 2023 15:55:29 +0200 Subject: [PATCH 6/8] add latest_uuid/bundle_uuid test --- core/txpool/txpool2_test.go | 78 +++++++++++++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/core/txpool/txpool2_test.go b/core/txpool/txpool2_test.go index 6d84975d83..3211cfd8dd 100644 --- a/core/txpool/txpool2_test.go +++ b/core/txpool/txpool2_test.go @@ -17,6 +17,7 @@ package txpool import ( "crypto/ecdsa" + "fmt" "math/big" "testing" @@ -26,6 +27,8 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/event" + "github.com/google/uuid" + "github.com/stretchr/testify/require" ) func pricedValuedTransaction(nonce uint64, value int64, gaslimit uint64, gasprice *big.Int, key *ecdsa.PrivateKey) *types.Transaction { @@ -210,3 +213,78 @@ func TestTransactionZAttack(t *testing.T) { newIvPending, ivPending, pool.config.GlobalSlots, newQueued) } } + +// Tests that cancellable bundle prefers latest with the same bundle_uuid, but fallbacks to bundle_hash equality +func TestCancellableBundles(t *testing.T) { + mb1 := types.MevBundle{ + Txs: nil, + BlockNumber: big.NewInt(1), + Uuid: uuid.MustParse("2fa47a9c-1eb2-4189-b1b0-d79bf2d0fc83"), + SigningAddress: common.HexToAddress("0x1"), + MinTimestamp: 0, + MaxTimestamp: 0, + RevertingTxHashes: []common.Hash{common.HexToHash("0x111")}, + Hash: common.HexToHash("0x2"), + } + muid1 := mb1.ComputeUUID() + + mb2 := types.MevBundle{ + Txs: nil, + BlockNumber: big.NewInt(1), + Uuid: uuid.MustParse("2fa47a9c-1eb2-4189-b1b0-d79bf2d0fc83"), + SigningAddress: common.HexToAddress("0x1"), + MinTimestamp: 0, + MaxTimestamp: 0, + RevertingTxHashes: nil, + Hash: common.HexToHash("0x2"), + } + muid2 := mb2.ComputeUUID() + _ = muid2 + + mb3 := types.MevBundle{ + Txs: nil, + BlockNumber: big.NewInt(1), + Uuid: uuid.MustParse("e2b1132f-7948-4227-aac4-041e9192110a"), + SigningAddress: common.HexToAddress("0x1"), + MinTimestamp: 0, + MaxTimestamp: 0, + RevertingTxHashes: nil, + Hash: common.HexToHash("0x3"), + } + + lubCh := make(chan []types.LatestUuidBundle, 1) + errCh := make(chan error, 1) + go func() { + lub1 := types.LatestUuidBundle{ + Uuid: uuid.MustParse("2fa47a9c-1eb2-4189-b1b0-d79bf2d0fc83"), + SigningAddress: common.HexToAddress("0x1"), + BundleHash: common.HexToHash("0x2"), + BundleUUID: muid1, + } + lub2 := types.LatestUuidBundle{ + Uuid: uuid.MustParse("e2b1132f-7948-4227-aac4-041e9192110a"), + SigningAddress: common.HexToAddress("0x1"), + BundleHash: common.HexToHash("0x3"), + } + lubCh <- []types.LatestUuidBundle{lub1, lub2} + errCh <- nil + }() + + uuidBundles := make(map[uuidBundleKey][]types.MevBundle) + firstUuidBK := uuidBundleKey{ + Uuid: uuid.MustParse("2fa47a9c-1eb2-4189-b1b0-d79bf2d0fc83"), + SigningAddress: common.HexToAddress("0x1"), + } + secondUuidBK := uuidBundleKey{ + Uuid: uuid.MustParse("e2b1132f-7948-4227-aac4-041e9192110a"), + SigningAddress: common.HexToAddress("0x1"), + } + + uuidBundles[firstUuidBK] = []types.MevBundle{mb1, mb2} + uuidBundles[secondUuidBK] = []types.MevBundle{mb3} + + mbs := resolveCancellableBundles(lubCh, errCh, uuidBundles) + require.Equal(t, mbs[0], mb1) + require.Equal(t, mbs[1], mb3) + fmt.Println(mbs) +} From 52206b888252210fcb8a76402532b83ba4836f00 Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Thu, 14 Sep 2023 17:20:19 +0200 Subject: [PATCH 7/8] fetch bundle_uuid for consistency --- flashbotsextra/database.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flashbotsextra/database.go b/flashbotsextra/database.go index 1a27b2ac1c..aee493a7ec 100644 --- a/flashbotsextra/database.go +++ b/flashbotsextra/database.go @@ -67,7 +67,7 @@ func NewDatabaseService(postgresDSN string) (*DatabaseService, error) { return nil, err } - fetchPrioBundlesStmt, err := db.PrepareNamed("select bundle_hash, param_signed_txs, param_block_number, param_timestamp, received_timestamp, param_reverting_tx_hashes, coinbase_diff, total_gas_used, state_block_number, gas_fees, eth_sent_to_coinbase from bundles where is_high_prio = :is_high_prio and coinbase_diff*1e18/total_gas_used > 1000000000 and param_block_number = :param_block_number order by coinbase_diff/total_gas_used DESC limit :limit") + fetchPrioBundlesStmt, err := db.PrepareNamed("select bundle_hash, param_signed_txs, param_block_number, param_timestamp, received_timestamp, param_reverting_tx_hashes, coinbase_diff, total_gas_used, state_block_number, gas_fees, eth_sent_to_coinbase, bundle_uuid from bundles where is_high_prio = :is_high_prio and coinbase_diff*1e18/total_gas_used > 1000000000 and param_block_number = :param_block_number order by coinbase_diff/total_gas_used DESC limit :limit") if err != nil { return nil, err } From 9777f5285bfb04a8a1ef1a6017c9d2ef681629ff Mon Sep 17 00:00:00 2001 From: Tymur Khrushchov Date: Tue, 19 Sep 2023 14:52:56 +0200 Subject: [PATCH 8/8] Filter out unique bundle_ids prior to inserting --- flashbotsextra/database.go | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/flashbotsextra/database.go b/flashbotsextra/database.go index aee493a7ec..80e8170607 100644 --- a/flashbotsextra/database.go +++ b/flashbotsextra/database.go @@ -225,13 +225,13 @@ func (ds *DatabaseService) insertBuildBlockBundleIds(tx *sqlx.Tx, ctx context.Co return err } -func (ds *DatabaseService) insertAllBlockBundleIds(tx *sqlx.Tx, ctx context.Context, blockId uint64, bundleIdsMap map[uuid.UUID]uint64) error { - if len(bundleIdsMap) == 0 { +func (ds *DatabaseService) insertAllBlockBundleIds(tx *sqlx.Tx, ctx context.Context, blockId uint64, bundleIds []uint64) error { + if len(bundleIds) == 0 { return nil } - toInsert := make([]blockAndBundleId, 0, len(bundleIdsMap)) - for _, bundleId := range bundleIdsMap { + toInsert := make([]blockAndBundleId, 0, len(bundleIds)) + for _, bundleId := range bundleIds { toInsert = append(toInsert, blockAndBundleId{blockId, bundleId}) } @@ -310,10 +310,20 @@ func (ds *DatabaseService) ConsumeBuiltBlock(block *types.Block, blockValue *big return } - err = ds.insertAllBlockBundleIds(tx, ctx, blockId, bundleIdsMap) + var uniqueBundleIDs = make(map[uint64]struct{}) + var allBundleIds []uint64 + // we need to filter out duplicates while we still have unique constraint on bundle_hash+block_number which leads to data discrepancies + for _, v := range bundleIdsMap { + if _, ok := uniqueBundleIDs[v]; ok { + continue + } + uniqueBundleIDs[v] = struct{}{} + allBundleIds = append(allBundleIds, v) + } + err = ds.insertAllBlockBundleIds(tx, ctx, blockId, allBundleIds) if err != nil { tx.Rollback() - log.Error("could not insert built block all bundles", "err", err) + log.Error("could not insert built block all bundles", "err", err, "block", block.NumberU64(), "commitedBundles", commitedBundlesIds) return }