From b324762236dc1d03bc2ca5121ad0db6e1a69f090 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Mon, 21 Nov 2022 01:16:07 +0300
Subject: [PATCH 01/26] implement a timeline of highest blocks

---
 feeds/timeline/timeline.go      | 99 +++++++++++++++++++++++++++++++++
 feeds/timeline/timeline_test.go | 52 +++++++++++++++++
 2 files changed, 151 insertions(+)
 create mode 100644 feeds/timeline/timeline.go
 create mode 100644 feeds/timeline/timeline_test.go

diff --git a/feeds/timeline/timeline.go b/feeds/timeline/timeline.go
new file mode 100644
index 00000000..3b514ee2
--- /dev/null
+++ b/feeds/timeline/timeline.go
@@ -0,0 +1,99 @@
+package timeline
+
+import (
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/common/hexutil"
+	"github.com/forta-network/forta-core-go/domain"
+	log "github.com/sirupsen/logrus"
+)
+
+// BlockTimeline implements a block feed subscriber and keeps track of the
+// latest block in every minute.
+type BlockTimeline struct {
+	maxMinutes int
+	minutes    []*Minute
+	mu         sync.RWMutex
+}
+
+// Minute represents a minute in a chain.
+type Minute struct {
+	HighestBlockNumber uint64
+	Timestamp          time.Time
+}
+
+// NewBlockTimeline creates a new block timeline.
+func NewBlockTimeline(maxMinutes int) *BlockTimeline {
+	bt := &BlockTimeline{
+		maxMinutes: maxMinutes,
+	}
+
+	go bt.cleanup()
+
+	return bt
+}
+
+func (bt *BlockTimeline) cleanup() {
+	ticker := time.NewTicker(time.Minute)
+	for {
+		<-ticker.C
+		bt.doCleanup()
+	}
+}
+
+func (bt *BlockTimeline) doCleanup() {
+	bt.mu.Lock()
+	defer bt.mu.Unlock()
+
+	currSize := len(bt.minutes)
+	if currSize > bt.maxMinutes {
+		extra := currSize - bt.maxMinutes
+		bt.minutes = bt.minutes[extra:] // release oldest
+	}
+}
+
+// HandleBlock handles a block incoming from block feed.
+func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
+	bt.mu.Lock()
+	defer bt.mu.Unlock()
+
+	blockTs, err := evt.Block.GetTimestamp()
+	if err != nil {
+		log.WithError(err).Error("failed to get block timestamp")
+		return nil
+	}
+	blockMinuteTs := blockTs.Truncate(time.Minute)
+	blockNum, err := hexutil.DecodeUint64(evt.Block.Number)
+	if err != nil {
+		log.WithError(err).Error("failed to decode block number")
+	}
+	for _, minute := range bt.minutes {
+		if minute.Timestamp.Equal(blockMinuteTs) {
+			if blockNum > minute.HighestBlockNumber {
+				minute.HighestBlockNumber = blockNum
+			}
+			return nil // we found the minute
+		}
+	}
+	// could not find the minute - append it
+	bt.minutes = append(bt.minutes, &Minute{
+		HighestBlockNumber: blockNum,
+		Timestamp:          blockMinuteTs,
+	})
+	return nil
+}
+
+// GetHighestBlockNumber returns the highest block number within the same minute of the given timestamp.
+func (bt *BlockTimeline) GetHighestBlockNumber(ts time.Time) (uint64, bool) {
+	bt.mu.RLock()
+	defer bt.mu.RUnlock()
+
+	ts = ts.Truncate(time.Minute)
+	for _, minute := range bt.minutes {
+		if minute.Timestamp.Equal(ts) {
+			return minute.HighestBlockNumber, true
+		}
+	}
+	return 0, false
+}
diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
new file mode 100644
index 00000000..65ea78bc
--- /dev/null
+++ b/feeds/timeline/timeline_test.go
@@ -0,0 +1,52 @@
+package timeline
+
+import (
+	"testing"
+
+	"github.com/forta-network/forta-core-go/domain"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTimeline(t *testing.T) {
+	r := require.New(t)
+
+	blockTimeline := &BlockTimeline{
+		maxMinutes: 2,
+	}
+
+	// add first minute block number
+	blockTimeline.HandleBlock(blockForTimestamp("0x1000000000", "0x1"))
+	// replace the first minute number
+	blockTimeline.HandleBlock(blockForTimestamp("0x1000000000", "0x2"))
+	// add new one for the next minute
+	blockTimeline.HandleBlock(blockForTimestamp("0x2000000000", "0x3"))
+	// replace the second minute number
+	blockTimeline.HandleBlock(blockForTimestamp("0x2000000000", "0x4"))
+	// replace the first minute number
+	blockTimeline.HandleBlock(blockForTimestamp("0x1000000000", "0x5"))
+	// replace the second minute number
+	blockTimeline.HandleBlock(blockForTimestamp("0x2000000000", "0x6"))
+	// add a third minute
+	blockTimeline.HandleBlock(blockForTimestamp("0x3000000000", "0x7"))
+
+	// verify state
+	r.EqualValues(5, blockTimeline.minutes[0].HighestBlockNumber)
+	r.EqualValues(6, blockTimeline.minutes[1].HighestBlockNumber)
+	r.EqualValues(7, blockTimeline.minutes[2].HighestBlockNumber)
+
+	// cleanup should remove the first minute because of the max minutes num
+	blockTimeline.doCleanup()
+
+	// verify state
+	r.EqualValues(6, blockTimeline.minutes[0].HighestBlockNumber)
+	r.EqualValues(7, blockTimeline.minutes[1].HighestBlockNumber)
+}
+
+func blockForTimestamp(ts, blockNumber string) *domain.BlockEvent {
+	return &domain.BlockEvent{
+		Block: &domain.Block{
+			Timestamp: ts,
+			Number:    blockNumber,
+		},
+	}
+}

From 6362833becf97eff9eea26efdb63d9bc19e33d99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Mon, 21 Nov 2022 03:29:41 +0300
Subject: [PATCH 02/26] add ability to disable logs

---
 feeds/blocks.go | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/feeds/blocks.go b/feeds/blocks.go
index 1b838de6..97d68783 100644
--- a/feeds/blocks.go
+++ b/feeds/blocks.go
@@ -35,6 +35,7 @@ type blockFeed struct {
 	cache       utils.Cache
 	chainID     *big.Int
 	tracing     bool
+	logs        bool
 	started     bool
 	rateLimit   *time.Ticker
 	maxBlockAge *time.Duration
@@ -52,6 +53,7 @@ type BlockFeedConfig struct {
 	ChainID             *big.Int
 	RateLimit           *time.Ticker
 	Tracing             bool
+	DisableLogs         bool
 	SkipBlocksOlderThan *time.Duration
 }
 
@@ -237,10 +239,13 @@ func (bf *blockFeed) forEachBlock() error {
 			traces = nil
 		}
 
-		logs, err := bf.logsForBlock(blockNumToAnalyze)
-		if err != nil {
-			logger.WithError(err).Errorf("error getting logs for block")
-			continue
+		var logs []domain.LogEntry
+		if bf.logs {
+			logs, err = bf.logsForBlock(blockNumToAnalyze)
+			if err != nil {
+				logger.WithError(err).Errorf("error getting logs for block")
+				continue
+			}
 		}
 
 		blockTs, err := block.GetTimestamp()
@@ -314,6 +319,7 @@ func NewBlockFeed(ctx context.Context, client ethereum.Client, traceClient ether
 		cache:       utils.NewCache(10000),
 		chainID:     cfg.ChainID,
 		tracing:     cfg.Tracing,
+		logs:        !cfg.DisableLogs,
 		rateLimit:   cfg.RateLimit,
 		maxBlockAge: cfg.SkipBlocksOlderThan,
 	}

From 1ab11b77cfbadfb95f7c8c5a7d03a54a6b6a59a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Mon, 16 Oct 2023 16:16:10 +0300
Subject: [PATCH 03/26] add functionality to calculate lag in a given minute

---
 feeds/timeline/timeline.go      | 54 +++++++++++++++++++++++++------
 feeds/timeline/timeline_test.go | 57 +++++++++++++++++++++++++--------
 2 files changed, 88 insertions(+), 23 deletions(-)

diff --git a/feeds/timeline/timeline.go b/feeds/timeline/timeline.go
index 3b514ee2..deb00603 100644
--- a/feeds/timeline/timeline.go
+++ b/feeds/timeline/timeline.go
@@ -12,9 +12,10 @@ import (
 // BlockTimeline implements a block feed subscriber and keeps track of the
 // latest block in every minute.
 type BlockTimeline struct {
-	maxMinutes int
-	minutes    []*Minute
-	mu         sync.RWMutex
+	maxMinutes   int
+	blockMinutes []*Minute // when the block was produced
+	localMinutes []*Minute // when we handled the block
+	mu           sync.RWMutex
 }
 
 // Minute represents a minute in a chain.
@@ -46,10 +47,16 @@ func (bt *BlockTimeline) doCleanup() {
 	bt.mu.Lock()
 	defer bt.mu.Unlock()
 
-	currSize := len(bt.minutes)
+	currSize := len(bt.blockMinutes)
 	if currSize > bt.maxMinutes {
 		extra := currSize - bt.maxMinutes
-		bt.minutes = bt.minutes[extra:] // release oldest
+		bt.blockMinutes = bt.blockMinutes[extra:] // release oldest
+	}
+
+	currSize = len(bt.localMinutes)
+	if currSize > bt.maxMinutes {
+		extra := currSize - bt.maxMinutes
+		bt.localMinutes = bt.localMinutes[extra:] // release oldest
 	}
 }
 
@@ -58,6 +65,7 @@ func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
 	bt.mu.Lock()
 	defer bt.mu.Unlock()
 
+	localMinuteTs := time.Now().Truncate(time.Minute)
 	blockTs, err := evt.Block.GetTimestamp()
 	if err != nil {
 		log.WithError(err).Error("failed to get block timestamp")
@@ -68,7 +76,7 @@ func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
 	if err != nil {
 		log.WithError(err).Error("failed to decode block number")
 	}
-	for _, minute := range bt.minutes {
+	for _, minute := range bt.blockMinutes {
 		if minute.Timestamp.Equal(blockMinuteTs) {
 			if blockNum > minute.HighestBlockNumber {
 				minute.HighestBlockNumber = blockNum
@@ -77,23 +85,49 @@ func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
 		}
 	}
 	// could not find the minute - append it
-	bt.minutes = append(bt.minutes, &Minute{
+	bt.blockMinutes = append(bt.blockMinutes, &Minute{
 		HighestBlockNumber: blockNum,
 		Timestamp:          blockMinuteTs,
 	})
+	bt.localMinutes = append(bt.localMinutes, &Minute{
+		HighestBlockNumber: blockNum,
+		Timestamp:          localMinuteTs,
+	})
 	return nil
 }
 
-// GetHighestBlockNumber returns the highest block number within the same minute of the given timestamp.
-func (bt *BlockTimeline) GetHighestBlockNumber(ts time.Time) (uint64, bool) {
+// GetGlobalHighest returns the global highest block number within the same minute of the given timestamp.
+func (bt *BlockTimeline) GetGlobalHighest(ts time.Time) (uint64, bool) {
+	return bt.getHighest(bt.blockMinutes, ts)
+}
+
+// GetLocalHighest returns the local highest block number within the same minute of the given timestamp.
+func (bt *BlockTimeline) GetLocalHighest(ts time.Time) (uint64, bool) {
+	return bt.getHighest(bt.localMinutes, ts)
+}
+
+func (bt *BlockTimeline) getHighest(minutes []*Minute, ts time.Time) (uint64, bool) {
 	bt.mu.RLock()
 	defer bt.mu.RUnlock()
 
 	ts = ts.Truncate(time.Minute)
-	for _, minute := range bt.minutes {
+	for _, minute := range minutes {
 		if minute.Timestamp.Equal(ts) {
 			return minute.HighestBlockNumber, true
 		}
 	}
 	return 0, false
 }
+
+// CalculateLag calculates the block number lag in a given minute.
+func (bt *BlockTimeline) CalculateLag(ts time.Time) (int64, bool) {
+	highestGlobal, ok := bt.GetGlobalHighest(ts)
+	if !ok {
+		return 0, false
+	}
+	highestLocal, ok := bt.GetLocalHighest(ts)
+	if !ok {
+		return 0, false
+	}
+	return int64(highestGlobal) - int64(highestLocal), true
+}
diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
index 65ea78bc..4abd232e 100644
--- a/feeds/timeline/timeline_test.go
+++ b/feeds/timeline/timeline_test.go
@@ -2,44 +2,75 @@ package timeline
 
 import (
 	"testing"
+	"time"
 
+	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/forta-network/forta-core-go/domain"
 	"github.com/stretchr/testify/require"
 )
 
-func TestTimeline(t *testing.T) {
+func TestTimeline_GlobalHighest(t *testing.T) {
 	r := require.New(t)
 
 	blockTimeline := &BlockTimeline{
 		maxMinutes: 2,
 	}
 
+	min1 := time.Now().UTC().Truncate(time.Minute)
+	min2 := min1.Add(time.Minute)
+	min1Ts := hexutil.EncodeUint64(uint64(min1.Unix()))
+	min2Ts := hexutil.EncodeUint64(uint64(min2.Unix()))
+	min3Ts := hexutil.EncodeUint64(uint64(min1.Add(time.Minute * 2).Unix()))
+
 	// add first minute block number
-	blockTimeline.HandleBlock(blockForTimestamp("0x1000000000", "0x1"))
+	blockTimeline.HandleBlock(blockForTimestamp(min1Ts, "0x1"))
 	// replace the first minute number
-	blockTimeline.HandleBlock(blockForTimestamp("0x1000000000", "0x2"))
+	blockTimeline.HandleBlock(blockForTimestamp(min1Ts, "0x2"))
 	// add new one for the next minute
-	blockTimeline.HandleBlock(blockForTimestamp("0x2000000000", "0x3"))
+	blockTimeline.HandleBlock(blockForTimestamp(min2Ts, "0x3"))
 	// replace the second minute number
-	blockTimeline.HandleBlock(blockForTimestamp("0x2000000000", "0x4"))
+	blockTimeline.HandleBlock(blockForTimestamp(min2Ts, "0x4"))
 	// replace the first minute number
-	blockTimeline.HandleBlock(blockForTimestamp("0x1000000000", "0x5"))
+	blockTimeline.HandleBlock(blockForTimestamp(min1Ts, "0x5"))
 	// replace the second minute number
-	blockTimeline.HandleBlock(blockForTimestamp("0x2000000000", "0x6"))
+	blockTimeline.HandleBlock(blockForTimestamp(min2Ts, "0x6"))
 	// add a third minute
-	blockTimeline.HandleBlock(blockForTimestamp("0x3000000000", "0x7"))
+	blockTimeline.HandleBlock(blockForTimestamp(min3Ts, "0x7"))
 
 	// verify state
-	r.EqualValues(5, blockTimeline.minutes[0].HighestBlockNumber)
-	r.EqualValues(6, blockTimeline.minutes[1].HighestBlockNumber)
-	r.EqualValues(7, blockTimeline.minutes[2].HighestBlockNumber)
+	r.EqualValues(5, blockTimeline.blockMinutes[0].HighestBlockNumber)
+	r.EqualValues(6, blockTimeline.blockMinutes[1].HighestBlockNumber)
+	r.EqualValues(7, blockTimeline.blockMinutes[2].HighestBlockNumber)
 
 	// cleanup should remove the first minute because of the max minutes num
 	blockTimeline.doCleanup()
 
 	// verify state
-	r.EqualValues(6, blockTimeline.minutes[0].HighestBlockNumber)
-	r.EqualValues(7, blockTimeline.minutes[1].HighestBlockNumber)
+	r.EqualValues(6, blockTimeline.blockMinutes[0].HighestBlockNumber)
+	r.EqualValues(7, blockTimeline.blockMinutes[1].HighestBlockNumber)
+
+	highestGlobal, ok := blockTimeline.GetGlobalHighest(min2)
+	r.True(ok)
+	r.Equal(uint64(6), highestGlobal)
+}
+
+func TestTimeline_CalculateLag(t *testing.T) {
+	r := require.New(t)
+
+	blockTimeline := &BlockTimeline{}
+
+	currMin := time.Now().UTC().Truncate(time.Minute)
+	blockTimeline.localMinutes = append(blockTimeline.localMinutes, &Minute{
+		Timestamp:          currMin,
+		HighestBlockNumber: 1,
+	})
+	blockTimeline.blockMinutes = append(blockTimeline.blockMinutes, &Minute{
+		Timestamp:          currMin,
+		HighestBlockNumber: 2,
+	})
+	lag, ok := blockTimeline.CalculateLag(currMin)
+	r.True(ok)
+	r.Equal(int64(1), lag)
 }
 
 func blockForTimestamp(ts, blockNumber string) *domain.BlockEvent {

From 34a40d63774c2fbae1adb8aab8e53e85b8c1a566 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Mon, 16 Oct 2023 21:13:01 +0300
Subject: [PATCH 04/26] add experiment as optional test

---
 feeds/timeline/timeline_test.go | 44 +++++++++++++++++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
index 4abd232e..8ed2e806 100644
--- a/feeds/timeline/timeline_test.go
+++ b/feeds/timeline/timeline_test.go
@@ -1,11 +1,17 @@
 package timeline
 
 import (
+	"context"
+	"fmt"
+	"math/big"
+	"os"
 	"testing"
 	"time"
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/forta-network/forta-core-go/domain"
+	"github.com/forta-network/forta-core-go/ethereum"
+	"github.com/forta-network/forta-core-go/feeds"
 	"github.com/stretchr/testify/require"
 )
 
@@ -81,3 +87,41 @@ func blockForTimestamp(ts, blockNumber string) *domain.BlockEvent {
 		},
 	}
 }
+
+func TestRealTimeLag(t *testing.T) {
+	if os.Getenv("TIMELINE_EXPERIMENT") != "1" {
+		return
+	}
+	r := require.New(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	ethClient, err := ethereum.NewStreamEthClient(ctx, "", os.Getenv("JSON_RPC_API"))
+	ethClient.SetRetryInterval(time.Second * 2)
+	r.NoError(err)
+	blockFeed, err := feeds.NewBlockFeed(ctx, ethClient, ethClient, feeds.BlockFeedConfig{
+		ChainID:     big.NewInt(137),
+		DisableLogs: true,
+	})
+	r.NoError(err)
+
+	blockTimeline := &BlockTimeline{}
+	errCh := blockFeed.Subscribe(func(evt *domain.BlockEvent) error {
+		return blockTimeline.HandleBlock(evt)
+	})
+
+	go blockFeed.Start()
+
+	go func() {
+		<-time.After(time.Minute * 4)
+		cancel()
+	}()
+
+	<-errCh
+
+	for _, minute := range blockTimeline.blockMinutes {
+		lag, ok := blockTimeline.CalculateLag(minute.Timestamp)
+		if !ok {
+			continue
+		}
+		fmt.Println(minute.Timestamp.Format(time.RFC3339), ":", lag)
+	}
+}

From 9f891eab0cb4f7200d22eb9d7dd825466ac70998 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Mon, 16 Oct 2023 21:19:38 +0300
Subject: [PATCH 05/26] add distance number per chain

---
 protocol/settings/chain.go | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/protocol/settings/chain.go b/protocol/settings/chain.go
index 364adc2f..0f3dd1e4 100644
--- a/protocol/settings/chain.go
+++ b/protocol/settings/chain.go
@@ -16,6 +16,10 @@ type ChainSettings struct {
 	DefaultOffset  int
 	SafeOffset     int
 	BlockThreshold int
+
+	// the expected lag (in block number difference) between the processed block
+	// and the chain highest at the end of every minute
+	DistanceToChainHighest int
 }
 
 // RateLimit is token bucket algorithm parameters.
@@ -39,6 +43,8 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     1,
 		BlockThreshold: 20,
+
+		DistanceToChainHighest: 4,
 	},
 	{
 		ChainID:             10,
@@ -50,6 +56,8 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     500,
 		BlockThreshold: 10000,
+
+		DistanceToChainHighest: 29,
 	},
 	{
 		ChainID:             56,
@@ -61,6 +69,8 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     3,
 		BlockThreshold: 50,
+
+		DistanceToChainHighest: 19,
 	},
 	{
 		ChainID:             137,
@@ -72,6 +82,8 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     4,
 		BlockThreshold: 70,
+
+		DistanceToChainHighest: 27,
 	},
 	{
 		ChainID:             250,
@@ -83,6 +95,8 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     5,
 		BlockThreshold: 100,
+
+		DistanceToChainHighest: 21,
 	},
 	{
 		ChainID:             42161,
@@ -94,6 +108,8 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     60,
 		BlockThreshold: 1200,
+
+		DistanceToChainHighest: 200,
 	},
 	{
 		ChainID:             43114,
@@ -105,6 +121,8 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     4,
 		BlockThreshold: 70,
+
+		DistanceToChainHighest: 26,
 	},
 }
 

From 151b354474abbce287781250fb78b8a69d0788d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Mon, 16 Oct 2023 21:20:36 +0300
Subject: [PATCH 06/26] reduce optimism threshold

---
 protocol/settings/chain.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/protocol/settings/chain.go b/protocol/settings/chain.go
index 0f3dd1e4..4d6c2a10 100644
--- a/protocol/settings/chain.go
+++ b/protocol/settings/chain.go
@@ -51,11 +51,11 @@ var allChainSettings = []ChainSettings{
 		Name:                "Optimism",
 		EnableTrace:         false,
 		JsonRpcRateLimiting: defaultRateLimiting,
-		InspectionInterval:  5000,
+		InspectionInterval:  100,
 
 		DefaultOffset:  0,
-		SafeOffset:     500,
-		BlockThreshold: 10000,
+		SafeOffset:     5,
+		BlockThreshold: 100,
 
 		DistanceToChainHighest: 29,
 	},

From ff76c5701a8f050c5173110de0e03ec861261058 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Mon, 16 Oct 2023 21:43:23 +0300
Subject: [PATCH 07/26] add method to check timeline size

---
 feeds/timeline/timeline.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/feeds/timeline/timeline.go b/feeds/timeline/timeline.go
index deb00603..16f5ad95 100644
--- a/feeds/timeline/timeline.go
+++ b/feeds/timeline/timeline.go
@@ -131,3 +131,11 @@ func (bt *BlockTimeline) CalculateLag(ts time.Time) (int64, bool) {
 	}
 	return int64(highestGlobal) - int64(highestLocal), true
 }
+
+// Size returns the minute count.
+func (bt *BlockTimeline) Size() int {
+	bt.mu.RLock()
+	defer bt.mu.RUnlock()
+
+	return len(bt.blockMinutes)
+}

From a321998e3b5c81d21bef7610178f5af834194101 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Tue, 17 Oct 2023 10:19:31 +0300
Subject: [PATCH 08/26] log delay

---
 feeds/timeline/timeline_test.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
index 8ed2e806..3ae40dd4 100644
--- a/feeds/timeline/timeline_test.go
+++ b/feeds/timeline/timeline_test.go
@@ -105,6 +105,9 @@ func TestRealTimeLag(t *testing.T) {
 
 	blockTimeline := &BlockTimeline{}
 	errCh := blockFeed.Subscribe(func(evt *domain.BlockEvent) error {
+		blockTs, _ := evt.Block.GetTimestamp()
+		delay := time.Since(*blockTs)
+		fmt.Println("delay:", delay)
 		return blockTimeline.HandleBlock(evt)
 	})
 

From 56cab57c0851410e6b74cbd88bc7a66f200205b7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Tue, 17 Oct 2023 10:25:23 +0300
Subject: [PATCH 09/26] return delay

---
 feeds/timeline/timeline.go | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/feeds/timeline/timeline.go b/feeds/timeline/timeline.go
index 16f5ad95..89292e92 100644
--- a/feeds/timeline/timeline.go
+++ b/feeds/timeline/timeline.go
@@ -15,6 +15,7 @@ type BlockTimeline struct {
 	maxMinutes   int
 	blockMinutes []*Minute // when the block was produced
 	localMinutes []*Minute // when we handled the block
+	delay        *time.Duration
 	mu           sync.RWMutex
 }
 
@@ -65,6 +66,10 @@ func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
 	bt.mu.Lock()
 	defer bt.mu.Unlock()
 
+	blockTs, _ := evt.Block.GetTimestamp()
+	delay := time.Since(*blockTs)
+	bt.delay = &delay
+
 	localMinuteTs := time.Now().Truncate(time.Minute)
 	blockTs, err := evt.Block.GetTimestamp()
 	if err != nil {
@@ -96,6 +101,16 @@ func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
 	return nil
 }
 
+func (bt *BlockTimeline) GetDelay() (time.Duration, bool) {
+	bt.mu.RLock()
+	defer bt.mu.RUnlock()
+
+	if bt.delay == nil {
+		return 0, false
+	}
+	return *bt.delay, true
+}
+
 // GetGlobalHighest returns the global highest block number within the same minute of the given timestamp.
 func (bt *BlockTimeline) GetGlobalHighest(ts time.Time) (uint64, bool) {
 	return bt.getHighest(bt.blockMinutes, ts)

From 1f88ce47dd8646c92828f8274ac95d1536e3bae5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Tue, 17 Oct 2023 11:51:30 +0300
Subject: [PATCH 10/26] fix the timeline construction

---
 feeds/timeline/timeline.go | 48 ++++++++++++++++++++++++++------------
 1 file changed, 33 insertions(+), 15 deletions(-)

diff --git a/feeds/timeline/timeline.go b/feeds/timeline/timeline.go
index 89292e92..f9e9afae 100644
--- a/feeds/timeline/timeline.go
+++ b/feeds/timeline/timeline.go
@@ -66,38 +66,56 @@ func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
 	bt.mu.Lock()
 	defer bt.mu.Unlock()
 
-	blockTs, _ := evt.Block.GetTimestamp()
-	delay := time.Since(*blockTs)
-	bt.delay = &delay
-
-	localMinuteTs := time.Now().Truncate(time.Minute)
 	blockTs, err := evt.Block.GetTimestamp()
 	if err != nil {
 		log.WithError(err).Error("failed to get block timestamp")
 		return nil
 	}
+	delay := time.Since(*blockTs)
+	bt.delay = &delay
+
+	localMinuteTs := time.Now().Truncate(time.Minute)
+
 	blockMinuteTs := blockTs.Truncate(time.Minute)
 	blockNum, err := hexutil.DecodeUint64(evt.Block.Number)
 	if err != nil {
 		log.WithError(err).Error("failed to decode block number")
 	}
+
+	var foundBlockMinute bool
 	for _, minute := range bt.blockMinutes {
 		if minute.Timestamp.Equal(blockMinuteTs) {
 			if blockNum > minute.HighestBlockNumber {
 				minute.HighestBlockNumber = blockNum
 			}
-			return nil // we found the minute
+			foundBlockMinute = true
+			break
 		}
 	}
-	// could not find the minute - append it
-	bt.blockMinutes = append(bt.blockMinutes, &Minute{
-		HighestBlockNumber: blockNum,
-		Timestamp:          blockMinuteTs,
-	})
-	bt.localMinutes = append(bt.localMinutes, &Minute{
-		HighestBlockNumber: blockNum,
-		Timestamp:          localMinuteTs,
-	})
+	if !foundBlockMinute {
+		bt.blockMinutes = append(bt.blockMinutes, &Minute{
+			HighestBlockNumber: blockNum,
+			Timestamp:          blockMinuteTs,
+		})
+	}
+
+	var foundLocalMinute bool
+	for _, minute := range bt.localMinutes {
+		if minute.Timestamp.Equal(localMinuteTs) {
+			if blockNum > minute.HighestBlockNumber {
+				minute.HighestBlockNumber = blockNum
+			}
+			foundLocalMinute = true
+			break
+		}
+	}
+	if !foundLocalMinute {
+		bt.localMinutes = append(bt.localMinutes, &Minute{
+			HighestBlockNumber: blockNum,
+			Timestamp:          localMinuteTs,
+		})
+	}
+
 	return nil
 }
 

From 81372b6383ee0e4920a2b90f90ed4e2bee2eda2e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Tue, 17 Oct 2023 12:03:29 +0300
Subject: [PATCH 11/26] remove distance stuff

---
 protocol/settings/chain.go | 18 ------------------
 1 file changed, 18 deletions(-)

diff --git a/protocol/settings/chain.go b/protocol/settings/chain.go
index 4d6c2a10..aa454824 100644
--- a/protocol/settings/chain.go
+++ b/protocol/settings/chain.go
@@ -16,10 +16,6 @@ type ChainSettings struct {
 	DefaultOffset  int
 	SafeOffset     int
 	BlockThreshold int
-
-	// the expected lag (in block number difference) between the processed block
-	// and the chain highest at the end of every minute
-	DistanceToChainHighest int
 }
 
 // RateLimit is token bucket algorithm parameters.
@@ -43,8 +39,6 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     1,
 		BlockThreshold: 20,
-
-		DistanceToChainHighest: 4,
 	},
 	{
 		ChainID:             10,
@@ -56,8 +50,6 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     5,
 		BlockThreshold: 100,
-
-		DistanceToChainHighest: 29,
 	},
 	{
 		ChainID:             56,
@@ -69,8 +61,6 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     3,
 		BlockThreshold: 50,
-
-		DistanceToChainHighest: 19,
 	},
 	{
 		ChainID:             137,
@@ -82,8 +72,6 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     4,
 		BlockThreshold: 70,
-
-		DistanceToChainHighest: 27,
 	},
 	{
 		ChainID:             250,
@@ -95,8 +83,6 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     5,
 		BlockThreshold: 100,
-
-		DistanceToChainHighest: 21,
 	},
 	{
 		ChainID:             42161,
@@ -108,8 +94,6 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     60,
 		BlockThreshold: 1200,
-
-		DistanceToChainHighest: 200,
 	},
 	{
 		ChainID:             43114,
@@ -121,8 +105,6 @@ var allChainSettings = []ChainSettings{
 		DefaultOffset:  0,
 		SafeOffset:     4,
 		BlockThreshold: 70,
-
-		DistanceToChainHighest: 26,
 	},
 }
 

From 3ed77f85b9fec07ab2a61e3d9f825a000c91ad37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Wed, 18 Oct 2023 18:30:50 +0300
Subject: [PATCH 12/26] add sorting and fix the experiment test

---
 feeds/timeline/timeline.go      |  79 +++++++++++------
 feeds/timeline/timeline_test.go | 153 ++++++++++++++++++++++++++------
 2 files changed, 178 insertions(+), 54 deletions(-)

diff --git a/feeds/timeline/timeline.go b/feeds/timeline/timeline.go
index f9e9afae..275b4b09 100644
--- a/feeds/timeline/timeline.go
+++ b/feeds/timeline/timeline.go
@@ -1,6 +1,7 @@
 package timeline
 
 import (
+	"sort"
 	"sync"
 	"time"
 
@@ -13,7 +14,7 @@ import (
 // latest block in every minute.
 type BlockTimeline struct {
 	maxMinutes   int
-	blockMinutes []*Minute // when the block was produced
+	chainMinutes []*Minute // when the block was produced
 	localMinutes []*Minute // when we handled the block
 	delay        *time.Duration
 	mu           sync.RWMutex
@@ -48,10 +49,10 @@ func (bt *BlockTimeline) doCleanup() {
 	bt.mu.Lock()
 	defer bt.mu.Unlock()
 
-	currSize := len(bt.blockMinutes)
+	currSize := len(bt.chainMinutes)
 	if currSize > bt.maxMinutes {
 		extra := currSize - bt.maxMinutes
-		bt.blockMinutes = bt.blockMinutes[extra:] // release oldest
+		bt.chainMinutes = bt.chainMinutes[extra:] // release oldest
 	}
 
 	currSize = len(bt.localMinutes)
@@ -83,7 +84,7 @@ func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
 	}
 
 	var foundBlockMinute bool
-	for _, minute := range bt.blockMinutes {
+	for _, minute := range bt.chainMinutes {
 		if minute.Timestamp.Equal(blockMinuteTs) {
 			if blockNum > minute.HighestBlockNumber {
 				minute.HighestBlockNumber = blockNum
@@ -93,7 +94,7 @@ func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
 		}
 	}
 	if !foundBlockMinute {
-		bt.blockMinutes = append(bt.blockMinutes, &Minute{
+		bt.chainMinutes = append(bt.chainMinutes, &Minute{
 			HighestBlockNumber: blockNum,
 			Timestamp:          blockMinuteTs,
 		})
@@ -116,6 +117,13 @@ func (bt *BlockTimeline) HandleBlock(evt *domain.BlockEvent) error {
 		})
 	}
 
+	sort.Slice(bt.chainMinutes, func(i, j int) bool {
+		return bt.chainMinutes[i].Timestamp.Before(bt.chainMinutes[j].Timestamp)
+	})
+	sort.Slice(bt.localMinutes, func(i, j int) bool {
+		return bt.localMinutes[i].Timestamp.Before(bt.localMinutes[j].Timestamp)
+	})
+
 	return nil
 }
 
@@ -129,20 +137,7 @@ func (bt *BlockTimeline) GetDelay() (time.Duration, bool) {
 	return *bt.delay, true
 }
 
-// GetGlobalHighest returns the global highest block number within the same minute of the given timestamp.
-func (bt *BlockTimeline) GetGlobalHighest(ts time.Time) (uint64, bool) {
-	return bt.getHighest(bt.blockMinutes, ts)
-}
-
-// GetLocalHighest returns the local highest block number within the same minute of the given timestamp.
-func (bt *BlockTimeline) GetLocalHighest(ts time.Time) (uint64, bool) {
-	return bt.getHighest(bt.localMinutes, ts)
-}
-
 func (bt *BlockTimeline) getHighest(minutes []*Minute, ts time.Time) (uint64, bool) {
-	bt.mu.RLock()
-	defer bt.mu.RUnlock()
-
 	ts = ts.Truncate(time.Minute)
 	for _, minute := range minutes {
 		if minute.Timestamp.Equal(ts) {
@@ -152,17 +147,47 @@ func (bt *BlockTimeline) getHighest(minutes []*Minute, ts time.Time) (uint64, bo
 	return 0, false
 }
 
-// CalculateLag calculates the block number lag in a given minute.
-func (bt *BlockTimeline) CalculateLag(ts time.Time) (int64, bool) {
-	highestGlobal, ok := bt.GetGlobalHighest(ts)
-	if !ok {
-		return 0, false
+func (bt *BlockTimeline) getLatestUpTo(minutes []*Minute, ts time.Time) (uint64, bool) {
+	ts = ts.Truncate(time.Minute)
+	var foundMinute *Minute
+	for _, minute := range minutes {
+		if minute.Timestamp.After(ts) {
+			break
+		}
+		foundMinute = minute
+	}
+	if foundMinute != nil {
+		return foundMinute.HighestBlockNumber, true
+	}
+	return 0, false
+}
+
+// CalculateLag calculates the block number lag by taking the average of each minute.
+func (bt *BlockTimeline) CalculateLag() (float64, bool) {
+	bt.mu.RLock()
+	defer bt.mu.RUnlock()
+
+	var (
+		total float64
+		count float64
+	)
+	for i, chainMinute := range bt.chainMinutes {
+		// exclude the last minute
+		if i == len(bt.chainMinutes)-1 {
+			break
+		}
+		// avoid calculation if we can't find a highest
+		localMinuteHighest, ok := bt.getLatestUpTo(bt.localMinutes, chainMinute.Timestamp)
+		if !ok {
+			continue
+		}
+		total += float64(chainMinute.HighestBlockNumber - localMinuteHighest)
+		count++
 	}
-	highestLocal, ok := bt.GetLocalHighest(ts)
-	if !ok {
+	if count == 0 {
 		return 0, false
 	}
-	return int64(highestGlobal) - int64(highestLocal), true
+	return total / count, true
 }
 
 // Size returns the minute count.
@@ -170,5 +195,5 @@ func (bt *BlockTimeline) Size() int {
 	bt.mu.RLock()
 	defer bt.mu.RUnlock()
 
-	return len(bt.blockMinutes)
+	return len(bt.chainMinutes)
 }
diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
index 3ae40dd4..15b3d1dc 100644
--- a/feeds/timeline/timeline_test.go
+++ b/feeds/timeline/timeline_test.go
@@ -12,6 +12,7 @@ import (
 	"github.com/forta-network/forta-core-go/domain"
 	"github.com/forta-network/forta-core-go/ethereum"
 	"github.com/forta-network/forta-core-go/feeds"
+	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
 
@@ -44,18 +45,18 @@ func TestTimeline_GlobalHighest(t *testing.T) {
 	blockTimeline.HandleBlock(blockForTimestamp(min3Ts, "0x7"))
 
 	// verify state
-	r.EqualValues(5, blockTimeline.blockMinutes[0].HighestBlockNumber)
-	r.EqualValues(6, blockTimeline.blockMinutes[1].HighestBlockNumber)
-	r.EqualValues(7, blockTimeline.blockMinutes[2].HighestBlockNumber)
+	r.EqualValues(5, blockTimeline.chainMinutes[0].HighestBlockNumber)
+	r.EqualValues(6, blockTimeline.chainMinutes[1].HighestBlockNumber)
+	r.EqualValues(7, blockTimeline.chainMinutes[2].HighestBlockNumber)
 
 	// cleanup should remove the first minute because of the max minutes num
 	blockTimeline.doCleanup()
 
 	// verify state
-	r.EqualValues(6, blockTimeline.blockMinutes[0].HighestBlockNumber)
-	r.EqualValues(7, blockTimeline.blockMinutes[1].HighestBlockNumber)
+	r.EqualValues(6, blockTimeline.chainMinutes[0].HighestBlockNumber)
+	r.EqualValues(7, blockTimeline.chainMinutes[1].HighestBlockNumber)
 
-	highestGlobal, ok := blockTimeline.GetGlobalHighest(min2)
+	highestGlobal, ok := blockTimeline.getLatestUpTo(blockTimeline.chainMinutes, min2)
 	r.True(ok)
 	r.Equal(uint64(6), highestGlobal)
 }
@@ -65,18 +66,108 @@ func TestTimeline_CalculateLag(t *testing.T) {
 
 	blockTimeline := &BlockTimeline{}
 
-	currMin := time.Now().UTC().Truncate(time.Minute)
+	start := time.Now().UTC().Truncate(time.Minute)
+
+	// minute: 1
+	// local:  1
+	// chain:  2
+	// lag:    1
+	min1 := start
 	blockTimeline.localMinutes = append(blockTimeline.localMinutes, &Minute{
-		Timestamp:          currMin,
+		Timestamp:          min1,
 		HighestBlockNumber: 1,
 	})
-	blockTimeline.blockMinutes = append(blockTimeline.blockMinutes, &Minute{
-		Timestamp:          currMin,
+	blockTimeline.chainMinutes = append(blockTimeline.chainMinutes, &Minute{
+		Timestamp:          min1,
 		HighestBlockNumber: 2,
 	})
-	lag, ok := blockTimeline.CalculateLag(currMin)
+	lag, ok := blockTimeline.CalculateLag()
+	r.False(ok)
+
+	// minute: 2
+	// local:  3
+	// chain:  8
+	// lag:    5
+	min2 := min1.Add(time.Minute)
+	blockTimeline.localMinutes = append(blockTimeline.localMinutes, &Minute{
+		Timestamp:          min2,
+		HighestBlockNumber: 3,
+	})
+	blockTimeline.chainMinutes = append(blockTimeline.chainMinutes, &Minute{
+		Timestamp:          min2,
+		HighestBlockNumber: 8,
+	})
+	lag, ok = blockTimeline.CalculateLag()
+	r.True(ok)
+	r.Equal(float64(1), lag) // because excludes the last minute: (2-1)/1
+
+	// lags for a while: these minutes have block minutes but no local processing minutes
+	// minute: 3
+	// chain:  12
+	// lag:    9 <-- using previous local (12 - 3)
+	blockTimeline.chainMinutes = append(blockTimeline.chainMinutes, &Minute{
+		Timestamp:          min1.Add(time.Minute * 2),
+		HighestBlockNumber: 12,
+	})
+	// minute: 3
+	// chain:  16
+	// lag:    13 <-- using previous local (16 - 3)
+	blockTimeline.chainMinutes = append(blockTimeline.chainMinutes, &Minute{
+		Timestamp:          min1.Add(time.Minute * 3),
+		HighestBlockNumber: 16,
+	})
+	// minute: 4
+	// chain:  18
+	// lag:    15 <-- using previous local (18 - 3)
+	blockTimeline.chainMinutes = append(blockTimeline.chainMinutes, &Minute{
+		Timestamp:          min1.Add(time.Minute * 4),
+		HighestBlockNumber: 18,
+	})
+
+	// catches up in this minute - processes up to 20
+	// minute: 6
+	// local:  20
+	// chain:  22
+	// lag:    2
+	min6 := min1.Add(time.Minute * 5)
+	blockTimeline.localMinutes = append(blockTimeline.localMinutes, &Minute{
+		Timestamp:          min6,
+		HighestBlockNumber: 20,
+	})
+	blockTimeline.chainMinutes = append(blockTimeline.chainMinutes, &Minute{
+		Timestamp:          min6,
+		HighestBlockNumber: 22,
+	})
+
+	// this last minute doesn't matter as last minutes are excluded from calculation
+	// minute: 7
+	// local:  22
+	// chain:  26
+	// lag:    4 <-- shouldn't matter
+	min7 := min1.Add(time.Minute * 6)
+	blockTimeline.localMinutes = append(blockTimeline.localMinutes, &Minute{
+		Timestamp:          min7,
+		HighestBlockNumber: 22,
+	})
+	blockTimeline.chainMinutes = append(blockTimeline.chainMinutes, &Minute{
+		Timestamp:          min7,
+		HighestBlockNumber: 26,
+	})
+
+	// we are iterating by the block minutes during the calculation so
+	// this local minute doesn't matter
+	// minute: 8
+	// local:  24
+	// lag:    2 <-- shouldn't matter, using previous local (24-22)
+	blockTimeline.localMinutes = append(blockTimeline.localMinutes, &Minute{
+		Timestamp:          min1.Add(time.Minute * 7),
+		HighestBlockNumber: 24,
+	})
+
+	// the final calculation
+	lag, ok = blockTimeline.CalculateLag()
 	r.True(ok)
-	r.Equal(int64(1), lag)
+	r.Equal(float64(1+5+9+13+15+2)/float64(6), lag)
 }
 
 func blockForTimestamp(ts, blockNumber string) *domain.BlockEvent {
@@ -92,39 +183,47 @@ func TestRealTimeLag(t *testing.T) {
 	if os.Getenv("TIMELINE_EXPERIMENT") != "1" {
 		return
 	}
+
+	logrus.SetLevel(logrus.ErrorLevel)
+
 	r := require.New(t)
 	ctx, cancel := context.WithCancel(context.Background())
 	ethClient, err := ethereum.NewStreamEthClient(ctx, "", os.Getenv("JSON_RPC_API"))
 	ethClient.SetRetryInterval(time.Second * 2)
 	r.NoError(err)
 	blockFeed, err := feeds.NewBlockFeed(ctx, ethClient, ethClient, feeds.BlockFeedConfig{
-		ChainID:     big.NewInt(137),
-		DisableLogs: true,
+		ChainID: big.NewInt(137),
+		//DisableLogs: true,
 	})
 	r.NoError(err)
 
 	blockTimeline := &BlockTimeline{}
 	errCh := blockFeed.Subscribe(func(evt *domain.BlockEvent) error {
-		blockTs, _ := evt.Block.GetTimestamp()
-		delay := time.Since(*blockTs)
-		fmt.Println("delay:", delay)
+		// blockTs, _ := evt.Block.GetTimestamp()
+		// delay := time.Since(*blockTs)
+		// fmt.Println("delay:", delay)
 		return blockTimeline.HandleBlock(evt)
 	})
 
 	go blockFeed.Start()
 
 	go func() {
-		<-time.After(time.Minute * 4)
-		cancel()
+		longTicker := time.After(time.Minute * 10)
+		shortTicker := time.NewTicker(time.Minute).C
+		for {
+			select {
+			case <-longTicker:
+				cancel()
+				return
+			case <-shortTicker:
+				lag, ok := blockTimeline.CalculateLag()
+				if !ok {
+					continue
+				}
+				fmt.Println("lag at", time.Now().Truncate(time.Minute).Format(time.RFC3339), ":", lag, "blocks")
+			}
+		}
 	}()
 
 	<-errCh
-
-	for _, minute := range blockTimeline.blockMinutes {
-		lag, ok := blockTimeline.CalculateLag(minute.Timestamp)
-		if !ok {
-			continue
-		}
-		fmt.Println(minute.Timestamp.Format(time.RFC3339), ":", lag)
-	}
 }

From d3a368a701ba8f71e88c87586f124b5e7c0776f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Wed, 18 Oct 2023 20:19:23 +0300
Subject: [PATCH 13/26] remove the size method

---
 feeds/timeline/timeline.go | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/feeds/timeline/timeline.go b/feeds/timeline/timeline.go
index 275b4b09..b5fae5c5 100644
--- a/feeds/timeline/timeline.go
+++ b/feeds/timeline/timeline.go
@@ -7,12 +7,14 @@ import (
 
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	"github.com/forta-network/forta-core-go/domain"
+	"github.com/forta-network/forta-core-go/protocol/settings"
 	log "github.com/sirupsen/logrus"
 )
 
 // BlockTimeline implements a block feed subscriber and keeps track of the
 // latest block in every minute.
 type BlockTimeline struct {
+	threshold    int
 	maxMinutes   int
 	chainMinutes []*Minute // when the block was produced
 	localMinutes []*Minute // when we handled the block
@@ -27,8 +29,9 @@ type Minute struct {
 }
 
 // NewBlockTimeline creates a new block timeline.
-func NewBlockTimeline(maxMinutes int) *BlockTimeline {
+func NewBlockTimeline(chainID, maxMinutes int) *BlockTimeline {
 	bt := &BlockTimeline{
+		threshold:  settings.GetChainSettings(chainID).BlockThreshold,
 		maxMinutes: maxMinutes,
 	}
 
@@ -189,11 +192,3 @@ func (bt *BlockTimeline) CalculateLag() (float64, bool) {
 	}
 	return total / count, true
 }
-
-// Size returns the minute count.
-func (bt *BlockTimeline) Size() int {
-	bt.mu.RLock()
-	defer bt.mu.RUnlock()
-
-	return len(bt.chainMinutes)
-}

From 44281aed20af75d38a4e57e8350d980f459d8af8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Wed, 18 Oct 2023 20:27:23 +0300
Subject: [PATCH 14/26] incorporate the score estimation

---
 feeds/timeline/timeline.go      | 16 ++++++++++++++++
 feeds/timeline/timeline_test.go |  5 ++++-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/feeds/timeline/timeline.go b/feeds/timeline/timeline.go
index b5fae5c5..ee29a6c5 100644
--- a/feeds/timeline/timeline.go
+++ b/feeds/timeline/timeline.go
@@ -192,3 +192,19 @@ func (bt *BlockTimeline) CalculateLag() (float64, bool) {
 	}
 	return total / count, true
 }
+
+// EstimateBlockScore estimates the block score based on the lag and the block threshold.
+func (bt *BlockTimeline) EstimateBlockScore() (float64, bool) {
+	lag, ok := bt.CalculateLag()
+	if !ok {
+		return 0, false
+	}
+	estimate := (float64(bt.threshold) - float64(lag)) / float64(bt.threshold)
+	if estimate < 0 {
+		estimate = 0
+	}
+	if estimate > 1 {
+		estimate = 1
+	}
+	return estimate, true
+}
diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
index 15b3d1dc..72f5b686 100644
--- a/feeds/timeline/timeline_test.go
+++ b/feeds/timeline/timeline_test.go
@@ -64,7 +64,7 @@ func TestTimeline_GlobalHighest(t *testing.T) {
 func TestTimeline_CalculateLag(t *testing.T) {
 	r := require.New(t)
 
-	blockTimeline := &BlockTimeline{}
+	blockTimeline := NewBlockTimeline(1, 1000000)
 
 	start := time.Now().UTC().Truncate(time.Minute)
 
@@ -168,6 +168,9 @@ func TestTimeline_CalculateLag(t *testing.T) {
 	lag, ok = blockTimeline.CalculateLag()
 	r.True(ok)
 	r.Equal(float64(1+5+9+13+15+2)/float64(6), lag)
+	estimate, ok := blockTimeline.EstimateBlockScore()
+	r.True(ok)
+	r.Equal(0.625, estimate)
 }
 
 func blockForTimestamp(ts, blockNumber string) *domain.BlockEvent {

From a75a2c5b3b1a0a0170b5ea68963195ec11ac66ff Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Wed, 18 Oct 2023 21:05:15 +0300
Subject: [PATCH 15/26] remove redundant method and increase coverage

---
 feeds/timeline/timeline.go      | 10 ----------
 feeds/timeline/timeline_test.go |  6 ++++++
 2 files changed, 6 insertions(+), 10 deletions(-)

diff --git a/feeds/timeline/timeline.go b/feeds/timeline/timeline.go
index ee29a6c5..d5f76a5b 100644
--- a/feeds/timeline/timeline.go
+++ b/feeds/timeline/timeline.go
@@ -140,16 +140,6 @@ func (bt *BlockTimeline) GetDelay() (time.Duration, bool) {
 	return *bt.delay, true
 }
 
-func (bt *BlockTimeline) getHighest(minutes []*Minute, ts time.Time) (uint64, bool) {
-	ts = ts.Truncate(time.Minute)
-	for _, minute := range minutes {
-		if minute.Timestamp.Equal(ts) {
-			return minute.HighestBlockNumber, true
-		}
-	}
-	return 0, false
-}
-
 func (bt *BlockTimeline) getLatestUpTo(minutes []*Minute, ts time.Time) (uint64, bool) {
 	ts = ts.Truncate(time.Minute)
 	var foundMinute *Minute
diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
index 72f5b686..1e93cf99 100644
--- a/feeds/timeline/timeline_test.go
+++ b/feeds/timeline/timeline_test.go
@@ -171,6 +171,12 @@ func TestTimeline_CalculateLag(t *testing.T) {
 	estimate, ok := blockTimeline.EstimateBlockScore()
 	r.True(ok)
 	r.Equal(0.625, estimate)
+
+	testDelay := time.Second
+	blockTimeline.delay = &testDelay
+	delay, ok := blockTimeline.GetDelay()
+	r.True(ok)
+	r.Equal(testDelay, delay)
 }
 
 func blockForTimestamp(ts, blockNumber string) *domain.BlockEvent {

From bf10332d9c8c69f66b352e0ae27e3b4fb8a41812 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Wed, 18 Oct 2023 22:27:16 +0300
Subject: [PATCH 16/26] fix tests

---
 feeds/blocks_test.go                | 1 +
 inspect/validation/validate_test.go | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/feeds/blocks_test.go b/feeds/blocks_test.go
index be7efe1d..d3b598e0 100644
--- a/feeds/blocks_test.go
+++ b/feeds/blocks_test.go
@@ -81,6 +81,7 @@ func getTestBlockFeed(t *testing.T) (*blockFeed, *mocks.MockClient, *mocks.MockC
 		traceClient: traceClient,
 		cache:       cache,
 		tracing:     true,
+		logs:        true,
 		maxBlockAge: &maxBlockAge,
 	}, client, traceClient, ctx, cancel
 }
diff --git a/inspect/validation/validate_test.go b/inspect/validation/validate_test.go
index eb0bfff8..404f8a91 100644
--- a/inspect/validation/validate_test.go
+++ b/inspect/validation/validate_test.go
@@ -11,8 +11,8 @@ import (
 )
 
 var testValidateEnv struct {
-	ScanAPI  string `envconfig:"scan_api" default:"https://rpcapi.fantom.network"`
-	TraceAPI string `envconfig:"trace_api" default:"https://rpcapi-tracing.fantom.network"`
+	ScanAPI  string `envconfig:"scan_api" default:"https://fantom-testnet.public.blastapi.io/"`
+	TraceAPI string `envconfig:"trace_api" default:"https://rpcapi-tracing.testnet.fantom.network/"`
 }
 
 func init() {

From 1105118b7d9618dc08727a50e3836fb42b802e8f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Wed, 18 Oct 2023 22:49:41 +0300
Subject: [PATCH 17/26] fix issues

---
 feeds/timeline/timeline_test.go           | 18 +++++++++---------
 inspect/scorecalc/pass_fail_calculator.go |  2 +-
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
index 1e93cf99..5d48852c 100644
--- a/feeds/timeline/timeline_test.go
+++ b/feeds/timeline/timeline_test.go
@@ -30,19 +30,19 @@ func TestTimeline_GlobalHighest(t *testing.T) {
 	min3Ts := hexutil.EncodeUint64(uint64(min1.Add(time.Minute * 2).Unix()))
 
 	// add first minute block number
-	blockTimeline.HandleBlock(blockForTimestamp(min1Ts, "0x1"))
+	r.NoError(blockTimeline.HandleBlock(blockForTimestamp(min1Ts, "0x1")))
 	// replace the first minute number
-	blockTimeline.HandleBlock(blockForTimestamp(min1Ts, "0x2"))
+	r.NoError(blockTimeline.HandleBlock(blockForTimestamp(min1Ts, "0x2")))
 	// add new one for the next minute
-	blockTimeline.HandleBlock(blockForTimestamp(min2Ts, "0x3"))
+	r.NoError(blockTimeline.HandleBlock(blockForTimestamp(min2Ts, "0x3")))
 	// replace the second minute number
-	blockTimeline.HandleBlock(blockForTimestamp(min2Ts, "0x4"))
+	r.NoError(blockTimeline.HandleBlock(blockForTimestamp(min2Ts, "0x4")))
 	// replace the first minute number
-	blockTimeline.HandleBlock(blockForTimestamp(min1Ts, "0x5"))
+	r.NoError(blockTimeline.HandleBlock(blockForTimestamp(min1Ts, "0x5")))
 	// replace the second minute number
-	blockTimeline.HandleBlock(blockForTimestamp(min2Ts, "0x6"))
+	r.NoError(blockTimeline.HandleBlock(blockForTimestamp(min2Ts, "0x6")))
 	// add a third minute
-	blockTimeline.HandleBlock(blockForTimestamp(min3Ts, "0x7"))
+	r.NoError(blockTimeline.HandleBlock(blockForTimestamp(min3Ts, "0x7")))
 
 	// verify state
 	r.EqualValues(5, blockTimeline.chainMinutes[0].HighestBlockNumber)
@@ -81,7 +81,7 @@ func TestTimeline_CalculateLag(t *testing.T) {
 		Timestamp:          min1,
 		HighestBlockNumber: 2,
 	})
-	lag, ok := blockTimeline.CalculateLag()
+	_, ok := blockTimeline.CalculateLag()
 	r.False(ok)
 
 	// minute: 2
@@ -97,7 +97,7 @@ func TestTimeline_CalculateLag(t *testing.T) {
 		Timestamp:          min2,
 		HighestBlockNumber: 8,
 	})
-	lag, ok = blockTimeline.CalculateLag()
+	lag, ok := blockTimeline.CalculateLag()
 	r.True(ok)
 	r.Equal(float64(1), lag) // because excludes the last minute: (2-1)/1
 
diff --git a/inspect/scorecalc/pass_fail_calculator.go b/inspect/scorecalc/pass_fail_calculator.go
index 21f8cc57..92a87c29 100644
--- a/inspect/scorecalc/pass_fail_calculator.go
+++ b/inspect/scorecalc/pass_fail_calculator.go
@@ -57,7 +57,7 @@ func (c *chainPassFailCalculator) CalculateScore(results *inspect.InspectionResu
 	if results.Indicators[inspect.IndicatorResourcesMemoryTotal] < c.config.MinTotalMemory {
 		return 0, nil
 	}
-	
+
 	if results.Inputs.IsETH2 && results.Indicators[inspect.IndicatorScanAPIIsETH2] == inspect.ResultFailure {
 		return 0, nil
 	}

From 70dfdf57f6912f981a5c04a674b78da9f93e2b09 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Wed, 18 Oct 2023 23:31:22 +0300
Subject: [PATCH 18/26] shorten the tests

---
 Makefile                     | 2 +-
 inspect/network_test.go      | 4 ++++
 inspect/proxy_api_test.go    | 4 ++++
 inspect/registry_api_test.go | 4 ++++
 inspect/scan_api_test.go     | 4 ++++
 inspect/trace_api_test.go    | 4 ++++
 6 files changed, 21 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 36a62ebb..7ad76a03 100644
--- a/Makefile
+++ b/Makefile
@@ -82,7 +82,7 @@ mocks:
 
 .PHONY: test
 test:
-	go test -v -count=1 -coverprofile=coverage.out ./...
+	go test -v -count=1 -short -coverprofile=coverage.out ./...
 
 .PHONY: coverage
 coverage:
diff --git a/inspect/network_test.go b/inspect/network_test.go
index 39fb7a1c..1315e702 100644
--- a/inspect/network_test.go
+++ b/inspect/network_test.go
@@ -8,6 +8,10 @@ import (
 )
 
 func TestNetworkInspection(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping validation test in short mode")
+	}
+
 	r := require.New(t)
 
 	DownloadTestSavingMode = true
diff --git a/inspect/proxy_api_test.go b/inspect/proxy_api_test.go
index cb31c76b..6b4d65b6 100644
--- a/inspect/proxy_api_test.go
+++ b/inspect/proxy_api_test.go
@@ -22,6 +22,10 @@ func init() {
 }
 
 func TestProxyAPIInspection(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping validation test in short mode")
+	}
+
 	r := require.New(t)
 
 	inspector := &ProxyAPIInspector{}
diff --git a/inspect/registry_api_test.go b/inspect/registry_api_test.go
index 761b930b..1e8309f8 100644
--- a/inspect/registry_api_test.go
+++ b/inspect/registry_api_test.go
@@ -17,6 +17,10 @@ func init() {
 }
 
 func TestRegistryAPIInspection(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping validation test in short mode")
+	}
+
 	r := require.New(t)
 
 	inspector := &RegistryAPIInspector{}
diff --git a/inspect/scan_api_test.go b/inspect/scan_api_test.go
index e612e9ce..55f61622 100644
--- a/inspect/scan_api_test.go
+++ b/inspect/scan_api_test.go
@@ -17,6 +17,10 @@ func init() {
 }
 
 func TestScanAPIInspection(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping validation test in short mode")
+	}
+
 	r := require.New(t)
 
 	recentBlockNumber := testGetRecentBlockNumber(r, testScanEnv.ScanAPI)
diff --git a/inspect/trace_api_test.go b/inspect/trace_api_test.go
index 1de5a84f..55d5c87b 100644
--- a/inspect/trace_api_test.go
+++ b/inspect/trace_api_test.go
@@ -18,6 +18,10 @@ func init() {
 }
 
 func TestTraceAPIInspection(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping validation test in short mode")
+	}
+
 	r := require.New(t)
 
 	recentBlockNumber := testGetRecentBlockNumber(r, testTraceEnv.TraceAPI)

From ba668cba8671f0885871f08b272aae25bcb984e7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Wed, 18 Oct 2023 23:33:20 +0300
Subject: [PATCH 19/26] fix typo

---
 inspect/network_test.go      | 2 +-
 inspect/proxy_api_test.go    | 2 +-
 inspect/registry_api_test.go | 2 +-
 inspect/scan_api_test.go     | 2 +-
 inspect/trace_api_test.go    | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/inspect/network_test.go b/inspect/network_test.go
index 1315e702..09d83792 100644
--- a/inspect/network_test.go
+++ b/inspect/network_test.go
@@ -9,7 +9,7 @@ import (
 
 func TestNetworkInspection(t *testing.T) {
 	if testing.Short() {
-		t.Skip("skipping validation test in short mode")
+		t.Skip("skipping inspection test in short mode")
 	}
 
 	r := require.New(t)
diff --git a/inspect/proxy_api_test.go b/inspect/proxy_api_test.go
index 6b4d65b6..e6e4eea9 100644
--- a/inspect/proxy_api_test.go
+++ b/inspect/proxy_api_test.go
@@ -23,7 +23,7 @@ func init() {
 
 func TestProxyAPIInspection(t *testing.T) {
 	if testing.Short() {
-		t.Skip("skipping validation test in short mode")
+		t.Skip("skipping inspection test in short mode")
 	}
 
 	r := require.New(t)
diff --git a/inspect/registry_api_test.go b/inspect/registry_api_test.go
index 1e8309f8..1b612c92 100644
--- a/inspect/registry_api_test.go
+++ b/inspect/registry_api_test.go
@@ -18,7 +18,7 @@ func init() {
 
 func TestRegistryAPIInspection(t *testing.T) {
 	if testing.Short() {
-		t.Skip("skipping validation test in short mode")
+		t.Skip("skipping inspection test in short mode")
 	}
 
 	r := require.New(t)
diff --git a/inspect/scan_api_test.go b/inspect/scan_api_test.go
index 55f61622..8b7a6a39 100644
--- a/inspect/scan_api_test.go
+++ b/inspect/scan_api_test.go
@@ -18,7 +18,7 @@ func init() {
 
 func TestScanAPIInspection(t *testing.T) {
 	if testing.Short() {
-		t.Skip("skipping validation test in short mode")
+		t.Skip("skipping inspection test in short mode")
 	}
 
 	r := require.New(t)
diff --git a/inspect/trace_api_test.go b/inspect/trace_api_test.go
index 55d5c87b..afbe613b 100644
--- a/inspect/trace_api_test.go
+++ b/inspect/trace_api_test.go
@@ -19,7 +19,7 @@ func init() {
 
 func TestTraceAPIInspection(t *testing.T) {
 	if testing.Short() {
-		t.Skip("skipping validation test in short mode")
+		t.Skip("skipping inspection test in short mode")
 	}
 
 	r := require.New(t)

From 4ae9aa565079f4b950c137388b6a4002232445e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Wed, 18 Oct 2023 23:51:26 +0300
Subject: [PATCH 20/26] one more short test

---
 registry/assignments_test.go | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/registry/assignments_test.go b/registry/assignments_test.go
index d701af59..6cf0cc67 100644
--- a/registry/assignments_test.go
+++ b/registry/assignments_test.go
@@ -132,6 +132,10 @@ var testExpectedAssignmentList = []*Assignment{
 }
 
 func TestGetAssignmentList(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping assignment list test in short mode")
+	}
+
 	r := require.New(t)
 
 	cfg := defaultConfig

From 99469604fcfd8f0c0f967e8ccb28d00b2503ec71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Thu, 19 Oct 2023 12:58:38 +0300
Subject: [PATCH 21/26] add one more short test

---
 feeds/timeline/timeline_test.go     | 3 +--
 inspect/validation/validate_test.go | 4 ++++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
index 5d48852c..2188bdef 100644
--- a/feeds/timeline/timeline_test.go
+++ b/feeds/timeline/timeline_test.go
@@ -12,7 +12,6 @@ import (
 	"github.com/forta-network/forta-core-go/domain"
 	"github.com/forta-network/forta-core-go/ethereum"
 	"github.com/forta-network/forta-core-go/feeds"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
 
@@ -193,7 +192,7 @@ func TestRealTimeLag(t *testing.T) {
 		return
 	}
 
-	logrus.SetLevel(logrus.ErrorLevel)
+	//logrus.SetLevel(logrus.ErrorLevel)
 
 	r := require.New(t)
 	ctx, cancel := context.WithCancel(context.Background())
diff --git a/inspect/validation/validate_test.go b/inspect/validation/validate_test.go
index 404f8a91..8051ab1a 100644
--- a/inspect/validation/validate_test.go
+++ b/inspect/validation/validate_test.go
@@ -20,6 +20,10 @@ func init() {
 }
 
 func TestValidateInspectionSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping inspection validation test in short mode")
+	}
+
 	ctx := context.Background()
 	r := require.New(t)
 

From 55b69f6efd571b6825b1ca0dad2108d6764b8254 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Thu, 19 Oct 2023 13:08:38 +0300
Subject: [PATCH 22/26] try changing cache key

---
 .github/actions/go/action.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/go/action.yml b/.github/actions/go/action.yml
index f8c21048..11e1c1e9 100644
--- a/.github/actions/go/action.yml
+++ b/.github/actions/go/action.yml
@@ -16,9 +16,9 @@ runs:
         path: |
           ~/go/pkg/mod
           ~/.cache/go-build
-        key: ${{ runner.os }}-go-validate-${{ hashFiles('**/go.sum') }}
+        key: ${{ runner.os }}-go-validate-cache-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.os }}-go-validate
+          ${{ runner.os }}-go-validate-cache
     - name: Vet
       shell: bash
       run: |

From e77a27a938c9bce25d86bd6a1f2389c791f06563 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Thu, 19 Oct 2023 13:13:12 +0300
Subject: [PATCH 23/26] reset cache key

---
 .github/actions/go/action.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/actions/go/action.yml b/.github/actions/go/action.yml
index 11e1c1e9..f8c21048 100644
--- a/.github/actions/go/action.yml
+++ b/.github/actions/go/action.yml
@@ -16,9 +16,9 @@ runs:
         path: |
           ~/go/pkg/mod
           ~/.cache/go-build
-        key: ${{ runner.os }}-go-validate-cache-${{ hashFiles('**/go.sum') }}
+        key: ${{ runner.os }}-go-validate-${{ hashFiles('**/go.sum') }}
         restore-keys: |
-          ${{ runner.os }}-go-validate-cache
+          ${{ runner.os }}-go-validate
     - name: Vet
       shell: bash
       run: |

From 7ab4acced33da8d375ec6e9fb7592e540024d75f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Thu, 19 Oct 2023 13:14:12 +0300
Subject: [PATCH 24/26] add skip to timeline experiment

---
 feeds/timeline/timeline_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/feeds/timeline/timeline_test.go b/feeds/timeline/timeline_test.go
index 2188bdef..a764080c 100644
--- a/feeds/timeline/timeline_test.go
+++ b/feeds/timeline/timeline_test.go
@@ -189,7 +189,7 @@ func blockForTimestamp(ts, blockNumber string) *domain.BlockEvent {
 
 func TestRealTimeLag(t *testing.T) {
 	if os.Getenv("TIMELINE_EXPERIMENT") != "1" {
-		return
+		t.Skip("skipping timeline experiment")
 	}
 
 	//logrus.SetLevel(logrus.ErrorLevel)

From 12c6c828d6ec7e94266a82df3a06b730ddb576ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Thu, 19 Oct 2023 22:02:57 +0300
Subject: [PATCH 25/26] fix tests

---
 ethereum/client.go                  |  15 ++-
 ethereum/client_test.go             |  14 ++-
 ethereum/mocks/mock_client.go       |  82 +++++++++++++++-
 inspect/api.go                      |   9 +-
 inspect/network_test.go             |   4 -
 inspect/offset.go                   |   7 +-
 inspect/offset_test.go              |   6 +-
 inspect/proxy_api_test.go           |  62 ++++++++++--
 inspect/registry_api_test.go        |  37 +++++---
 inspect/scan_api_test.go            |  60 ++++++++----
 inspect/trace_api_test.go           |  68 +++++++++-----
 inspect/validation/validate.go      |   7 +-
 inspect/validation/validate_test.go | 141 +++++++++++++++-------------
 13 files changed, 355 insertions(+), 157 deletions(-)

diff --git a/ethereum/client.go b/ethereum/client.go
index 55b8cf4e..8614d135 100644
--- a/ethereum/client.go
+++ b/ethereum/client.go
@@ -42,7 +42,12 @@ type RPCClient interface {
 	Close()
 	Call(result interface{}, method string, args ...interface{}) error
 	CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error
-	Subscribe(ctx context.Context, namespace string, channel interface{}, args ...interface{}) (*rpc.ClientSubscription, error)
+}
+
+// Subscriber subscribes to Ethereum namespaces.
+type Subscriber interface {
+	RPCClient
+	Subscribe(ctx context.Context, namespace string, channel interface{}, args ...interface{}) (domain.ClientSubscription, error)
 }
 
 // Client is an interface encompassing all ethereum actions
@@ -94,6 +99,7 @@ var maxBackoff = 1 * time.Minute
 type streamEthClient struct {
 	apiName       string
 	rpcClient     RPCClient
+	subscriber    Subscriber
 	retryInterval time.Duration
 	isWebsocket   bool
 
@@ -343,7 +349,7 @@ func (e *streamEthClient) SubscribeToHead(ctx context.Context) (domain.HeaderCh,
 	log.Debug("subscribing to blockchain head")
 	recvCh := make(chan *types.Header)
 	sendCh := make(chan *types.Header)
-	sub, err := e.rpcClient.Subscribe(ctx, "eth", recvCh, "newHeads")
+	sub, err := e.subscriber.Subscribe(ctx, "eth", recvCh, "newHeads")
 	if err != nil {
 		return nil, fmt.Errorf("failed to subscribe: %v", err)
 	}
@@ -396,6 +402,11 @@ type rpcClient struct {
 	*rpc.Client
 }
 
+func (rc *rpcClient) Subscribe(ctx context.Context, namespace string, channel interface{}, args ...interface{}) (domain.ClientSubscription, error) {
+	sub, err := rc.Subscribe(ctx, namespace, channel, args...)
+	return sub, err
+}
+
 var wsBufferPool = new(sync.Pool)
 
 func NewRpcClient(ctx context.Context, url string) (*rpc.Client, error) {
diff --git a/ethereum/client_test.go b/ethereum/client_test.go
index 7ecf3700..f252e04b 100644
--- a/ethereum/client_test.go
+++ b/ethereum/client_test.go
@@ -20,20 +20,21 @@ const testBlockHash = "0x4fc0862e76691f5312964883954d5c2db35e2b8f7a4f191775a4f50
 
 var testErr = errors.New("test err")
 
-func initClient(t *testing.T) (*streamEthClient, *mocks.MockRPCClient, context.Context) {
+func initClient(t *testing.T) (*streamEthClient, *mocks.MockRPCClient, *mocks.MockSubscriber, context.Context) {
 	minBackoff = 1 * time.Millisecond
 	maxBackoff = 1 * time.Millisecond
 	ctx := context.Background()
 	ctrl := gomock.NewController(t)
 	client := mocks.NewMockRPCClient(ctrl)
+	subscriber := mocks.NewMockSubscriber(ctrl)
 
-	return &streamEthClient{rpcClient: client}, client, ctx
+	return &streamEthClient{rpcClient: client, subscriber: subscriber}, client, subscriber, ctx
 }
 
 func TestEthClient_BlockByHash(t *testing.T) {
 	r := require.New(t)
 
-	ethClient, client, ctx := initClient(t)
+	ethClient, client, _, ctx := initClient(t)
 	hash := testBlockHash
 	// verify retry
 	client.EXPECT().CallContext(gomock.Any(), gomock.Any(), blocksByHash, testBlockHash).Return(testErr).Times(1)
@@ -50,10 +51,10 @@ func TestEthClient_BlockByHash(t *testing.T) {
 func TestEthClient_SubscribeToHeader_Err(t *testing.T) {
 	r := require.New(t)
 
-	ethClient, client, ctx := initClient(t)
+	ethClient, _, subscriber, ctx := initClient(t)
 	sub := mock_domain.NewMockClientSubscription(gomock.NewController(t))
 
-	client.EXPECT().Subscribe(gomock.Any(), gomock.Any(), "newHeads").Return(sub, nil).Times(2)
+	subscriber.EXPECT().Subscribe(gomock.Any(), "eth", gomock.Any(), "newHeads").Return(sub, nil).Times(2)
 	errCh := make(chan error, 1)
 	errCh <- errors.New("subscription encountered some error")
 	sub.EXPECT().Err().Return(errCh).Times(2)
@@ -64,9 +65,12 @@ func TestEthClient_SubscribeToHeader_Err(t *testing.T) {
 
 	headerCh, err = ethClient.SubscribeToHead(ctx)
 	r.NoError(err)
+	var blocked bool
 	select {
 	case <-time.After(time.Second):
 		// should continue from here
+		blocked = true
 	case <-headerCh: // should block
 	}
+	r.True(blocked)
 }
diff --git a/ethereum/mocks/mock_client.go b/ethereum/mocks/mock_client.go
index 146407d3..9396d2ab 100644
--- a/ethereum/mocks/mock_client.go
+++ b/ethereum/mocks/mock_client.go
@@ -13,7 +13,6 @@ import (
 	ethereum "github.com/ethereum/go-ethereum"
 	common "github.com/ethereum/go-ethereum/common"
 	types "github.com/ethereum/go-ethereum/core/types"
-	rpc "github.com/ethereum/go-ethereum/rpc"
 	health "github.com/forta-network/forta-core-go/clients/health"
 	domain "github.com/forta-network/forta-core-go/domain"
 	gomock "github.com/golang/mock/gomock"
@@ -371,24 +370,97 @@ func (mr *MockRPCClientMockRecorder) Close() *gomock.Call {
 	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockRPCClient)(nil).Close))
 }
 
+// MockSubscriber is a mock of Subscriber interface.
+type MockSubscriber struct {
+	ctrl     *gomock.Controller
+	recorder *MockSubscriberMockRecorder
+}
+
+// MockSubscriberMockRecorder is the mock recorder for MockSubscriber.
+type MockSubscriberMockRecorder struct {
+	mock *MockSubscriber
+}
+
+// NewMockSubscriber creates a new mock instance.
+func NewMockSubscriber(ctrl *gomock.Controller) *MockSubscriber {
+	mock := &MockSubscriber{ctrl: ctrl}
+	mock.recorder = &MockSubscriberMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockSubscriber) EXPECT() *MockSubscriberMockRecorder {
+	return m.recorder
+}
+
+// Call mocks base method.
+func (m *MockSubscriber) Call(result interface{}, method string, args ...interface{}) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{result, method}
+	for _, a := range args {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "Call", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Call indicates an expected call of Call.
+func (mr *MockSubscriberMockRecorder) Call(result, method interface{}, args ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{result, method}, args...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Call", reflect.TypeOf((*MockSubscriber)(nil).Call), varargs...)
+}
+
+// CallContext mocks base method.
+func (m *MockSubscriber) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
+	m.ctrl.T.Helper()
+	varargs := []interface{}{ctx, result, method}
+	for _, a := range args {
+		varargs = append(varargs, a)
+	}
+	ret := m.ctrl.Call(m, "CallContext", varargs...)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// CallContext indicates an expected call of CallContext.
+func (mr *MockSubscriberMockRecorder) CallContext(ctx, result, method interface{}, args ...interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	varargs := append([]interface{}{ctx, result, method}, args...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CallContext", reflect.TypeOf((*MockSubscriber)(nil).CallContext), varargs...)
+}
+
+// Close mocks base method.
+func (m *MockSubscriber) Close() {
+	m.ctrl.T.Helper()
+	m.ctrl.Call(m, "Close")
+}
+
+// Close indicates an expected call of Close.
+func (mr *MockSubscriberMockRecorder) Close() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockSubscriber)(nil).Close))
+}
+
 // Subscribe mocks base method.
-func (m *MockRPCClient) Subscribe(ctx context.Context, namespace string, channel interface{}, args ...interface{}) (*rpc.ClientSubscription, error) {
+func (m *MockSubscriber) Subscribe(ctx context.Context, namespace string, channel interface{}, args ...interface{}) (domain.ClientSubscription, error) {
 	m.ctrl.T.Helper()
 	varargs := []interface{}{ctx, namespace, channel}
 	for _, a := range args {
 		varargs = append(varargs, a)
 	}
 	ret := m.ctrl.Call(m, "Subscribe", varargs...)
-	ret0, _ := ret[0].(*rpc.ClientSubscription)
+	ret0, _ := ret[0].(domain.ClientSubscription)
 	ret1, _ := ret[1].(error)
 	return ret0, ret1
 }
 
 // Subscribe indicates an expected call of Subscribe.
-func (mr *MockRPCClientMockRecorder) Subscribe(ctx, namespace, channel interface{}, args ...interface{}) *gomock.Call {
+func (mr *MockSubscriberMockRecorder) Subscribe(ctx, namespace, channel interface{}, args ...interface{}) *gomock.Call {
 	mr.mock.ctrl.T.Helper()
 	varargs := append([]interface{}{ctx, namespace, channel}, args...)
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Subscribe", reflect.TypeOf((*MockRPCClient)(nil).Subscribe), varargs...)
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Subscribe", reflect.TypeOf((*MockSubscriber)(nil).Subscribe), varargs...)
 }
 
 // MockClient is a mock of Client interface.
diff --git a/inspect/api.go b/inspect/api.go
index b2a81b41..5c20cefd 100644
--- a/inspect/api.go
+++ b/inspect/api.go
@@ -147,28 +147,31 @@ func SupportsETH2(ctx context.Context, rpcClient ethereum.RPCClient) bool {
 
 	var block domain.Block
 	if err := getRpcResponse(ctx, rpcClient, &block, blockByNumber, latestBlock, true); err != nil {
+		fmt.Println("@@@@@ response failed", err)
 		return false
 	}
 	if block.Difficulty == nil {
+		fmt.Println("@@@@@ nil difficulty", block.Difficulty)
 		return false
 	}
 	if block.Nonce == nil {
+		fmt.Println("@@@@@ nil nonce", block.Nonce)
 		return false
 	}
 	difficulty, err := hexutil.DecodeBig(*block.Difficulty)
 	if err != nil {
+		fmt.Println("@@@@@ error parsing difficulty", block.Difficulty, err)
 		return false
 	}
 	var nonce types.BlockNonce
 	if err := (&nonce).UnmarshalText([]byte(*block.Nonce)); err != nil {
-		return false
-	}
-	if err != nil {
+		fmt.Println("@@@@@ error parsing nonce", *block.Nonce)
 		return false
 	}
 
 	if difficulty.Sign() == 0 && nonce.Uint64() == 0 {
 		return true
 	}
+	fmt.Println("@@@@@ one of the numbers is not zero", difficulty, nonce)
 	return false
 }
diff --git a/inspect/network_test.go b/inspect/network_test.go
index 09d83792..39fb7a1c 100644
--- a/inspect/network_test.go
+++ b/inspect/network_test.go
@@ -8,10 +8,6 @@ import (
 )
 
 func TestNetworkInspection(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping inspection test in short mode")
-	}
-
 	r := require.New(t)
 
 	DownloadTestSavingMode = true
diff --git a/inspect/offset.go b/inspect/offset.go
index 16918218..2b10e46d 100644
--- a/inspect/offset.go
+++ b/inspect/offset.go
@@ -6,6 +6,7 @@ import (
 	"math/big"
 	"time"
 
+	"github.com/forta-network/forta-core-go/ethereum"
 	"github.com/montanaflynn/stats"
 	"golang.org/x/sync/errgroup"
 )
@@ -19,7 +20,7 @@ type offsetStats struct {
 
 func calculateOffsetStats(
 	ctx context.Context, primaryClient,
-	secondaryClient ProxyAPIClient,
+	secondaryClient ethereum.EthClient,
 ) (offsetStats, error) {
 	ds, err := collectOffsetData(ctx, primaryClient, secondaryClient)
 	if err != nil {
@@ -31,7 +32,7 @@ func calculateOffsetStats(
 
 // collectOffsetData measures how long does it take to receive a recently created block and compares given eth clients.
 // The idea is to mimic the behavior of Scanner feed and Bot proxy query.
-func collectOffsetData(ctx context.Context, primaryClient, secondaryClient ProxyAPIClient) (
+func collectOffsetData(ctx context.Context, primaryClient, secondaryClient ethereum.EthClient) (
 	[]float64, error,
 ) {
 	maxDuration := time.Second * 20
@@ -102,7 +103,7 @@ func collectOffsetData(ctx context.Context, primaryClient, secondaryClient Proxy
 		}
 	}
 }
-func measureBlockDelay(ctx context.Context, client ProxyAPIClient, blockNum uint64) (int64, error) {
+func measureBlockDelay(ctx context.Context, client ethereum.EthClient, blockNum uint64) (int64, error) {
 	t := time.Millisecond * 200
 
 	start := time.Now()
diff --git a/inspect/offset_test.go b/inspect/offset_test.go
index 17ce72f6..2aaa3f09 100644
--- a/inspect/offset_test.go
+++ b/inspect/offset_test.go
@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	mock_inspect "github.com/forta-network/forta-core-go/inspect/mocks"
+	mock_ethereum "github.com/forta-network/forta-core-go/ethereum/mocks"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 )
@@ -17,8 +17,8 @@ func TestCalculateOffsetStats(t *testing.T) {
 	defer cancel()
 
 	ctrl := gomock.NewController(t)
-	primaryClient := mock_inspect.NewMockProxyAPIClient(ctrl)
-	secondaryClient := mock_inspect.NewMockProxyAPIClient(ctrl)
+	primaryClient := mock_ethereum.NewMockEthClient(ctrl)
+	secondaryClient := mock_ethereum.NewMockEthClient(ctrl)
 
 	// Test when everything is successful
 	primaryClient.EXPECT().BlockNumber(gomock.Any()).Return(uint64(5), nil)
diff --git a/inspect/proxy_api_test.go b/inspect/proxy_api_test.go
index d0d5df70..b8bdfbcd 100644
--- a/inspect/proxy_api_test.go
+++ b/inspect/proxy_api_test.go
@@ -2,26 +2,75 @@ package inspect
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"math/big"
 	"testing"
 
 	types "github.com/ethereum/go-ethereum/core/types"
+	"github.com/forta-network/forta-core-go/ethereum"
 	mock_ethereum "github.com/forta-network/forta-core-go/ethereum/mocks"
+	"github.com/forta-network/forta-core-go/registry"
+	mock_registry "github.com/forta-network/forta-core-go/registry/mocks"
 	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
 func TestProxyAPIInspection(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping inspection test in short mode")
+	r := require.New(t)
+	ctx := context.Background()
+
+	ctrl := gomock.NewController(t)
+	rpcClient := mock_ethereum.NewMockRPCClient(ctrl)
+	ethClient := mock_ethereum.NewMockEthClient(ctrl)
+	regClient := mock_registry.NewMockClient(ctrl)
+
+	RPCDialContext = func(ctx context.Context, rawurl string) (ethereum.RPCClient, error) {
+		return rpcClient, nil
+	}
+	EthClientDialContext = func(ctx context.Context, rawurl string) (ethereum.EthClient, error) {
+		return ethClient, nil
+	}
+	RegistryNewClient = func(ctx context.Context, cfg registry.ClientConfig) (registry.Client, error) {
+		return regClient, nil
 	}
 
-	r := require.New(t)
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "net_version").
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"0x5"`), result)
+			return nil
+		}).AnyTimes()
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_chainId").
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"0x5"`), result)
+			return nil
+		}).AnyTimes()
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "web3_clientVersion").Return(nil)
+	ethClient.EXPECT().BlockNumber(gomock.Any()).Return(uint64(123), nil)
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_getBlockByNumber", gomock.Any()).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"{}"`), result)
+			return nil
+		})
+
+	// oldest supported block inspection calls
+	ethClient.EXPECT().BlockByNumber(gomock.Any(), big.NewInt(VeryOldBlockNumber)).Return(&types.Block{}, nil)
+
+	// eth2 support inspection calls
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_getBlockByNumber", "latest", true).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`{"difficulty":"0x0","nonce":"0x0000000000000000"}`), result)
+			return nil
+		})
+
+	// offset inspection calls
+	ethClient.EXPECT().BlockNumber(gomock.Any()).Return(uint64(123), nil).AnyTimes()
+	ethClient.EXPECT().BlockByNumber(gomock.Any(), gomock.Any()).Return(&types.Block{}, nil).AnyTimes()
 
 	inspector := &ProxyAPIInspector{}
 	results, err := inspector.Inspect(
-		context.Background(), InspectionConfig{},
+		ctx, InspectionConfig{},
 	)
 	r.NoError(err)
 
@@ -44,7 +93,7 @@ func TestProxyAPIInspection(t *testing.T) {
 
 	r.Equal(
 		map[string]string{
-			MetadataProxyAPIBlockByNumberHash: "7232705dbb71b9d5ef65891c2c6e7020137ffb652ed938a88621b322f09ab4a4",
+			MetadataProxyAPIBlockByNumberHash: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
 		}, results.Metadata,
 	)
 }
@@ -52,9 +101,6 @@ func TestProxyAPIInspection(t *testing.T) {
 func TestFindOldestSupportedBlock(t *testing.T) {
 	// Create a new Gomock controller
 	ctrl := gomock.NewController(t)
-	defer ctrl.Finish()
-
-	// Create a mock ethclient.Client
 	mockClient := mock_ethereum.NewMockEthClient(ctrl)
 
 	// Create a test context
diff --git a/inspect/registry_api_test.go b/inspect/registry_api_test.go
index 1b612c92..c014972e 100644
--- a/inspect/registry_api_test.go
+++ b/inspect/registry_api_test.go
@@ -4,31 +4,40 @@ import (
 	"context"
 	"testing"
 
-	"github.com/kelseyhightower/envconfig"
+	"github.com/forta-network/forta-core-go/ethereum"
+	mock_ethereum "github.com/forta-network/forta-core-go/ethereum/mocks"
+	"github.com/forta-network/forta-core-go/registry"
+	mock_registry "github.com/forta-network/forta-core-go/registry/mocks"
+	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 )
 
-var testRegistryEnv struct {
-	RegistryAPI string `envconfig:"registry_api" default:"https://rpc.ankr.com/polygon"`
-}
+func TestRegistryAPIInspection(t *testing.T) {
+	r := require.New(t)
 
-func init() {
-	envconfig.MustProcess("test", &testRegistryEnv)
-}
+	scannerAddr := "0x3DC45b47B7559Ca3b231E5384D825F9B461A0398"
 
-func TestRegistryAPIInspection(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping inspection test in short mode")
+	ctrl := gomock.NewController(t)
+	rpcClient := mock_ethereum.NewMockRPCClient(ctrl)
+	ethClient := mock_ethereum.NewMockEthClient(ctrl)
+	regClient := mock_registry.NewMockClient(ctrl)
+
+	RPCDialContext = func(ctx context.Context, rawurl string) (ethereum.RPCClient, error) {
+		return rpcClient, nil
+	}
+	EthClientDialContext = func(ctx context.Context, rawurl string) (ethereum.EthClient, error) {
+		return ethClient, nil
+	}
+	RegistryNewClient = func(ctx context.Context, cfg registry.ClientConfig) (registry.Client, error) {
+		return regClient, nil
 	}
 
-	r := require.New(t)
+	regClient.EXPECT().GetAssignmentHash(scannerAddr).Return(&registry.AssignmentHash{}, nil)
 
 	inspector := &RegistryAPIInspector{}
 	results, err := inspector.Inspect(
 		context.Background(), InspectionConfig{
-			RegistryAPIURL:     testRegistryEnv.RegistryAPI,
-			ENSContractAddress: "0x08f42fcc52a9C2F391bF507C4E8688D0b53e1bd7",
-			ScannerAddress:     "0x3DC45b47B7559Ca3b231E5384D825F9B461A0398",
+			ScannerAddress: scannerAddr,
 		},
 	)
 	r.NoError(err)
diff --git a/inspect/scan_api_test.go b/inspect/scan_api_test.go
index 8b7a6a39..b3634ce3 100644
--- a/inspect/scan_api_test.go
+++ b/inspect/scan_api_test.go
@@ -2,35 +2,63 @@ package inspect
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
 
-	"github.com/kelseyhightower/envconfig"
+	"github.com/forta-network/forta-core-go/ethereum"
+	mock_ethereum "github.com/forta-network/forta-core-go/ethereum/mocks"
+	"github.com/forta-network/forta-core-go/registry"
+	mock_registry "github.com/forta-network/forta-core-go/registry/mocks"
+	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 )
 
-var testScanEnv struct {
-	ScanAPI string `envconfig:"scan_api" default:"https://goerli.infura.io/v3/9aa3d95b3bc440fa88ea12eaa4456161"`
-}
+func TestScanAPIInspection(t *testing.T) {
+	r := require.New(t)
 
-func init() {
-	envconfig.MustProcess("test", &testScanEnv)
-}
+	ctrl := gomock.NewController(t)
+	rpcClient := mock_ethereum.NewMockRPCClient(ctrl)
+	ethClient := mock_ethereum.NewMockEthClient(ctrl)
+	regClient := mock_registry.NewMockClient(ctrl)
 
-func TestScanAPIInspection(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping inspection test in short mode")
+	RPCDialContext = func(ctx context.Context, rawurl string) (ethereum.RPCClient, error) {
+		return rpcClient, nil
+	}
+	EthClientDialContext = func(ctx context.Context, rawurl string) (ethereum.EthClient, error) {
+		return ethClient, nil
+	}
+	RegistryNewClient = func(ctx context.Context, cfg registry.ClientConfig) (registry.Client, error) {
+		return regClient, nil
 	}
 
-	r := require.New(t)
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "net_version").
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"0x5"`), result)
+			return nil
+		}).AnyTimes()
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_chainId").
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"0x5"`), result)
+			return nil
+		}).AnyTimes()
+
+	// block response hash inspection
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_getBlockByNumber", gomock.Any()).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"{}"`), result)
+			return nil
+		})
 
-	recentBlockNumber := testGetRecentBlockNumber(r, testScanEnv.ScanAPI)
+	// eth2 support inspection calls
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_getBlockByNumber", "latest", true).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`{"difficulty":"0x0","nonce":"0x0000000000000000"}`), result)
+			return nil
+		})
 
 	inspector := &ScanAPIInspector{}
 	results, err := inspector.Inspect(
-		context.Background(), InspectionConfig{
-			ScanAPIURL:  testScanEnv.ScanAPI,
-			BlockNumber: recentBlockNumber,
-		},
+		context.Background(), InspectionConfig{},
 	)
 	r.NoError(err)
 
diff --git a/inspect/trace_api_test.go b/inspect/trace_api_test.go
index afbe613b..3e42dc80 100644
--- a/inspect/trace_api_test.go
+++ b/inspect/trace_api_test.go
@@ -2,36 +2,64 @@ package inspect
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/ethclient"
-	"github.com/kelseyhightower/envconfig"
+	"github.com/forta-network/forta-core-go/ethereum"
+	mock_ethereum "github.com/forta-network/forta-core-go/ethereum/mocks"
+	"github.com/forta-network/forta-core-go/registry"
+	mock_registry "github.com/forta-network/forta-core-go/registry/mocks"
+	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 )
 
-var testTraceEnv struct {
-	TraceAPI string `envconfig:"trace_api" default:"https://rpcapi-tracing.testnet.fantom.network/"`
-}
+func TestTraceAPIInspection(t *testing.T) {
+	r := require.New(t)
 
-func init() {
-	envconfig.MustProcess("test", &testTraceEnv)
-}
+	ctrl := gomock.NewController(t)
+	rpcClient := mock_ethereum.NewMockRPCClient(ctrl)
+	ethClient := mock_ethereum.NewMockEthClient(ctrl)
+	regClient := mock_registry.NewMockClient(ctrl)
 
-func TestTraceAPIInspection(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping inspection test in short mode")
+	RPCDialContext = func(ctx context.Context, rawurl string) (ethereum.RPCClient, error) {
+		return rpcClient, nil
+	}
+	EthClientDialContext = func(ctx context.Context, rawurl string) (ethereum.EthClient, error) {
+		return ethClient, nil
+	}
+	RegistryNewClient = func(ctx context.Context, cfg registry.ClientConfig) (registry.Client, error) {
+		return regClient, nil
 	}
 
-	r := require.New(t)
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "net_version").
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"4002"`), result)
+			return nil
+		}).AnyTimes()
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_chainId").
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"4002"`), result)
+			return nil
+		}).AnyTimes()
+
+	// trace response hash inspection
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "trace_block", gomock.Any()).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"{}"`), result)
+			return nil
+		})
 
-	recentBlockNumber := testGetRecentBlockNumber(r, testTraceEnv.TraceAPI)
+	// block response hash inspection
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_getBlockByNumber", gomock.Any()).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"{}"`), result)
+			return nil
+		})
 
 	inspector := &TraceAPIInspector{}
 	results, err := inspector.Inspect(
 		context.Background(), InspectionConfig{
-			TraceAPIURL: testTraceEnv.TraceAPI,
-			BlockNumber: recentBlockNumber,
-			CheckTrace:  true,
+			CheckTrace: true,
 		},
 	)
 	r.NoError(err)
@@ -48,11 +76,3 @@ func TestTraceAPIInspection(t *testing.T) {
 	r.NotEmpty(results.Metadata[MetadataTraceAPIBlockByNumberHash])
 	r.NotEmpty(results.Metadata[MetadataTraceAPITraceBlockHash])
 }
-
-func testGetRecentBlockNumber(r *require.Assertions, apiURL string) uint64 {
-	client, err := ethclient.Dial(apiURL)
-	r.NoError(err)
-	block, err := client.BlockByNumber(context.Background(), nil)
-	r.NoError(err)
-	return block.NumberU64() - 100
-}
diff --git a/inspect/validation/validate.go b/inspect/validation/validate.go
index 86cbc1cd..21e2cc52 100644
--- a/inspect/validation/validate.go
+++ b/inspect/validation/validate.go
@@ -5,7 +5,6 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/forta-network/forta-core-go/ethereum"
 	"github.com/forta-network/forta-core-go/inspect"
 	"github.com/hashicorp/go-multierror"
@@ -33,19 +32,19 @@ func NewValidator(ctx context.Context, inspectionCfg inspect.InspectionConfig) (
 		validator InspectionValidator
 		err       error
 	)
-	validator.scanRpcClient, err = rpc.DialContext(ctx, inspectionCfg.ScanAPIURL)
+	validator.scanRpcClient, err = inspect.RPCDialContext(ctx, inspectionCfg.ScanAPIURL)
 	if err != nil {
 		log.WithError(err).Error("failed to dial scan api")
 		return nil, inspect.ErrReferenceScanAPI
 	}
 	if inspectionCfg.CheckTrace {
-		validator.traceRpcClient, err = rpc.DialContext(ctx, inspectionCfg.TraceAPIURL)
+		validator.traceRpcClient, err = inspect.RPCDialContext(ctx, inspectionCfg.TraceAPIURL)
 		if err != nil {
 			log.WithError(err).Error("failed to dial trace api")
 			return nil, inspect.ErrReferenceTraceAPI
 		}
 	}
-	validator.proxyRpcClient, err = rpc.DialContext(ctx, inspectionCfg.ProxyAPIURL)
+	validator.proxyRpcClient, err = inspect.RPCDialContext(ctx, inspectionCfg.ProxyAPIURL)
 	if err != nil {
 		log.WithError(err).Error("failed to dial proxy api")
 		return nil, inspect.ErrReferenceProxyAPI
diff --git a/inspect/validation/validate_test.go b/inspect/validation/validate_test.go
index 8051ab1a..983fd22a 100644
--- a/inspect/validation/validate_test.go
+++ b/inspect/validation/validate_test.go
@@ -2,52 +2,62 @@ package validation
 
 import (
 	"context"
+	"encoding/json"
 	"testing"
 
-	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/forta-network/forta-core-go/ethereum"
+	mock_ethereum "github.com/forta-network/forta-core-go/ethereum/mocks"
 	"github.com/forta-network/forta-core-go/inspect"
-	"github.com/kelseyhightower/envconfig"
+	"github.com/golang/mock/gomock"
 	"github.com/stretchr/testify/require"
 )
 
-var testValidateEnv struct {
-	ScanAPI  string `envconfig:"scan_api" default:"https://fantom-testnet.public.blastapi.io/"`
-	TraceAPI string `envconfig:"trace_api" default:"https://rpcapi-tracing.testnet.fantom.network/"`
-}
-
-func init() {
-	envconfig.MustProcess("test", &testValidateEnv)
-}
-
 func TestValidateInspectionSuccess(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skipping inspection validation test in short mode")
-	}
-
 	ctx := context.Background()
 	r := require.New(t)
 
-	recentBlockNumber := testGetRecentBlockNumber(r, testValidateEnv.ScanAPI)
+	ctrl := gomock.NewController(t)
+	rpcClient := mock_ethereum.NewMockRPCClient(ctrl)
+
+	inspect.RPCDialContext = func(ctx context.Context, rawurl string) (ethereum.RPCClient, error) {
+		return rpcClient, nil
+	}
+
 	inspectionCfg := inspect.InspectionConfig{
-		ScanAPIURL:  testValidateEnv.ScanAPI,
-		ProxyAPIURL: testValidateEnv.ScanAPI,
-		TraceAPIURL: testValidateEnv.TraceAPI,
-		BlockNumber: recentBlockNumber,
-		CheckTrace:  true,
+		CheckTrace: true,
 	}
 
-	// make only scan and trace api inspections using the inspection config
-	results, err := inspect.InspectAll(ctx, []inspect.Inspector{
-		&inspect.ScanAPIInspector{},
-		&inspect.ProxyAPIInspector{},
-		&inspect.TraceAPIInspector{},
-	}, inspectionCfg)
-	r.NoError(err)
+	expectedHash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+
+	// trace response hash inspection
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "trace_block", gomock.Any()).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"{}"`), result)
+			return nil
+		})
+
+	// block response hash inspection
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_getBlockByNumber", gomock.Any()).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"{}"`), result)
+			return nil
+		}).Times(3)
 
 	// validate the inspection using the same config
 	validator, err := NewValidator(ctx, inspectionCfg)
 	r.NoError(err)
-	_, err = validator.Validate(ctx, results)
+	_, err = validator.Validate(ctx, &inspect.InspectionResults{
+		Inputs: inspect.InspectionConfig{
+			BlockNumber: 10,
+			CheckTrace:  true,
+		},
+		Metadata: map[string]string{
+			inspect.MetadataScanAPIBlockByNumberHash:  expectedHash,
+			inspect.MetadataProxyAPIBlockByNumberHash: expectedHash,
+			inspect.MetadataTraceAPIBlockByNumberHash: expectedHash,
+			inspect.MetadataTraceAPITraceBlockHash:    expectedHash,
+		},
+	})
 	r.NoError(err)
 }
 
@@ -55,42 +65,49 @@ func TestValidateInspectionFail(t *testing.T) {
 	ctx := context.Background()
 	r := require.New(t)
 
-	recentBlockNumber := testGetRecentBlockNumber(r, testValidateEnv.ScanAPI)
-	inspectionCfg1 := inspect.InspectionConfig{
-		ScanAPIURL:  testValidateEnv.ScanAPI,
-		ProxyAPIURL: testValidateEnv.ScanAPI,
-		TraceAPIURL: testValidateEnv.TraceAPI,
-		BlockNumber: recentBlockNumber,
-		CheckTrace:  true,
-	}
+	ctrl := gomock.NewController(t)
+	rpcClient := mock_ethereum.NewMockRPCClient(ctrl)
 
-	// make only scan api inspection
-	results, err := inspect.InspectAll(ctx, []inspect.Inspector{
-		&inspect.ScanAPIInspector{},
-		&inspect.ProxyAPIInspector{},
-	}, inspectionCfg1)
-	r.NoError(err)
+	inspect.RPCDialContext = func(ctx context.Context, rawurl string) (ethereum.RPCClient, error) {
+		return rpcClient, nil
+	}
 
-	// now let's tamper with the initial conditions so trace inspection result is different
-	inspectionCfg2 := inspect.InspectionConfig{
-		ScanAPIURL:  testValidateEnv.ScanAPI,
-		ProxyAPIURL: testValidateEnv.ScanAPI,
-		TraceAPIURL: testValidateEnv.TraceAPI,
-		BlockNumber: recentBlockNumber - 10,
-		CheckTrace:  true,
+	inspectionCfg := inspect.InspectionConfig{
+		CheckTrace: true,
 	}
 
-	// make only trace api inspection
-	traceResults, err := inspect.InspectAll(ctx, []inspect.Inspector{
-		&inspect.TraceAPIInspector{},
-	}, inspectionCfg2)
-	r.NoError(err)
-	results.CopyFrom(traceResults)
+	expectedHash := "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+	unexpectedHash := "foobar"
+
+	// trace response hash inspection
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "trace_block", gomock.Any()).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"{}"`), result)
+			return nil
+		})
+
+	// block response hash inspection
+	rpcClient.EXPECT().CallContext(gomock.Any(), gomock.Any(), "eth_getBlockByNumber", gomock.Any()).
+		DoAndReturn(func(ctx interface{}, result interface{}, method interface{}, args ...interface{}) error {
+			json.Unmarshal([]byte(`"{}"`), result)
+			return nil
+		}).Times(3)
 
 	// validate the inspection using the first config
-	validator, err := NewValidator(ctx, inspectionCfg1)
+	validator, err := NewValidator(ctx, inspectionCfg)
 	r.NoError(err)
-	verrs, err := validator.Validate(ctx, results)
+	verrs, err := validator.Validate(ctx, &inspect.InspectionResults{
+		Inputs: inspect.InspectionConfig{
+			BlockNumber: 10,
+			CheckTrace:  true,
+		},
+		Metadata: map[string]string{
+			inspect.MetadataScanAPIBlockByNumberHash:  expectedHash,
+			inspect.MetadataProxyAPIBlockByNumberHash: expectedHash,
+			inspect.MetadataTraceAPIBlockByNumberHash: unexpectedHash,
+			inspect.MetadataTraceAPITraceBlockHash:    unexpectedHash,
+		},
+	})
 
 	// expect error(s)
 	r.Error(err)
@@ -107,11 +124,3 @@ func TestValidateInspectionFail(t *testing.T) {
 	r.True(verrs.HasCode(inspect.ErrResultTraceAPIBlockMismatch.Code()))
 	r.True(verrs.HasCode(inspect.ErrResultTraceAPITraceBlockMismatch.Code()))
 }
-
-func testGetRecentBlockNumber(r *require.Assertions, apiURL string) uint64 {
-	client, err := ethclient.Dial(apiURL)
-	r.NoError(err)
-	block, err := client.BlockByNumber(context.Background(), nil)
-	r.NoError(err)
-	return block.NumberU64() - 20
-}

From 3dfb95fb08ba3b9545e96e86c8beb4efb9bfa0d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Caner=20=C3=87=C4=B1dam?= <canercidam01@gmail.com>
Date: Thu, 19 Oct 2023 22:03:49 +0300
Subject: [PATCH 26/26] remove some debug lines

---
 inspect/api.go | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/inspect/api.go b/inspect/api.go
index 5c20cefd..b0d2aa7f 100644
--- a/inspect/api.go
+++ b/inspect/api.go
@@ -147,31 +147,25 @@ func SupportsETH2(ctx context.Context, rpcClient ethereum.RPCClient) bool {
 
 	var block domain.Block
 	if err := getRpcResponse(ctx, rpcClient, &block, blockByNumber, latestBlock, true); err != nil {
-		fmt.Println("@@@@@ response failed", err)
 		return false
 	}
 	if block.Difficulty == nil {
-		fmt.Println("@@@@@ nil difficulty", block.Difficulty)
 		return false
 	}
 	if block.Nonce == nil {
-		fmt.Println("@@@@@ nil nonce", block.Nonce)
 		return false
 	}
 	difficulty, err := hexutil.DecodeBig(*block.Difficulty)
 	if err != nil {
-		fmt.Println("@@@@@ error parsing difficulty", block.Difficulty, err)
 		return false
 	}
 	var nonce types.BlockNonce
 	if err := (&nonce).UnmarshalText([]byte(*block.Nonce)); err != nil {
-		fmt.Println("@@@@@ error parsing nonce", *block.Nonce)
 		return false
 	}
 
 	if difficulty.Sign() == 0 && nonce.Uint64() == 0 {
 		return true
 	}
-	fmt.Println("@@@@@ one of the numbers is not zero", difficulty, nonce)
 	return false
 }