Feature/bci 2996 local finality violation (#13048)

* implement RPCClient changes

* simplify interception of RPC's view on chain

* Remove redundant changes

* remove redundant changes

* do not pick RPC as active if it's lagging on finalized block

* fix race

* minor refactor

* Added FinalizedBlockOffset to config

* HeadTracker support of FinalizedBlockOffset

* fix lint issues

* switch logpoller to head tracker

* remove redundant changes

* fix build

* Capture latest chain info on RPC level to avoid race on state transition if we are communicating with RPC with highest finalized block

* EnforceRepeatableRead config option

* ensure HeadTracked does not miss block if it's committed  between initialHead processing and subscription

* hot fix flakey test

* fix build & changeset

* remove outdated test

* fix headtracker tests

* use polling in head tracker to get latest finalized block

* fix relay

* regen mocks

* use proper context in the LogPoller's latestBlocks

* fix flaky test

* fixes for changes from develop branch

* replace panic with FailNow

* report dead node only if it's dead for long enough

* fix racy test

* rollback foundry change caused by merge with develop

* use active node as source of HighestChainInfo to reduce number of transitions to nodeStateFinalizedBlockOutOfSync

* optimised loading of latest finalized block

* fix merge conflicts

* make generate

* simplify capturing of app layer observations

* ensure logpoller is not affected by FinalityTagBypass

* fix comment

* reset keystone to develop

* reset keystone

* reset to develop

* ungoimport

* move newChainIDSubForwarder into the test

* fix typo

* Move deathDeclarationDelay to config

* Comments adjustments

* fix merge issues

* regen mocks

* fix multinode flaky test

* refactor SetTotalDifficultyIfGt to MaxTotalDifficulty

* Use softer language in LatestFinalizedBlock comments

Co-authored-by: Dimitris Grigoriou <[email protected]>

* nits

* Ensure HistoryDepth is >= FinalizedBlockOffset

* regen mocks with newer version

---------

Co-authored-by: Dimitris Grigoriou <[email protected]>
Co-authored-by: Domino Valdano <[email protected]>

.changeset/dull-ants-collect.md

@@ -0,0 +1,9 @@
+---
+"chainlink": patch
+---
+
+Fixed local finality violation caused by an RPC lagging behind on latest finalized block.
+
+Added `EVM.FinalizedBlockOffset` and `EVM.NodePool.EnforceRepeatableRead` config options.
+With `EnforceRepeatableRead = true`, RPC is considered healthy only if its most recent finalized block is larger or equal to the highest finalized block observed by the Node minus `FinalizedBlockOffset`.
+#bugfix

common/client/ctx.go

@@ -0,0 +1,17 @@
+package client
+
+import "context"
+
+type multiNodeContextKey int
+
+const (
+	contextKeyHeathCheckRequest multiNodeContextKey = iota + 1
+)
+
+func CtxAddHealthCheckFlag(ctx context.Context) context.Context {
+	return context.WithValue(ctx, contextKeyHeathCheckRequest, struct{}{})
+}
+
+func CtxIsHeathCheckRequest(ctx context.Context) bool {
+	return ctx.Value(contextKeyHeathCheckRequest) != nil
+}

common/client/ctx_test.go

@@ -0,0 +1,16 @@
+package client
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/smartcontractkit/chainlink-common/pkg/utils/tests"
+)
+
+func TestContext(t *testing.T) {
+	ctx := tests.Context(t)
+	assert.False(t, CtxIsHeathCheckRequest(ctx), "expected false for test context")
+	ctx = CtxAddHealthCheckFlag(ctx)
+	assert.True(t, CtxIsHeathCheckRequest(ctx), "expected context to contain the healthcheck flag")
+}

common/client/mocks/config.go

@@ -3,9 +3,10 @@ package mocks
 import "time"
 
 type ChainConfig struct {
-	IsFinalityTagEnabled   bool
-	FinalityDepthVal       uint32
-	NoNewHeadsThresholdVal time.Duration
+	IsFinalityTagEnabled    bool
+	FinalityDepthVal        uint32
+	NoNewHeadsThresholdVal  time.Duration
+	FinalizedBlockOffsetVal uint32
 }
 
 func (t ChainConfig) NodeNoNewHeadsThreshold() time.Duration {
@@ -19,3 +20,7 @@ func (t ChainConfig) FinalityDepth() uint32 {
 func (t ChainConfig) FinalityTagEnabled() bool {
 	return t.IsFinalityTagEnabled
 }
+
+func (t ChainConfig) FinalizedBlockOffset() uint32 {
+	return t.FinalizedBlockOffsetVal
+}