Skip to content

Commit

Permalink
Add processed kvs to catchup logs (#168)
Browse files Browse the repository at this point in the history 
* Add processed kvs to catchup logs

* Add to unit test
  • Loading branch information
@jasonpaulos
jasonpaulos authored Jun 6, 2024
1 parent 20c0512 commit a05ce2c
Show file tree
Hide file tree
Showing 2 changed files with 36 additions and 13 deletions.
5 changes: 4 additions & 1 deletion conduit/plugins/importers/algod/algod_importer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -144,11 +144,14 @@ func (algodImp *algodImporter) monitorCatchpointCatchup() error {
running = stat.Catchpoint != ""
switch {
case !running:
break
// break out of loop
case stat.CatchpointAcquiredBlocks > 0:
algodImp.logger.Infof("catchup phase Acquired Blocks: %d / %d", stat.CatchpointAcquiredBlocks, stat.CatchpointTotalBlocks)
// VerifiedKvs are not shown because kv verification appears to be interleaved with account verification
case stat.CatchpointVerifiedAccounts > 0:
algodImp.logger.Infof("catchup phase Verified Accounts: %d / %d", stat.CatchpointVerifiedAccounts, stat.CatchpointTotalAccounts)
case stat.CatchpointProcessedKvs > 0:
algodImp.logger.Infof("catchup phase Processed KVs: %d / %d", stat.CatchpointProcessedKvs, stat.CatchpointTotalKvs)
case stat.CatchpointProcessedAccounts > 0:
algodImp.logger.Infof("catchup phase Processed Accounts: %d / %d", stat.CatchpointProcessedAccounts, stat.CatchpointTotalAccounts)
default:
Expand Down
44 changes: 32 additions & 12 deletions conduit/plugins/importers/algod/algod_importer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,9 +309,14 @@ func TestInitCatchup(t *testing.T) {
MakePostSyncRoundResponder(http.StatusOK),
MakeJsonResponderSeries("/v2/status", []int{http.StatusOK}, []interface{}{
models.NodeStatus{LastRound: 1235},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedAccounts: 1, CatchpointTotalAccounts: 1},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointVerifiedAccounts: 1, CatchpointTotalAccounts: 1},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointAcquiredBlocks: 1, CatchpointTotalBlocks: 1},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedAccounts: 1, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedKvs: 2, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointVerifiedAccounts: 4, CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointVerifiedAccounts: 10, CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointAcquiredBlocks: 3, CatchpointTotalBlocks: 30, CatchpointVerifiedAccounts: 10, CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointAcquiredBlocks: 30, CatchpointTotalBlocks: 30, CatchpointVerifiedAccounts: 10, CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd"},
models.NodeStatus{LastRound: 1236},
}),
Expand All @@ -320,9 +325,14 @@ func TestInitCatchup(t *testing.T) {
errInit: "received unexpected error (StatusAfterBlock) waiting for node to catchup: did not reach expected round",
errGetGen: "",
logs: []string{
"catchup phase Processed Accounts: 1 / 1",
"catchup phase Verified Accounts: 1 / 1",
"catchup phase Acquired Blocks: 1 / 1",
"catchup phase Processed Accounts: 1 / 10",
"catchup phase Processed Accounts: 10 / 10",
"catchup phase Processed KVs: 2 / 20",
"catchup phase Processed KVs: 20 / 20",
"catchup phase Verified Accounts: 4 / 10",
"catchup phase Verified Accounts: 10 / 10",
"catchup phase Acquired Blocks: 3 / 30",
"catchup phase Acquired Blocks: 30 / 30",
"catchup phase Verified Blocks",
}},
{
Expand All @@ -336,9 +346,14 @@ func TestInitCatchup(t *testing.T) {
MakePostSyncRoundResponder(http.StatusOK),
MakeJsonResponderSeries("/v2/status", []int{http.StatusOK}, []interface{}{
models.NodeStatus{LastRound: 1235},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedAccounts: 1, CatchpointTotalAccounts: 1},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointVerifiedAccounts: 1, CatchpointTotalAccounts: 1},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointAcquiredBlocks: 1, CatchpointTotalBlocks: 1},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedAccounts: 1, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedKvs: 2, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointVerifiedAccounts: 4, CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointVerifiedAccounts: 10, CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointAcquiredBlocks: 3, CatchpointTotalBlocks: 30, CatchpointVerifiedAccounts: 10, CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd", CatchpointAcquiredBlocks: 30, CatchpointTotalBlocks: 30, CatchpointVerifiedAccounts: 10, CatchpointProcessedKvs: 20, CatchpointTotalKvs: 20, CatchpointProcessedAccounts: 10, CatchpointTotalAccounts: 10},
models.NodeStatus{Catchpoint: "1236#abcd"},
models.NodeStatus{LastRound: 1237}, // this is the only difference from the previous test
}),
Expand All @@ -347,9 +362,14 @@ func TestInitCatchup(t *testing.T) {
errInit: "",
errGetGen: "",
logs: []string{
"catchup phase Processed Accounts: 1 / 1",
"catchup phase Verified Accounts: 1 / 1",
"catchup phase Acquired Blocks: 1 / 1",
"catchup phase Processed Accounts: 1 / 10",
"catchup phase Processed Accounts: 10 / 10",
"catchup phase Processed KVs: 2 / 20",
"catchup phase Processed KVs: 20 / 20",
"catchup phase Verified Accounts: 4 / 10",
"catchup phase Verified Accounts: 10 / 10",
"catchup phase Acquired Blocks: 3 / 30",
"catchup phase Acquired Blocks: 30 / 30",
"catchup phase Verified Blocks",
}},
}
Expand Down

0 comments on commit a05ce2c

Please sign in to comment.