From 7af836b2309a8004d858aa54a72d7ea23f351e43 Mon Sep 17 00:00:00 2001
From: Darun Seethammagari <darun@near.org>
Date: Thu, 1 Aug 2024 05:00:31 +0530
Subject: [PATCH] fix: Increase timeout for block stream health check (#957)

Increasing timeout for the block stream health check to ensure unwanted
stalled flagging of properly executing block streams. When QueryApi is
restarted, a thundering herd situation can lead to slower queries.
---
 block-streamer/src/block_stream.rs        | 2 +-
 coordinator/src/handlers/block_streams.rs | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/block-streamer/src/block_stream.rs b/block-streamer/src/block_stream.rs
index a9af3f10..c9a25ac6 100644
--- a/block-streamer/src/block_stream.rs
+++ b/block-streamer/src/block_stream.rs
@@ -131,7 +131,7 @@ impl BlockStream {
                     redis.get_last_processed_block(&config).await.unwrap();
 
                 loop {
-                    tokio::time::sleep(std::time::Duration::from_secs(15)).await;
+                    tokio::time::sleep(std::time::Duration::from_secs(120)).await;
 
                     let new_last_processed_block =
                         if let Ok(block) = redis.get_last_processed_block(&config).await {
diff --git a/coordinator/src/handlers/block_streams.rs b/coordinator/src/handlers/block_streams.rs
index 01bb88a8..b3e9fb5c 100644
--- a/coordinator/src/handlers/block_streams.rs
+++ b/coordinator/src/handlers/block_streams.rs
@@ -248,7 +248,7 @@ impl BlockStreamsHandler {
             let updated_at =
                 SystemTime::UNIX_EPOCH + Duration::from_secs(health.updated_at_timestamp_secs);
 
-            let stale = updated_at.elapsed().unwrap_or_default() > Duration::from_secs(60);
+            let stale = updated_at.elapsed().unwrap_or_default() > Duration::from_secs(180);
             let stalled = matches!(
                 health.processing_state.try_into(),
                 Ok(ProcessingState::Stalled)
@@ -257,7 +257,11 @@ impl BlockStreamsHandler {
             if !stale && !stalled {
                 return Ok(());
             } else {
-                tracing::info!(stale, stalled, "Restarting stalled block stream");
+                tracing::info!(
+                    stale,
+                    stalled,
+                    "Restarting stalled block stream after {RESTART_TIMEOUT_SECONDS} seconds"
+                );
             }
         } else {
             tracing::info!(