fix: Increase timeout for block stream health check (#957)

Increasing timeout for the block stream health check to ensure unwanted stalled flagging of properly executing block streams. When QueryApi is restarted, a thundering herd situation can lead to slower queries.
near · Jul 31, 2024 · 7af836b · 7af836b
1 parent 73d1fdd
commit 7af836b
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 3 deletions.
diff --git a/block-streamer/src/block_stream.rs b/block-streamer/src/block_stream.rs
@@ -131,7 +131,7 @@ impl BlockStream {
                     redis.get_last_processed_block(&config).await.unwrap();
 
                 loop {
-                    tokio::time::sleep(std::time::Duration::from_secs(15)).await;
+                    tokio::time::sleep(std::time::Duration::from_secs(120)).await;
 
                     let new_last_processed_block =
                         if let Ok(block) = redis.get_last_processed_block(&config).await {

diff --git a/coordinator/src/handlers/block_streams.rs b/coordinator/src/handlers/block_streams.rs
@@ -248,7 +248,7 @@ impl BlockStreamsHandler {
             let updated_at =
                 SystemTime::UNIX_EPOCH + Duration::from_secs(health.updated_at_timestamp_secs);
 
-            let stale = updated_at.elapsed().unwrap_or_default() > Duration::from_secs(60);
+            let stale = updated_at.elapsed().unwrap_or_default() > Duration::from_secs(180);
             let stalled = matches!(
                 health.processing_state.try_into(),
                 Ok(ProcessingState::Stalled)
@@ -257,7 +257,11 @@ impl BlockStreamsHandler {
             if !stale && !stalled {
                 return Ok(());
             } else {
-                tracing::info!(stale, stalled, "Restarting stalled block stream");
+                tracing::info!(
+                    stale,
+                    stalled,
+                    "Restarting stalled block stream after {RESTART_TIMEOUT_SECONDS} seconds"
+                );
             }
         } else {
             tracing::info!(