feat: Add contract sync liveness metrics (#5181)

### Description Add contract sync liveness metrics ### Related issues - Contributes into #5098 ### Backward compatibility Yes ### Testing E2E testing in CI --------- Co-authored-by: Danil Nemirovsky <[email protected]>
hyperlane-xyz · Jan 16, 2025 · ee5124e · ee5124e
1 parent a17cbaa
commit ee5124e
Show file tree

Hide file tree

Showing 2 changed files with 27 additions and 0 deletions.
diff --git a/rust/main/hyperlane-base/src/contract_sync/metrics.rs b/rust/main/hyperlane-base/src/contract_sync/metrics.rs
@@ -26,6 +26,9 @@ pub struct ContractSyncMetrics {
     /// See `last_known_message_nonce` in CoreMetrics.
     pub message_nonce: IntGaugeVec,
 
+    /// Contract sync liveness metric
+    pub liveness_metrics: IntGaugeVec,
+
     /// Metrics for SequenceAware and RateLimited cursors.
     pub cursor_metrics: Arc<CursorMetrics>,
 }
@@ -49,13 +52,22 @@ impl ContractSyncMetrics {
             )
             .expect("failed to register stored_events metric");
 
+        let liveness_metrics = metrics
+            .new_int_gauge(
+                "contract_sync_liveness",
+                "Last timestamp observed by contract sync",
+                &["data_type", "chain"],
+            )
+            .expect("failed to register liveness metric");
+
         let message_nonce = metrics.last_known_message_nonce();
         let cursor_metrics = Arc::new(CursorMetrics::new(metrics));
 
         ContractSyncMetrics {
             indexed_height,
             stored_events,
             message_nonce,
+            liveness_metrics,
             cursor_metrics,
         }
     }

diff --git a/rust/main/hyperlane-base/src/contract_sync/mod.rs b/rust/main/hyperlane-base/src/contract_sync/mod.rs
@@ -1,5 +1,6 @@
 use std::{
     collections::HashSet, fmt::Debug, hash::Hash, marker::PhantomData, sync::Arc, time::Duration,
+    time::UNIX_EPOCH,
 };
 
 use axum::async_trait;
@@ -98,8 +99,13 @@ where
             .metrics
             .stored_events
             .with_label_values(&[label, chain_name]);
+        let liveness_metric = self
+            .metrics
+            .liveness_metrics
+            .with_label_values(&[label, chain_name]);
 
         loop {
+            Self::update_liveness_metric(&liveness_metric);
             if let Some(rx) = opts.tx_id_receiver.as_mut() {
                 self.fetch_logs_from_receiver(rx, &stored_logs_metric).await;
             }
@@ -120,6 +126,15 @@ where
         info!(chain = chain_name, label, "contract sync loop exit");
     }
 
+    fn update_liveness_metric(liveness_metric: &GenericGauge<AtomicI64>) {
+        liveness_metric.set(
+            UNIX_EPOCH
+                .elapsed()
+                .map(|d| d.as_secs() as i64)
+                .unwrap_or(0),
+        );
+    }
+
     #[instrument(fields(domain=self.domain().name()), skip(self, recv, stored_logs_metric))]
     async fn fetch_logs_from_receiver(
         &self,