added stale entry deletion

nym-node-status-api/migrations/002_session_stats.sql → nym-node-status-api/migrations/001_session_stats.sql

@@ -13,4 +13,5 @@ CREATE TABLE gateway_session_stats (
     UNIQUE (node_id, day) -- This constraint automatically creates an index
   );
 CREATE INDEX idx_gateway_session_stats_identity_key ON gateway_session_stats (gateway_identity_key);
+CREATE INDEX idx_gateway_session_stats_day ON gateway_session_stats (day);
 

nym-node-status-api/src/db/queries/gateways_stats.rs

@@ -3,13 +3,14 @@ use crate::{
     http::models::SessionStats,
 };
 use futures_util::TryStreamExt;
+use time::Date;
 use tracing::error;
 
 pub(crate) async fn insert_session_records(
     pool: &DbPool,
     records: Vec<GatewaySessionsRecord>,
 ) -> anyhow::Result<()> {
-    let mut db = pool.acquire().await?;
+    let mut tx = pool.begin().await?;
     for record in records {
         sqlx::query!(
             "INSERT OR IGNORE INTO gateway_session_stats
@@ -27,9 +28,10 @@ pub(crate) async fn insert_session_records(
             record.mixnet_sessions,
             record.unknown_sessions,
         )
-        .execute(&mut *db)
+        .execute(&mut *tx)
         .await?;
     }
+    tx.commit().await?;
 
     Ok(())
 }
@@ -64,3 +66,11 @@ pub(crate) async fn get_sessions_stats(pool: &DbPool) -> anyhow::Result<Vec<Sess
 
     Ok(items)
 }
+
+pub(crate) async fn delete_old_records(pool: &DbPool, cut_off: Date) -> anyhow::Result<()> {
+    let mut conn = pool.acquire().await?;
+    sqlx::query!("DELETE FROM gateway_session_stats WHERE day <= ?", cut_off)
+        .execute(&mut *conn)
+        .await?;
+    Ok(())
+}

nym-node-status-api/src/db/queries/mod.rs

@@ -15,4 +15,4 @@ pub(crate) use mixnodes::{
 };
 pub(crate) use summary::{get_summary, get_summary_history};
 
-pub(crate) use gateways_stats::{get_sessions_stats, insert_session_records};
+pub(crate) use gateways_stats::{delete_old_records, get_sessions_stats, insert_session_records};

nym-node-status-api/src/node_scraper/mod.rs

@@ -19,6 +19,7 @@ mod error;
 
 const FAILURE_RETRY_DELAY: Duration = Duration::from_secs(60);
 const REFRESH_INTERVAL: Duration = Duration::from_secs(21600); //6h, data only update once a day
+const STALE_DURATION: Duration = Duration::from_secs(86400 * 365); //one year
 
 #[instrument(level = "debug", name = "node_scraper", skip_all)]
 pub(crate) async fn spawn_in_background(db_pool: DbPool, nym_api_client_timeout: Duration) {
@@ -90,6 +91,12 @@ async fn run(
         .map(|_| {
             tracing::debug!("Session info written to DB!");
         })?;
+    let cut_off_date = (OffsetDateTime::now_utc() - STALE_DURATION).date();
+    queries::delete_old_records(pool, cut_off_date)
+        .await
+        .map(|_| {
+            tracing::debug!("Cleared old data before {}", cut_off_date);
+        })?;
 
     Ok(())
 }