feat: Measure lock wait time in LakeS3Client

block-streamer/src/lake_s3_client.rs

@@ -76,24 +76,33 @@ impl FuturesCache {
         }
     }
 
+    async fn lock(
+        &self,
+    ) -> tokio::sync::MutexGuard<'_, SizedCache<String, SharedGetObjectBytesFuture>> {
+        let timer = metrics::LAKE_CACHE_LOCK_WAIT_SECONDS.start_timer();
+
+        let lock = self.cache.lock().await;
+
+        timer.observe_duration();
+
+        lock
+    }
+
     #[cfg(test)]
     pub async fn get(&self, key: &str) -> Option<SharedGetObjectBytesFuture> {
-        let mut cache = self.cache.lock().await;
-        cache.cache_get(key).cloned()
+        self.lock().await.cache_get(key).cloned()
     }
 
     pub async fn get_or_set_with(
         &self,
         key: String,
         f: impl FnOnce() -> SharedGetObjectBytesFuture,
     ) -> SharedGetObjectBytesFuture {
-        let mut cache = self.cache.lock().await;
-        cache.cache_get_or_set_with(key, f).clone()
+        self.lock().await.cache_get_or_set_with(key, f).clone()
     }
 
     pub async fn remove(&self, key: &str) {
-        let mut cache = self.cache.lock().await;
-        cache.cache_remove(key);
+        self.lock().await.cache_remove(key);
     }
 }
 

block-streamer/src/metrics.rs

@@ -1,13 +1,18 @@
 use actix_web::{get, App, HttpServer, Responder};
 use lazy_static::lazy_static;
 use prometheus::{
-    register_int_counter, register_int_counter_vec, register_int_gauge_vec, Encoder, IntCounter,
-    IntCounterVec, IntGaugeVec,
+    register_histogram, register_int_counter, register_int_counter_vec, register_int_gauge_vec,
+    Encoder, Histogram, IntCounter, IntCounterVec, IntGaugeVec,
 };
 use tracing_subscriber::layer::Context;
 use tracing_subscriber::Layer;
 
 lazy_static! {
+    pub static ref LAKE_CACHE_LOCK_WAIT_SECONDS: Histogram = register_histogram!(
+        "queryapi_block_streamer_lake_cache_lock_wait_seconds",
+        "Time spent waiting for lock acquisition in LakeS3Client cache",
+    )
+    .unwrap();
     pub static ref LAKE_S3_GET_REQUEST_COUNT: IntCounter = register_int_counter!(
         "queryapi_block_streamer_lake_s3_get_request_count",
         "Number of requests made to S3 from near lake framework",