From de572d5f983720cfcc84c3463894098fe896aa5a Mon Sep 17 00:00:00 2001
From: Boruch Chalk <boruchc@mobileye.com>
Date: Thu, 21 Mar 2024 12:46:54 +0200
Subject: [PATCH] remove caching object store as it stores stale keys

---
 .../polars-io/src/cloud/object_store_setup.rs | 39 -------------------
 1 file changed, 39 deletions(-)

diff --git a/crates/polars-io/src/cloud/object_store_setup.rs b/crates/polars-io/src/cloud/object_store_setup.rs
index bd518589ea974..35dbc4876c189 100644
--- a/crates/polars-io/src/cloud/object_store_setup.rs
+++ b/crates/polars-io/src/cloud/object_store_setup.rs
@@ -2,21 +2,10 @@ use std::sync::Arc;
 
 use object_store::local::LocalFileSystem;
 use object_store::ObjectStore;
-use once_cell::sync::Lazy;
 use polars_error::{polars_bail, to_compute_err, PolarsError, PolarsResult};
-use polars_utils::aliases::PlHashMap;
-use tokio::sync::RwLock;
-use url::Url;
 
 use super::{parse_url, CloudLocation, CloudOptions, CloudType};
 
-/// Object stores must be cached. Every object-store will do DNS lookups and
-/// get rate limited when querying the DNS (can take up to 5s).
-/// Other reasons are connection pools that must be shared between as much as possible.
-#[allow(clippy::type_complexity)]
-static OBJECT_STORE_CACHE: Lazy<RwLock<PlHashMap<String, Arc<dyn ObjectStore>>>> =
-    Lazy::new(Default::default);
-
 type BuildResult = PolarsResult<(CloudLocation, Arc<dyn ObjectStore>)>;
 
 #[allow(dead_code)]
@@ -27,16 +16,6 @@ fn err_missing_feature(feature: &str, scheme: &str) -> BuildResult {
     );
 }
 
-/// Get the key of a url for object store registration.
-/// The credential info will be removed
-fn url_to_key(url: &Url) -> String {
-    format!(
-        "{}://{}",
-        url.scheme(),
-        &url[url::Position::BeforeHost..url::Position::AfterPort],
-    )
-}
-
 /// Build an [`ObjectStore`] based on the URL and passed in url. Return the cloud location and an implementation of the object store.
 pub async fn build_object_store(
     url: &str,
@@ -49,16 +28,6 @@ pub async fn build_object_store(
     let parsed = parse_url(url).map_err(to_compute_err)?;
     let cloud_location = CloudLocation::from_url(&parsed)?;
 
-    let key = url_to_key(&parsed);
-    let mut allow_cache = true;
-
-    {
-        let cache = OBJECT_STORE_CACHE.read().await;
-        if let Some(store) = cache.get(&key) {
-            return Ok((cloud_location, store.clone()));
-        }
-    }
-
     #[cfg(any(feature = "aws", feature = "gcp", feature = "azure"))]
     let options = options.map(std::borrow::Cow::Borrowed).unwrap_or_default();
 
@@ -114,13 +83,5 @@ pub async fn build_object_store(
             return err_missing_feature("http", &cloud_location.scheme);
         },
     }?;
-    if allow_cache {
-        let mut cache = OBJECT_STORE_CACHE.write().await;
-        // Clear the cache if we surpass a certain amount of buckets.
-        if cache.len() > 32 {
-            cache.clear()
-        }
-        cache.insert(key, store.clone());
-    }
     Ok((cloud_location, store))
 }