chore: re-work http id list syncing (#62)

* chore: re-work http id list syncing

* chore: add more tests around id list syncing

* chore: add ability to shutdown id list adapter

* chore: locking id lists in spec store

* chore: refactor idlist update listener trait

* chore: checking id list in evaluator

* chore: make id list optional

statsig-lib/src/evaluation/evaluator.rs

@@ -161,13 +161,42 @@ fn evaluate_condition<'a>(ctx: &mut EvaluatorContext<'a>, condition: &'a Conditi
         "eq" => value == target_value,
         "neq" => value != target_value,
 
+        // id_lists
+        "in_segment_list" | "not_in_segment_list" => {
+            evaluate_id_list(ctx, operator, target_value, value)
+        }
+
         _ => {
             ctx.result.unsupported = true;
             return;
         }
     }
 }
 
+fn evaluate_id_list<'a>(
+    ctx: &mut EvaluatorContext<'a>,
+    op: &str,
+    target_value: &DynamicValue,
+    value: &DynamicValue,
+) -> bool {
+    let list_name = unwrap_or_return!(&target_value.string_value, false);
+    let id_lists = &ctx.spec_store_data.id_lists;
+
+    let list = unwrap_or_return!(id_lists.get(list_name), false);
+
+    let value = unwrap_or_return!(&value.string_value, false);
+    let hashed = ctx.sha_hasher.hash_name(value);
+    let lookup_id: String = hashed.chars().take(8).collect();
+
+    let is_in_list = list.ids.contains(&lookup_id);
+
+    if op == "not_in_segment_list" {
+        return !is_in_list;
+    }
+
+    is_in_list
+}
+
 fn evaluate_nested_gate<'a>(
     ctx: &mut EvaluatorContext<'a>,
     target_value: &'a DynamicValue,

statsig-lib/src/id_lists_adapter/id_list.rs

@@ -0,0 +1,65 @@
+use serde::Serialize;
+
+use crate::{
+    id_lists_adapter::{IdListMetadata, IdListUpdate},
+    unwrap_or_noop,
+};
+use std::collections::HashSet;
+
+#[derive(Clone, Serialize)]
+pub struct IdList {
+    pub metadata: IdListMetadata,
+
+    #[serde(skip_serializing)]
+    pub ids: HashSet<String>,
+}
+
+impl IdList {
+    pub fn new(metadata: IdListMetadata) -> Self {
+        let mut local_metadata = metadata;
+        local_metadata.size = 0;
+
+        Self {
+            metadata: local_metadata,
+            ids: HashSet::new(),
+        }
+    }
+
+    pub fn apply_update(&mut self, update: &IdListUpdate) {
+        let changed = &update.new_metadata;
+        let current = &self.metadata;
+
+        if changed.file_id != current.file_id && changed.creation_time >= current.creation_time {
+            self.reset();
+        }
+
+        let changeset_data = unwrap_or_noop!(&update.raw_changeset);
+
+        for change in changeset_data.lines() {
+            let trimmed = change.trim();
+            if trimmed.len() <= 1 {
+                continue;
+            }
+
+            let op = change.chars().next();
+            let id = &change[1..];
+
+            match op {
+                Some('+') => {
+                    self.ids.insert(id.to_string());
+                }
+                Some('-') => {
+                    self.ids.remove(id);
+                }
+                _ => continue,
+            }
+        }
+
+        self.metadata.size += changeset_data.len() as u64;
+    }
+
+    pub fn reset(&mut self) {
+        self.metadata.size = 0;
+        self.ids.clear();
+    }
+}

statsig-lib/src/id_lists_adapter/id_lists_adapter_trait.rs

@@ -1,36 +1,45 @@
 use crate::StatsigErr;
 use async_trait::async_trait;
-use serde::{Deserialize, Serialize};
-use std::collections::{HashMap, HashSet};
+use std::collections::HashMap;
 use std::sync::Arc;
 use std::time::Duration;
 use tokio::runtime::Handle;
 
-pub type IdListsResponse = HashMap<String, IdListEntry>;
+use serde::{Deserialize, Serialize};
 
 #[derive(Serialize, Deserialize, Debug, Clone)]
 #[serde(rename_all = "camelCase")]
-pub struct IdListEntry {
+pub struct IdListMetadata {
     pub name: String,
-    pub url: Option<String>,
+    pub url: String,
 
     #[serde(rename = "fileID")]
     pub file_id: Option<String>,
 
     pub size: u64,
     pub creation_time: i64,
+}
 
-    #[serde(skip)]
-    pub loaded_ids: HashSet<String>,
+pub struct IdListUpdate {
+    pub raw_changeset: Option<String>,
+    pub new_metadata: IdListMetadata,
 }
 
 #[async_trait]
 pub trait IdListsAdapter: Send + Sync {
-    async fn start(self: Arc<Self>, runtime_handle: &Handle) -> Result<(), StatsigErr>;
+    async fn start(
+        self: Arc<Self>,
+        runtime_handle: &Handle,
+        listener: Arc<dyn IdListsUpdateListener + Send + Sync>,
+    ) -> Result<(), StatsigErr>;
+
+    async fn shutdown(&self, timeout: Duration) -> Result<(), StatsigErr>;
 
     async fn sync_id_lists(&self) -> Result<(), StatsigErr>;
+}
 
-    fn does_list_contain_id(&self, list_name: &str, id: &str) -> bool;
+pub trait IdListsUpdateListener: Send + Sync {
+    fn get_current_id_list_metadata(&self) -> HashMap<String, IdListMetadata>;
 
-    async fn shutdown(&self, timeout: Duration) -> Result<(), StatsigErr>;
+    fn did_receive_id_list_updates(&self, updates: HashMap<String, IdListUpdate>);
 }

statsig-lib/src/id_lists_adapter/mod.rs

@@ -1,3 +1,7 @@
+pub use id_list::*;
 pub use id_lists_adapter_trait::*;
+pub use statsig_http_id_lists_adapter::*;
+
+mod statsig_http_id_lists_adapter;
+mod id_list;
 mod id_lists_adapter_trait;
-mod statsig_http_id_lists_adapter;