pin some deps so installing the package works (#303)

* work around pkg deps

* add instructor embedding

* add data folder for tests

* add test csv file

* pin pydantic version

.gitignore

@@ -17,10 +17,7 @@ dist
 env
 
 # Dataset/Log files
-*.csv
-*.json
 *.log
-data
 
 # UpTrain
 uptrain_*

setup.py

@@ -12,6 +12,9 @@
 this_directory = Path(__file__).parent
 long_description = (this_directory / "README.md").read_text()
 
+# sentence-tranformers deps (it installs torch-gpu by default)
+# pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+
 setup(
     name="uptrain",
     version="0.2.0",
@@ -44,7 +47,7 @@
     install_requires=[
         # utilities
         "tqdm>=4.0",
-        "pydantic>=1.9.0",
+        "pydantic<1.10.10",
         "aiolimiter>=1.1",
         "loguru",
         "lazy_loader",
@@ -73,10 +76,12 @@
             "xgboost",
         ],
         "full": [
-            "river",
+            "river<0.18",
             "scikit_learn>=1.0.0",
             "umap-learn",
             "rouge-score",
+            "sentence-transformers",
+            "InstructorEmbedding",
         ],
     },  # Optional
     tests_require=["pytest>=7.0"],

uptrain/tests/test_operators.py → tests/test_operators.py

@@ -1,6 +1,8 @@
+import os
 from uptrain.framework import Settings
 
 SETTINGS = Settings()
+SELF_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
 # uptrain.operators.drift
@@ -17,7 +19,7 @@ def test_concept_drift():
 
     # Run the operator on the input data
     input_data = (
-        CsvReader(fpath="uptrain/tests/data/predictions.csv")
+        CsvReader(fpath=os.path.join(SELF_DIR, "data/predictions.csv"))
         .setup(SETTINGS)
         .run()["output"]
     )
@@ -64,7 +66,7 @@ def test_embs_cosine_distribution():
 
     # Run the operator on the input data
     input_data = (
-        JsonReader(fpath="uptrain/tests/data/qna_on_docs_samples.jsonl")
+        JsonReader(fpath=os.path.join(SELF_DIR, "data/qna_on_docs_samples.jsonl"))
         .setup(SETTINGS)
         .run()["output"]
     )
@@ -89,7 +91,7 @@ def test_embs_rouge_score():
     )
     # Run the operator on the input data
     input_data = (
-        JsonReader(fpath="uptrain/tests/data/qna_on_docs_samples.jsonl")
+        JsonReader(fpath=os.path.join(SELF_DIR, "data/qna_on_docs_samples.jsonl"))
         .setup(SETTINGS)
         .run()["output"]
     )
@@ -110,7 +112,7 @@ def test_embs_umap_operator():
 
     # Run the operator on the input data
     input_data = (
-        JsonReader(fpath="uptrain/tests/data/qna_on_docs_samples.jsonl")
+        JsonReader(fpath=os.path.join(SELF_DIR, "data/qna_on_docs_samples.jsonl"))
         .setup(SETTINGS)
         .run()["output"]
     )
@@ -161,7 +163,7 @@ def test_accuracy_operator():
 
     # Run the operator on the input data
     input_data = (
-        CsvReader(fpath="uptrain/tests/data/predictions.csv")
+        CsvReader(fpath=os.path.join(SELF_DIR, "data/predictions.csv"))
         .setup(SETTINGS)
         .run()["output"]
     )

uptrain/framework/remote.py

@@ -29,14 +29,14 @@ def add(self, name: str, fpath: str):
 
     def get(self, name: str, version: t.Optional[int] = None):
         url = f"{self.base_url}/dataset"
-        params = {"name": name}
+        params: dict = {"name": name}
         if version is not None:
             params["version"] = version
         response = self.client.get(url, params=params)
         return response.json()
 
     def list(self, skip: int = 0, limit: int = 100):
-        url = f"{self.client.base_url}/datasets"
+        url = f"{self.base_url}/datasets"
         params = {"skip": skip, "limit": limit}
         response = self.client.get(url, params=params)
         return response.json()
@@ -54,7 +54,7 @@ def add(self, name: str, checkset: CheckSet):
 
     def get(self, name: str, version: t.Optional[int] = None):
         url = f"{self.base_url}/checkset"
-        params = {"name": name}
+        params: dict = {"name": name}
         if version is not None:
             params["version"] = version
         response = self.client.get(url, params=params)
@@ -67,36 +67,12 @@ def list(self, skip: int = 0, limit: int = 100):
         return response.json()
 
 
-class APIClient:
+@dataclass
+class RunsAPI:
     base_url: str
     client: httpx.Client
-    datasets: DatasetsAPI
-    checksets: ChecksetsAPI
-
-    def __init__(self, settings: Settings) -> None:
-        server_url = settings.check_and_get("uptrain_server_url")
-        api_key = settings.check_and_get("uptrain_access_token")
 
-        self.base_url = server_url.rstrip("/") + "/api/public"
-        self.client = httpx.Client(headers={"uptrain-access-token": api_key})
-        self.datasets = DatasetsAPI(base_url=self.base_url, client=self.client)
-        self.checksets = ChecksetsAPI(base_url=self.base_url, client=self.client)
-
-    def check_auth(self):
-        """Ping the server to check if the client is authenticated."""
-        url = f"{self.base_url}/auth"
-        try:
-            response = self.client.get(url)
-            if not response.is_success:
-                raise RuntimeError(
-                    f"Failed to authenticate with the Uptrain server: {response.json()}"
-                )
-        except httpx.ConnectError as e:
-            raise RuntimeError(
-                f"Failed to connect to the Uptrain server at {self.base_url}"
-            ) from e
-
-    def add_run(self, dataset: str, checkset: str) -> dict:
+    def add(self, dataset: str, checkset: str) -> dict:
         """Schedules an evaluation on the server. Specify the dataset and checkset to use.
 
         Args:
@@ -112,7 +88,7 @@ def add_run(self, dataset: str, checkset: str) -> dict:
         )
         return response.json()
 
-    def get_run_status(self, run_id: str) -> str:
+    def get(self, run_id: str) -> str:
         """Get the status of a run.
 
         Args:
@@ -125,22 +101,54 @@ def get_run_status(self, run_id: str) -> str:
         response = self.client.get(url)
         return response.json()
 
-    def list_all_active_runs(self):
+    def list(self, only_active: bool = True):
         """List all the runs on the server.
 
         - filter by scheduled/completed/in-process?
         - filter by timestamp?
         """
         ...
 
-    def cancel_active_run(self):
+    def cancel(self, run_id: str):
         """Cancel an active run on the server."""
         ...
 
-    def get_logs_for_run(self):
+    def get_logs(self, run_id: str):
         """Get logs for a run."""
         ...
 
-    def get_results_for_run(self):
+    def get_results(self, run_id: str):
         """Get results for a run."""
         ...
+
+
+class APIClient:
+    base_url: str
+    client: httpx.Client
+    datasets: DatasetsAPI
+    checksets: ChecksetsAPI
+    runs: RunsAPI
+
+    def __init__(self, settings: Settings) -> None:
+        server_url = settings.check_and_get("uptrain_server_url")
+        api_key = settings.check_and_get("uptrain_access_token")
+
+        self.base_url = server_url.rstrip("/") + "/api/public"
+        self.client = httpx.Client(headers={"uptrain-access-token": api_key})
+        self.datasets = DatasetsAPI(base_url=self.base_url, client=self.client)
+        self.checksets = ChecksetsAPI(base_url=self.base_url, client=self.client)
+        self.runs = RunsAPI(base_url=self.base_url, client=self.client)
+
+    def check_auth(self):
+        """Ping the server to check if the client is authenticated."""
+        url = f"{self.base_url}/auth"
+        try:
+            response = self.client.get(url)
+            if not response.is_success:
+                raise RuntimeError(
+                    f"Failed to authenticate with the Uptrain server: {response.json()}"
+                )
+        except httpx.ConnectError as e:
+            raise RuntimeError(
+                f"Failed to connect to the Uptrain server at {self.base_url}"
+            ) from e