fix checkset run for managed user (#572)

uptrain-ai · Mar 1, 2024 · 9ba6489 · 9ba6489
1 parent ad9dc4d
commit 9ba6489
Show file tree

Hide file tree

Showing 10 changed files with 40 additions and 40 deletions.
diff --git a/uptrain/operators/code/detection.py b/uptrain/operators/code/detection.py
@@ -56,7 +56,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -70,7 +70,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["context"] = row.pop(self.col_context)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate("code_hallucination", data_send)

diff --git a/uptrain/operators/language/context_quality.py b/uptrain/operators/language/context_quality.py
@@ -78,7 +78,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -91,7 +91,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["context"] = row.pop(self.col_context)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(
@@ -224,7 +224,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -238,7 +238,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["context"] = row.pop(self.col_context)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(
@@ -382,7 +382,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -396,7 +396,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["reranked_context"] = row.pop(self.col_reranked_context)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(
@@ -530,7 +530,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -544,7 +544,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["concise_context"] = row.pop(self.col_concise_context)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(

diff --git a/uptrain/operators/language/conversation.py b/uptrain/operators/language/conversation.py
@@ -67,7 +67,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -79,7 +79,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["conversation"] = row[self.col_conversation]
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(

diff --git a/uptrain/operators/language/factual_accuracy.py b/uptrain/operators/language/factual_accuracy.py
@@ -70,7 +70,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -84,7 +84,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["context"] = row.pop(self.col_context)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(

diff --git a/uptrain/operators/language/guideline.py b/uptrain/operators/language/guideline.py
@@ -68,7 +68,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -81,7 +81,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["response"] = row.pop(self.col_response)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(

diff --git a/uptrain/operators/language/jailbreak.py b/uptrain/operators/language/jailbreak.py
@@ -66,7 +66,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -78,7 +78,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["question"] = row.pop(self.col_question)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(
@@ -214,7 +214,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -226,7 +226,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["question"] = row.pop(self.col_question)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(

diff --git a/uptrain/operators/language/language_quality.py b/uptrain/operators/language/language_quality.py
@@ -72,7 +72,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -84,7 +84,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["response"] = row.pop(self.col_response)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate("critique_language", data_send)
@@ -212,7 +212,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -224,7 +224,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["response"] = row.pop(self.col_response)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate("critique_language", data_send)

diff --git a/uptrain/operators/language/response_quality.py b/uptrain/operators/language/response_quality.py
@@ -77,7 +77,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -90,7 +90,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["response"] = row.pop(self.col_response)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(
@@ -223,7 +223,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -236,7 +236,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["response"] = row.pop(self.col_response)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(
@@ -366,7 +366,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -378,7 +378,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["response"] = row.pop(self.col_response)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(
@@ -507,7 +507,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -519,7 +519,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["response"] = row.pop(self.col_response)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(
@@ -646,7 +646,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -658,7 +658,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["response"] = row.pop(self.col_response)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(
@@ -775,7 +775,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             # TODO: Add support for local evaluation for all methods
             if self.method != "llm":
                 raise Exception(
@@ -796,7 +796,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["ground_truth"] = row.pop(self.col_ground_truth)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(

diff --git a/uptrain/operators/language/subquery.py b/uptrain/operators/language/subquery.py
@@ -59,7 +59,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -72,7 +72,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["sub_questions"] = row.pop(self.col_sub_questions)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(

diff --git a/uptrain/operators/language/tone.py b/uptrain/operators/language/tone.py
@@ -66,7 +66,7 @@ def setup(self, settings: t.Optional[Settings] = None):
 
         assert settings is not None
         self.settings = settings
-        if self.settings.evaluate_locally:
+        if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
             self._api_client = LLMMulticlient(settings)
         else:
             self._api_client = APIClient(settings)
@@ -78,7 +78,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT:
             row["response"] = row.pop(self.col_response)
 
         try:
-            if self.settings.evaluate_locally:
+            if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)):
                 results = self.evaluate_local(data_send)
             else:
                 results = self._api_client.evaluate(