diff --git a/uptrain/operators/code/detection.py b/uptrain/operators/code/detection.py index 8c8b24af..0310b8bb 100644 --- a/uptrain/operators/code/detection.py +++ b/uptrain/operators/code/detection.py @@ -56,7 +56,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -70,7 +70,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["context"] = row.pop(self.col_context) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate("code_hallucination", data_send) diff --git a/uptrain/operators/language/context_quality.py b/uptrain/operators/language/context_quality.py index 7b1ea139..ae372ccc 100644 --- a/uptrain/operators/language/context_quality.py +++ b/uptrain/operators/language/context_quality.py @@ -78,7 +78,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -91,7 +91,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["context"] = row.pop(self.col_context) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( @@ -224,7 +224,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -238,7 +238,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["context"] = row.pop(self.col_context) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( @@ -382,7 +382,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -396,7 +396,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["reranked_context"] = row.pop(self.col_reranked_context) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( @@ -530,7 +530,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -544,7 +544,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["concise_context"] = row.pop(self.col_concise_context) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( diff --git a/uptrain/operators/language/conversation.py b/uptrain/operators/language/conversation.py index 4b55ed46..eb1eb103 100644 --- a/uptrain/operators/language/conversation.py +++ b/uptrain/operators/language/conversation.py @@ -67,7 +67,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -79,7 +79,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["conversation"] = row[self.col_conversation] try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( diff --git a/uptrain/operators/language/factual_accuracy.py b/uptrain/operators/language/factual_accuracy.py index acc536e6..17c91747 100644 --- a/uptrain/operators/language/factual_accuracy.py +++ b/uptrain/operators/language/factual_accuracy.py @@ -70,7 +70,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -84,7 +84,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["context"] = row.pop(self.col_context) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( diff --git a/uptrain/operators/language/guideline.py b/uptrain/operators/language/guideline.py index 4b4f298e..ae171ef9 100644 --- a/uptrain/operators/language/guideline.py +++ b/uptrain/operators/language/guideline.py @@ -68,7 +68,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -81,7 +81,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["response"] = row.pop(self.col_response) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( diff --git a/uptrain/operators/language/jailbreak.py b/uptrain/operators/language/jailbreak.py index 8261345c..f95ee1f7 100644 --- a/uptrain/operators/language/jailbreak.py +++ b/uptrain/operators/language/jailbreak.py @@ -66,7 +66,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -78,7 +78,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["question"] = row.pop(self.col_question) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( @@ -214,7 +214,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -226,7 +226,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["question"] = row.pop(self.col_question) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( diff --git a/uptrain/operators/language/language_quality.py b/uptrain/operators/language/language_quality.py index ac963c9b..1fbba235 100644 --- a/uptrain/operators/language/language_quality.py +++ b/uptrain/operators/language/language_quality.py @@ -72,7 +72,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -84,7 +84,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["response"] = row.pop(self.col_response) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate("critique_language", data_send) @@ -212,7 +212,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -224,7 +224,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["response"] = row.pop(self.col_response) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate("critique_language", data_send) diff --git a/uptrain/operators/language/response_quality.py b/uptrain/operators/language/response_quality.py index f44ffc14..872a1632 100644 --- a/uptrain/operators/language/response_quality.py +++ b/uptrain/operators/language/response_quality.py @@ -77,7 +77,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -90,7 +90,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["response"] = row.pop(self.col_response) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( @@ -223,7 +223,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -236,7 +236,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["response"] = row.pop(self.col_response) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( @@ -366,7 +366,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -378,7 +378,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["response"] = row.pop(self.col_response) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( @@ -507,7 +507,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -519,7 +519,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["response"] = row.pop(self.col_response) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( @@ -646,7 +646,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -658,7 +658,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["response"] = row.pop(self.col_response) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( @@ -775,7 +775,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): # TODO: Add support for local evaluation for all methods if self.method != "llm": raise Exception( @@ -796,7 +796,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["ground_truth"] = row.pop(self.col_ground_truth) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( diff --git a/uptrain/operators/language/subquery.py b/uptrain/operators/language/subquery.py index 590a51ee..d593e6de 100644 --- a/uptrain/operators/language/subquery.py +++ b/uptrain/operators/language/subquery.py @@ -59,7 +59,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -72,7 +72,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["sub_questions"] = row.pop(self.col_sub_questions) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate( diff --git a/uptrain/operators/language/tone.py b/uptrain/operators/language/tone.py index f41eed5f..b440ecfb 100644 --- a/uptrain/operators/language/tone.py +++ b/uptrain/operators/language/tone.py @@ -66,7 +66,7 @@ def setup(self, settings: t.Optional[Settings] = None): assert settings is not None self.settings = settings - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): self._api_client = LLMMulticlient(settings) else: self._api_client = APIClient(settings) @@ -78,7 +78,7 @@ def run(self, data: pl.DataFrame) -> TYPE_TABLE_OUTPUT: row["response"] = row.pop(self.col_response) try: - if self.settings.evaluate_locally: + if self.settings.evaluate_locally and (self.settings.uptrain_access_token is None or not len(self.settings.uptrain_access_token)): results = self.evaluate_local(data_send) else: results = self._api_client.evaluate(