Merge from main

backend/adapter_processor_v2/adapter_processor.py

@@ -92,10 +92,11 @@ def test_adapter(adapter_id: str, adapter_metadata: dict[str, Any]) -> bool:
 
             adapter_instance = adapter_class(adapter_metadata)
             test_result: bool = adapter_instance.test_connection()
-            logger.info(f"{adapter_id} test result: {test_result}")
             return test_result
         except SdkError as e:
-            raise TestAdapterError(str(e))
+            raise TestAdapterError(
+                e, adapter_name=adapter_metadata[AdapterKeys.ADAPTER_NAME]
+            )
 
     @staticmethod
     def __fetch_adapters_by_key_value(key: str, value: Any) -> Adapter:

backend/adapter_processor_v2/exceptions.py

@@ -2,6 +2,7 @@
 
 from adapter_processor_v2.constants import AdapterKeys
 from rest_framework.exceptions import APIException
+from unstract.sdk.exceptions import SdkError
 
 
 class IdIsMandatory(APIException):
@@ -51,6 +52,20 @@ class TestAdapterError(APIException):
     status_code = 500
     default_detail = "Error while testing adapter"
 
+    def __init__(
+        self,
+        sdk_err: SdkError,
+        detail: Optional[str] = None,
+        code: Optional[str] = None,
+        adapter_name: Optional[str] = None,
+    ):
+        if sdk_err.status_code:
+            self.status_code = sdk_err.status_code
+        if detail is None:
+            adapter_name = f"'{adapter_name}'" if adapter_name else "adapter"
+            detail = f"Error testing {adapter_name}. {str(sdk_err)}"
+        super().__init__(detail, code)
+
 
 class TestAdapterInputError(APIException):
     status_code = 400

backend/api_v2/deployment_helper.py

@@ -1,5 +1,4 @@
 import logging
-import uuid
 from typing import Any, Optional
 from urllib.parse import urlencode
 
@@ -24,7 +23,8 @@
 from workflow_manager.endpoint_v2.source import SourceConnector
 from workflow_manager.workflow_v2.dto import ExecutionResponse
 from workflow_manager.workflow_v2.enums import ExecutionStatus
-from workflow_manager.workflow_v2.models.workflow import Workflow
+from workflow_manager.workflow_v2.execution import WorkflowExecutionServiceHelper
+from workflow_manager.workflow_v2.models import Workflow, WorkflowExecution
 from workflow_manager.workflow_v2.workflow_helper import WorkflowHelper
 
 logger = logging.getLogger(__name__)
@@ -152,7 +152,12 @@ def execute_workflow(
         """
         workflow_id = api.workflow.id
         pipeline_id = api.id
-        execution_id = str(uuid.uuid4())
+        workflow_execution = WorkflowExecutionServiceHelper.create_workflow_execution(
+            workflow_id=workflow_id,
+            pipeline_id=pipeline_id,
+            mode=WorkflowExecution.Mode.QUEUE,
+        )
+        execution_id = workflow_execution.id
 
         hash_values_of_files = SourceConnector.add_input_file_to_api_storage(
             workflow_id=workflow_id,

backend/prompt_studio/prompt_studio_core_v2/exceptions.py

@@ -24,6 +24,10 @@ class AnswerFetchError(APIException):
     status_code = 500
     default_detail = "Error occured while fetching response for the prompt"
 
+    def __init__(self, detail: Optional[str] = None, status_code: int = 500):
+        super().__init__(detail)
+        self.status_code = status_code
+
 
 class DefaultProfileError(APIException):
     status_code = 500

backend/prompt_studio/prompt_studio_core_v2/prompt_studio_helper.py

@@ -894,7 +894,8 @@ def _fetch_response(
             error_message = answer.get("error", "")
             raise AnswerFetchError(
                 "Error while fetching response for "
-                f"'{prompt.prompt_key}' with '{doc_name}'. {error_message}"
+                f"'{prompt.prompt_key}' with '{doc_name}'. {error_message}",
+                status_code=int(answer.get("status_code")),
             )
         output_response = json.loads(answer["structure_output"])
         return output_response

backend/prompt_studio/prompt_studio_core_v2/views.py

@@ -437,13 +437,15 @@ def fetch_contents_ide(self, request: HttpRequest, pk: Any = None) -> Response:
                     file_path += file_name
                 contents = FileManagerHelper.fetch_file_contents(file_system, file_path)
         else:
-            contents = PromptStudioFileHelper.fetch_file_contents(
-                file_name=file_name,
-                org_id=UserSessionUtils.get_organization_id(request),
-                user_id=custom_tool.created_by.user_id,
-                tool_id=str(custom_tool.tool_id),
-            )
-
+            try:
+                contents = PromptStudioFileHelper.fetch_file_contents(
+                    file_name=file_name,
+                    org_id=UserSessionUtils.get_organization_id(request),
+                    user_id=custom_tool.created_by.user_id,
+                    tool_id=str(custom_tool.tool_id),
+                )
+            except FileNotFoundError:
+                raise FileNotFound()
         return Response({"data": contents}, status=status.HTTP_200_OK)
 
     @action(detail=True, methods=["post"])

backend/pyproject.toml

@@ -32,7 +32,7 @@ dependencies = [
     "python-socketio==5.9.0", # For log_events
     "social-auth-app-django==5.3.0", # For OAuth
     "social-auth-core==4.4.2", # For OAuth
-    "unstract-sdk~=0.54.0rc5",
+    "unstract-sdk~=0.54.0rc6",
     # ! IMPORTANT!
     # Indirect local dependencies usually need to be added in their own projects
     # as: https://pdm-project.org/latest/usage/dependency/#local-dependencies.

backend/sample.env

@@ -116,12 +116,14 @@ SYSTEM_ADMIN_EMAIL="[email protected]"
 SESSION_COOKIE_AGE=86400
 
 
-# Control async extraction of LLM Whisperer
+# Control async extraction of LLMWhisperer
 # Time in seconds to wait before polling LLMWhisperer's status API
 ADAPTER_LLMW_POLL_INTERVAL=30
 # Total number of times to poll the status API.
 # 500 mins to allow 1500 (max pages limit) * 20 (approx time in sec to process a page)
 ADAPTER_LLMW_MAX_POLLS=1000
+# Number of times to retry the /whisper-status API before failing the extraction
+ADAPTER_LLMW_STATUS_RETRIES=5
 
 # Enable logging of workflow history.
 ENABLE_LOG_HISTORY=True

backend/tool_instance_v2/tool_processor.py

@@ -17,7 +17,6 @@
 
 class ToolProcessor:
     TOOL_NOT_IN_REGISTRY_MESSAGE = "Tool does not exist in registry"
-    tool_registry = ToolRegistry()
 
     @staticmethod
     def get_tool_by_uid(tool_uid: str) -> Tool:

backend/workflow_manager/endpoint_v2/destination.py

@@ -10,6 +10,7 @@
 from connector_v2.models import ConnectorInstance
 from fsspec.implementations.local import LocalFileSystem
 from unstract.sdk.constants import ToolExecKey
+from unstract.sdk.file_storage.constants import FileOperationParams
 from unstract.sdk.tool.mime_types import EXT_MIME_MAP
 from unstract.workflow_execution.constants import ToolOutputType
 from utils.user_context import UserContext
@@ -457,7 +458,9 @@ def get_result_with_file_storage(
         file_storage = file_system.get_file_storage()
         try:
             # TODO: SDK handles validation; consider removing here.
-            file_type = file_storage.mime_type(output_file)
+            file_type = file_storage.mime_type(
+                path=output_file, read_length=FileOperationParams.READ_ENTIRE_LENGTH
+            )
             if output_type == ToolOutputType.JSON:
                 if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
                     logger.error(f"Output type json mismatched {file_type}")

backend/workflow_manager/workflow_v2/workflow_helper.py

@@ -465,8 +465,12 @@ def execute_workflow_async(
             WorkflowExecutionServiceHelper.update_execution_err(
                 execution_id, str(error)
             )
-            logger.error(f"Errors while job enqueueing {str(error)}")
-            logger.error(f"Error {traceback.format_exc()}")
+            logger.error(
+                f"Error while enqueuing async job for WF '{workflow_id}', "
+                f"execution '{execution_id}': {str(error)}",
+                exc_info=True,
+                stack_info=True,
+            )
             return ExecutionResponse(
                 workflow_id,
                 execution_id,

frontend/src/components/log-in/Login.css

@@ -79,7 +79,6 @@
 }
 
 .text-wrapper {
-    margin: 40px 0px;
     width: 100%;
     display: flex;
     flex-direction: column;

frontend/src/index.css

@@ -175,3 +175,7 @@ body {
 .ant-tabs-nav-operations {
   display: none !important;
 }
+
+.text-align-center {
+  text-align: center;
+}