Merge branch 'main' into fix/confidence-data-to-prompt-service

backend/pyproject.toml

@@ -37,7 +37,7 @@ dependencies = [
     # Indirect local dependencies usually need to be added in their own projects
     # as: https://pdm-project.org/latest/usage/dependency/#local-dependencies.
     #
-    # However, such indirect local dependencies which are not direct depedency of
+    # However, such indirect local dependencies which are not direct dependency of
     # main project appear as absolute paths in pdm.lock of main project, making it
     # impossible to check in the lock file.
     #
@@ -77,6 +77,7 @@ dev = [
     "-e unstract-tool-registry @ file:///${PROJECT_ROOT}/../unstract/tool-registry",
     "-e unstract-tool-sandbox @ file:///${PROJECT_ROOT}/../unstract/tool-sandbox",
     "-e unstract-workflow-execution @ file:///${PROJECT_ROOT}/../unstract/workflow-execution",
+    "-e unstract-filesystem @ file:///${PROJECT_ROOT}/../unstract/filesystem",
 ]
 
 [tool.pytest.ini_options]

docker/scripts/pdm-lock-gen/pdm-lock.sh

@@ -73,6 +73,7 @@ directories=(
     "backend"
     "prompt-service"
     "worker"
+    "unstract/filesystem"
     "unstract/core"
     "unstract/flags"
     "platform-service"

tools/classifier/requirements.txt

@@ -1,4 +1,6 @@
 # Add your dependencies here
 
 # Required for all unstract tools
-unstract-sdk~=0.53.1
+unstract-sdk~=0.54.0rc5
+# TODO: remove once it added in sdk
+s3fs[boto3]==2024.6.0

tools/structure/requirements.txt

@@ -1,4 +1,6 @@
 # Add your dependencies here
 
 # Required for all unstract tools
-unstract-sdk~=0.53.2
+unstract-sdk~=0.54.0rc5
+# TODO: remove once it added in sdk
+s3fs[boto3]==2024.6.0

tools/structure/src/main.py

@@ -6,7 +6,7 @@
 from typing import Any, Callable, Optional
 
 from constants import SettingsKeys  # type: ignore [attr-defined]
-from unstract.sdk.constants import LogLevel, LogState, MetadataKey
+from unstract.sdk.constants import LogLevel, LogState, MetadataKey, ToolEnv
 from unstract.sdk.index import Index
 from unstract.sdk.prompt import PromptTool
 from unstract.sdk.tool.base import BaseTool
@@ -97,7 +97,10 @@ def run(
         _, file_name = os.path.split(input_file)
         if summarize_as_source:
             file_name = SettingsKeys.SUMMARIZE
-        tool_data_dir = Path(self.get_env_or_die(SettingsKeys.TOOL_DATA_DIR))
+        if hasattr(self, "workflow_filestorage"):
+            tool_data_dir = Path(self.get_env_or_die(ToolEnv.EXECUTION_DATA_DIR))
+        else:
+            tool_data_dir = Path(self.get_env_or_die(SettingsKeys.TOOL_DATA_DIR))
         execution_run_data_folder = Path(
             self.get_env_or_die(SettingsKeys.EXECUTION_RUN_DATA_FOLDER)
         )
@@ -253,8 +256,13 @@ def run(
             self.stream_log("Writing parsed output...")
             source_name = self.get_exec_metadata.get(MetadataKey.SOURCE_NAME)
             output_path = Path(output_dir) / f"{Path(source_name).stem}.json"
-            with open(output_path, "w", encoding="utf-8") as f:
-                f.write(structured_output)
+            if hasattr(self, "workflow_filestorage"):
+                self.workflow_filestorage.json_dump(
+                    path=output_path, data=structured_output_dict
+                )
+            else:
+                with open(output_path, "w", encoding="utf-8") as f:
+                    f.write(structured_output)
         except OSError as e:
             self.stream_error_and_exit(f"Error creating output file: {e}")
         except json.JSONDecodeError as e:
@@ -336,8 +344,13 @@ def _summarize_and_index(
             structure_output = json.loads(response[SettingsKeys.STRUCTURE_OUTPUT])
             summarized_context = structure_output.get(SettingsKeys.DATA, "")
             self.stream_log("Writing summarized context to a file")
-            with open(summarize_file_path, "w", encoding="utf-8") as f:
-                f.write(summarized_context)
+            if hasattr(self, "workflow_filestorage"):
+                self.workflow_filestorage.write(
+                    path=summarize_file_path, mode="w", data=summarized_context
+                )
+            else:
+                with open(summarize_file_path, "w", encoding="utf-8") as f:
+                    f.write(summarized_context)
 
         self.stream_log("Indexing summarized context")
         summarize_file_hash: str = ToolUtils.get_hash_from_file(

tools/text_extractor/requirements.txt

@@ -1,4 +1,6 @@
 # Add your dependencies here
 
 # Required for all unstract tools
-unstract-sdk~=0.53.1
+unstract-sdk~=0.54.0rc5
+# TODO: remove once it added in sdk
+s3fs[boto3]==2024.6.0