Cache Files for ETL Processing (#72)

* remove markdown tokenizer * submodules * submodule merge --------- Co-authored-by: JWittmeyer <[email protected]>
code-kern-ai · Oct 18, 2024 · 3e66b31 · 3e66b31
1 parent 834b4a0
commit 3e66b31
Show file tree

Hide file tree

Showing 4 changed files with 3 additions and 146 deletions.
diff --git a/app.py b/app.py
@@ -1,7 +1,7 @@
 from fastapi import FastAPI, responses, status
 
 
-from controller import task_manager, tokenization_manager, markdown_file_content
+from controller import task_manager, tokenization_manager
 from misc import util
 from handler import config_handler, tokenizer_handler
 from request_classes import (
@@ -101,21 +101,6 @@ def save_tokenizer_as_pickle(request: SaveTokenizer) -> responses.PlainTextRespo
     return responses.PlainTextResponse(status_code=status.HTTP_200_OK)
 
 
-@app.put("/cognition/rework-content/{org_id}/{file_id}/{step}")
-def rework_markdown_file_content(
-    org_id: str, file_id: str, step: str
-) -> responses.Response:
-    try:
-        r = markdown_file_content.rework_markdown_file_content(
-            org_id, file_id, step.upper()
-        )
-    except Exception:
-        pass
-    if not r:
-        return responses.Response(status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
-    return responses.Response(status_code=status.HTTP_200_OK)
-
-
 @app.put("/config_changed")
 def config_changed() -> responses.PlainTextResponse:
     config_handler.refresh_config()

diff --git a/controller/markdown_file_content.py b/controller/markdown_file_content.py
diff --git a/submodules/model b/submodules/model
diff --git a/submodules/s3 b/submodules/s3
+28 −0		business_objects/monitor.py
+5 −30		cognition_objects/environment_variable.py
+146 −0		cognition_objects/file_extraction.py
+91 −0		cognition_objects/file_reference.py
+172 −0		cognition_objects/file_transformation.py
+9 −59		cognition_objects/markdown_file.py
+5 −2		cognition_objects/project.py
+8 −1		cognition_objects/strategy.py
+29 −1		enums.py
+120 −4		models.py
+9 −0		connections/aws.py
+9 −0		connections/minio.py
+25 −0		controller.py