Dspy codebase pydantic workflow (#28)

* feat: added function summary * feat: improvement in models * feat: improved models * feat: added dspy modules * latest dspy pipelines * fix: commit latest changes * fix: ready for release --------- Co-authored-by: jghiya <[email protected]>
unoplat · Jun 29, 2024 · 0c4cd39 · 0c4cd39
1 parent 2ea05e6
commit 0c4cd39
Show file tree

Hide file tree

Showing 25 changed files with 367 additions and 1,652 deletions.
diff --git a/.gitignore b/.gitignore
@@ -31,3 +31,5 @@ unoplat-code-confluence/nodeparser/tests/__pycache__
 unoplat-code-confluence/settings/__pycache__
 unoplat-code-confluence/utility/__pycache__
 unoplat-code-confluence/data_models/dspy/__pycache__
+unoplat-code-confluence/__pycache__
+unoplat-code-confluence/summary_parser/__pycache__
diff --git a/unoplat-code-confluence/__main__.py b/unoplat-code-confluence/__main__.py
@@ -8,39 +8,39 @@
 import re
 from data_models.chapi_unoplat_codebase import UnoplatCodebase
 from downloader.downloader import Downloader
+from dspy_class_summary import CodeConfluenceClassModule
+from dspy_codebase_summary import CodeConfluenceCodebaseModule
+from dspy_function_summary import CodeConfluenceFunctionModule
+from dspy_package_summary import CodeConfluencePackageModule
 from loader import iload_json, iparse_json
 from loader.json_loader import JsonLoader
 from loader.parse_json import JsonParser
 from nodeparser.nodesummariser import NodeSummariser
 from nodeparser.isummariser import ISummariser
 from settings.appsettings import AppSettings
+from summary_parser.codebase_summary import CodebaseSummaryParser
 
 
 def main(iload_json, iparse_json,isummariser,json_configuration_data):
-    #settings = AppSettings()
-    get_codebase_metadata(json_configuration_data,iload_json,iparse_json,isummariser)
+    settings = AppSettings()
+    get_codebase_metadata(json_configuration_data,settings,iload_json,iparse_json,isummariser)
 
 
-def handle_toggle(value):
-    global selected_language
-    selected_language = value
-    logger.info(f"Selected language: {value}")
-
-
-def get_codebase_metadata(json_configuration_data,iload_json,iparse_json,isummariser):
+def get_codebase_metadata(json_configuration_data,settings: AppSettings,iload_json,iparse_json,isummariser):
     # Collect necessary inputs from the user to set up the codebase indexing
     local_workspace_path = json_configuration_data["local_workspace_path"]
     programming_language = json_configuration_data["programming_language"]
     output_path_field = json_configuration_data["output_path"]
     codebase_name_field = json_configuration_data["codebase_name"]
-    github_token = json_configuration_data["repo"]["github_token"]
+    github_token = settings.github_token
     arcguard_cli_repo = json_configuration_data["repo"]["download_url"]
     local_download_directory = json_configuration_data["repo"]["download_directory"]
 
 
     # Button to submit the indexing
     start_parsing(
         local_workspace_path,
+        settings,
         # move this when expanding to new languages
         programming_language,
         output_path_field,
@@ -83,7 +83,7 @@ def ensure_jar_downloaded(github_token,arcguard_cli_repo,local_download_director
 
     return jar_path
 
-def start_parsing(local_workspace_path, programming_language, output_path, codebase_name, github_token, arcguard_cli_repo, local_download_directory, iload_json, iparse_json, isummariser):
+def start_parsing(local_workspace_path, settings, programming_language, output_path, codebase_name, github_token, arcguard_cli_repo, local_download_directory, iload_json, iparse_json, isummariser):
 
     # Log the start of the parsing process
     logger.info("Starting parsing process...")
@@ -116,7 +116,17 @@ def start_parsing(local_workspace_path, programming_language, output_path, codeb
 
     unoplat_codebase : UnoplatCodebase = iparse_json.parse_json_to_nodes(chapi_metadata, isummariser)
 
-    print(unoplat_codebase.model_dump())
+    dspy_function_pipeline_summary : CodeConfluenceFunctionModule = CodeConfluenceFunctionModule()
+
+    dspy_class_pipeline_summary : CodeConfluenceClassModule = CodeConfluenceClassModule()
+
+    dspy_package_pipeline_summary : CodeConfluencePackageModule = CodeConfluencePackageModule()
+
+    dspy_codebase_pipeline_summary: CodeConfluenceCodebaseModule = CodeConfluenceCodebaseModule()
+
+    codebase_summary = CodebaseSummaryParser(unoplat_codebase,dspy_function_pipeline_summary, dspy_class_pipeline_summary,dspy_package_pipeline_summary,dspy_codebase_pipeline_summary,settings)
+
+    codebase_summary.parse_codebase()
 
     # with open(os.path.join(output_path, output_filename), 'a+') as md_file:
     #     for node in iparse_json.parse_json_to_nodes(chapi_metadata, isummariser):

diff --git a/unoplat-code-confluence/codebase_overview_spec.md b/unoplat-code-confluence/codebase_overview_spec.md
diff --git a/unoplat-code-confluence/codebase_summary.json b/unoplat-code-confluence/codebase_summary.json