feat: perf improvement through parallelisation of classes

unoplat-code-confluence/unoplat_code_confluence/__main__.py

@@ -1,4 +1,5 @@
 import argparse
+import asyncio
 import os
 from loguru import logger
 import datetime
@@ -21,7 +22,7 @@
 from packaging import version
 
 
-def start_pipeline():
+async def start_pipeline():
     parser = argparse.ArgumentParser(description="Codebase Parser CLI")
     parser.add_argument("--config", help="Path to configuration file for unoplat utility", default=os.getcwd() + '/default_config.json', type=str)
     args = parser.parse_args()
@@ -37,31 +38,31 @@ def start_pipeline():
     # logger.configure(handlers=logging_config["handlers"])
 
 
-    get_codebase_metadata(json_configuration_data,iload_json,iparse_json,isummariser)
+    await get_codebase_metadata(json_configuration_data,iload_json,iparse_json,isummariser)
 
 
-def get_codebase_metadata(json_configuration_data,iload_json,iparse_json,isummariser):
+async def get_codebase_metadata(json_configuration_data,iload_json,iparse_json,isummariser):
     # Collect necessary inputs from the user to set up the codebase indexing
     app_config = AppConfig(**json_configuration_data)
     logger.configure(handlers=app_config.handlers)
 
 
     # Button to submit the indexing
-    start_parsing(
+    await start_parsing(
         app_config,
         iload_json,
         iparse_json,
         isummariser
     )
 
 
-def ensure_jar_downloaded(github_token, arcguard_cli_repo, local_download_directory):
+async def ensure_jar_downloaded(github_token, arcguard_cli_repo, local_download_directory):
 
     jar_path = Downloader.download_latest_jar(arcguard_cli_repo, local_download_directory, github_token)
 
     return jar_path
 
-def get_extension(programming_language: str):
+async def get_extension(programming_language: str):
     #TODO: convert this to enum based check
     if programming_language == "java":
         return "java"
@@ -70,21 +71,21 @@ def get_extension(programming_language: str):
     else:
         raise ValueError(f"Unsupported programming language: {programming_language}")
 
-def start_parsing(app_config: AppConfig, iload_json: JsonLoader, iparse_json: JsonParser, isummariser: MarkdownSummariser):
+async def start_parsing(app_config: AppConfig, iload_json: JsonLoader, iparse_json: JsonParser, isummariser: MarkdownSummariser):
 
     # Log the start of the parsing process
     logger.info("Starting parsing process...")
 
     # Ensure the JAR is downloaded or use the existing one
-    jar_path = ensure_jar_downloaded(app_config.api_tokens["github_token"],app_config.repo.download_url,app_config.repo.download_directory)
+    jar_path = await ensure_jar_downloaded(app_config.api_tokens["github_token"],app_config.repo.download_url,app_config.repo.download_directory)
 
     logger.info(f"Local Workspace URL: {app_config.local_workspace_path}")
     logger.info(f"Programming Language: {app_config.programming_language}")
     logger.info(f"Output Path: {app_config.output_path}")
     logger.info(f"Codebase Name: {app_config.codebase_name}")
 
     # based on programming_language convert to extension
-    extension = get_extension(app_config.programming_language)
+    extension = await get_extension(app_config.programming_language)
 
     # Initialize the ArchGuard handler with the collected parameters.
     archguard_handler = ArchGuardHandler(
@@ -120,7 +121,7 @@ def start_parsing(app_config: AppConfig, iload_json: JsonLoader, iparse_json: Js
 
     codebase_summary = CodebaseSummaryParser(unoplat_codebase,dspy_function_pipeline_summary, dspy_class_pipeline_summary,dspy_package_pipeline_summary,dspy_codebase_pipeline_summary,app_config)
 
-    unoplat_codebase_summary: DspyUnoplatCodebaseSummary = codebase_summary.parse_codebase()
+    unoplat_codebase_summary: DspyUnoplatCodebaseSummary = await codebase_summary.parse_codebase()
 
     # now write to a markdown dspy unoplat codebase summary
 
@@ -136,5 +137,5 @@ def start_parsing(app_config: AppConfig, iload_json: JsonLoader, iparse_json: Js
 
     warnings.filterwarnings("ignore", category=DeprecationWarning, module='pydantic.*')
 
-    start_pipeline()
+    asyncio.run(start_pipeline())
 

unoplat-code-confluence/unoplat_code_confluence/dspy_function_summary.py

@@ -37,7 +37,7 @@ def __init__(self):
         self.generate_function_objective = dspy.TypedPredictor(CodeConfluenceFunctionObjectiveSignature)
 
     def forward(self, function_metadata: DspyUnoplatFunctionSubset, class_metadata: DspyUnoplatNodeSubset):
-        logger.debug(f"Generating function summary for {function_metadata.name}")
+        logger.debug(f"Generating function summary for {function_metadata.name} present in class {class_metadata.node_name}")
 
         class_subset = str(class_metadata.model_dump_json())
         function_subset = str(function_metadata.model_dump_json()) 

unoplat-code-confluence/unoplat_code_confluence/loguru.json

@@ -2,7 +2,7 @@
     "handlers": [
         {
             "sink": "./app.log",
-            "format": "<green>{time:YYYY-MM-DD at HH:mm:ss}</green> | <level>{level}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
+            "format": "<green>{time:YYYY-MM-DD at HH:mm:ss}</green> | <level>{level}</level> | <cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> | <magenta>{thread.name}</magenta> - <level>{message}</level>",
             "rotation": "10 MB",
             "retention": "10 days",
             "level": "INFO"