Additional logging

tableau · Dec 20, 2023 · fe7870d · fe7870d
1 parent 1991e4f
commit fe7870d
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 0 deletions.
diff --git a/altimeter/core/artifact_io/writer.py b/altimeter/core/artifact_io/writer.py
@@ -122,7 +122,9 @@ def write_graph_set(
             artifact_path = os.path.join(self.output_dir, f"{name}.rdf.gz")
         else:
             raise ValueError(f"Unknown compression arg {compression}")
+        logger.info(event=LogEvent.GraphSetToRDFStart)
         graph = graph_set.to_rdf()
+        logger.info(event=LogEvent.GraphSetToRDFEnd)
         with logger.bind(artifact_path=artifact_path):
             logger.info(event=LogEvent.WriteToFSStart)
             with open(artifact_path, "wb") as fp:

diff --git a/altimeter/core/log_events.py b/altimeter/core/log_events.py
@@ -11,9 +11,15 @@ class LogEvent(BaseLogEvent):
     AuthToAccountEnd: EventName
     AuthToAccountFailure: EventName
 
+    CompileGraphsStart: EventName
+    CompileGraphsEnd: EventName
+
     GraphLoadedSNSNotificationStart: EventName
     GraphLoadedSNSNotificationEnd: EventName
 
+    GraphSetToRDFStart: EventName
+    GraphSetToRDFEnd: EventName
+
     MetadataGraphUpdateStart: EventName
     MetadataGraphUpdateEnd: EventName
 
@@ -48,6 +54,9 @@ class LogEvent(BaseLogEvent):
     ScanResourceTypeStart: EventName
     ScanResourceTypeEnd: EventName
 
+    ValidateGraphStart: EventName
+    ValidateGraphEnd: EventName
+
     WriteToFSStart: EventName
     WriteToFSEnd: EventName
 

diff --git a/bin/sfn_compile_graphs.py b/bin/sfn_compile_graphs.py
@@ -12,6 +12,8 @@
 from altimeter.core.base_model import BaseImmutableModel
 from altimeter.core.config import AWSConfig
 from altimeter.core.graph.graph_set import GraphSet, ValidatedGraphSet
+from altimeter.core.log import Logger
+from altimeter.core.log_events import LogEvent
 
 
 class CompileGraphsInput(BaseImmutableModel):
@@ -31,6 +33,7 @@ def lambda_handler(event: Dict[str, Any], _: Any) -> Dict[str, Any]:
     if root.handlers:
         for handler in root.handlers:
             root.removeHandler(handler)
+    logger = Logger()
     compile_graphs_input = CompileGraphsInput(**event)
 
     scanned_accounts: List[str] = []
@@ -45,6 +48,7 @@ def lambda_handler(event: Dict[str, Any], _: Any) -> Dict[str, Any]:
         scan_id=compile_graphs_input.scan_id,
     )
 
+    logger.info(event=LogEvent.CompileGraphsStart)
     for account_scan_manifest in compile_graphs_input.account_scan_manifests:
         account_id = account_scan_manifest.account_id
         if account_scan_manifest.errors:
@@ -60,7 +64,12 @@ def lambda_handler(event: Dict[str, Any], _: Any) -> Dict[str, Any]:
             unscanned_accounts.add(account_id)
     if not graph_sets:
         raise Exception("BUG: No graph_sets generated.")
+    logger.info(event=LogEvent.CompileGraphsEnd)
+
+    logger.info(event=LogEvent.ValidateGraphStart)
     validated_graph_set = ValidatedGraphSet.from_graph_set(GraphSet.from_graph_sets(graph_sets))
+    logger.info(event=LogEvent.ValidateGraphEnd)
+
     if compile_graphs_input.high_mem:
         master_artifact_path = artifact_writer.write_json(name="master", data=validated_graph_set)
         start_time = validated_graph_set.start_time