feature/debug documentation update errors

src/api/terragrunt/TerragruntCliFacade.ts

@@ -99,6 +99,16 @@ export class TerragruntCliFacade {
     });
   }
 
+  async moduleGroups(cwd: string): Promise<Record<string, string[]>> {
+    const cmds = ["terragrunt", "output-module-groups"];
+
+    const result = await this.quietRunner.run(cmds, {
+      cwd,
+    });
+
+    return JSON.parse(result.stdout);
+  }
+
   async collectOutput(cwd: string, outputName: string) {
     const cmds = [
       "terragrunt",
@@ -112,4 +122,19 @@ export class TerragruntCliFacade {
       cwd,
     });
   }
+
+  async collectOutputs(cwd: string, outputName: string) {
+    const cmds = [
+      "terragrunt",
+      "run-all",
+      "output",
+      "-raw",
+      outputName,
+      "--terragrunt-non-interactive", // disable terragrunt's own prompts, e.g. at the start of a terragrunt run-all run
+    ];
+
+    return await this.quietRunner.run(cmds, {
+      cwd,
+    });
+  }
 }

src/cli/Logger.ts

@@ -63,8 +63,10 @@ export class Logger {
     console.error(colors.red(message));
   }
 
-  public tip(msg: string) {
-    printTip(msg);
+  public tip(msg: string | ((fmt: FormatUtils) => string)) {
+    const message = typeof msg === "string" ? msg : msg(this.fmtUtils);
+
+    printTip(message);
   }
 
   public tipCommand(msg: string, command: string) {

src/docs/PlatformDocumentationGenerator.ts

@@ -15,6 +15,10 @@ import { PlatformConfig } from "../model/PlatformConfig.ts";
 import { DocumentationRepository } from "./DocumentationRepository.ts";
 import { MarkdownUtils } from "../model/MarkdownUtils.ts";
 import { ComplianceControlRepository } from "../compliance/ComplianceControlRepository.ts";
+import {
+  RunAllPlatformModuleOutputCollector,
+  RunIndividualPlatformModuleOutputCollector,
+} from "./PlatformModuleOutputCollector.ts";
 
 export class PlatformDocumentationGenerator {
   constructor(
@@ -67,30 +71,89 @@ export class PlatformDocumentationGenerator {
     dependencies: PlatformDependencies,
     docsRepo: DocumentationRepository,
   ) {
+    const platformPath = this.foundation.resolvePlatformPath(
+      dependencies.platform,
+    );
     const platformProgress = new ProgressReporter(
       "generate documentation",
-      this.repo.relativePath(
-        this.foundation.resolvePlatformPath(dependencies.platform),
-      ),
+      this.repo.relativePath(platformPath),
       this.logger,
     );
 
-    const tasks = dependencies.modules.map(
-      async (x) =>
-        await this.generatePlatformModuleDocumentation(
-          x,
-          docsRepo,
-          dependencies.platform,
-        ),
-    );
+    const platformModuleDocumentation = await this
+      .buildPlatformModuleOutputCollector(dependencies.platform);
 
-    await Promise.all(tasks);
+    // as a fallback process modules serially, unfortunately this is the only "safe" way to collect output
+    // see https://github.com/meshcloud/collie-cli/issues/265
+    for (const dep of dependencies.modules) {
+      const documentationMd = await platformModuleDocumentation.getOutput(dep);
+
+      await this.generatePlatformModuleDocumentation(
+        dep,
+        documentationMd,
+        docsRepo,
+        dependencies.platform,
+      );
+    }
 
     platformProgress.done();
   }
 
+  private async buildPlatformModuleOutputCollector(platform: PlatformConfig) {
+    const platformHclPath = this.foundation.resolvePlatformPath(
+      platform,
+      "platform.hcl",
+    );
+
+    const platformHcl = await Deno.readTextFile(platformHclPath);
+
+    const fastModeIdentifier =
+      "--- BEGIN COLLIE PLATFORM MODULE OUTPUT: ${path_relative_to_include()} ---";
+
+    this.logger.verbose(
+      (fmt) =>
+        `detecting if fast output collection is supported in ${
+          fmt.kitPath(
+            platformHclPath,
+          )
+        } by looking for a before_hook emitting "${fastModeIdentifier}"`,
+    );
+    const enableFastMode = platformHcl.includes(fastModeIdentifier);
+    this.logger.verbose(
+      (_) => "fast output collection is supported: " + enableFastMode,
+    );
+
+    if (enableFastMode) {
+      const platformPath = this.foundation.resolvePlatformPath(platform);
+      const collector = new RunAllPlatformModuleOutputCollector(
+        this.terragrunt,
+        this.logger,
+      );
+
+      await collector.initialize(platformPath);
+
+      return collector;
+    } else {
+      this.logger.tip(
+        (f) =>
+          `Enable fast output collection for collie in ${
+            f.kitPath(
+              platformHclPath,
+            )
+          }`,
+      );
+
+      return new RunIndividualPlatformModuleOutputCollector(
+        this.repo,
+        this.terragrunt,
+        this.logger,
+      );
+    }
+  }
+
   private async generatePlatformModuleDocumentation(
     dep: KitModuleDependency,
+    documentationMd: string,
     docsRepo: DocumentationRepository,
     platform: PlatformConfig,
   ) {
@@ -99,51 +162,34 @@ export class PlatformDocumentationGenerator {
       dep.kitModuleId,
     );
 
-    const result = await this.terragrunt.collectOutput(
-      this.repo.resolvePath(path.dirname(dep.sourcePath)),
-      "documentation_md",
-    );
+    await fs.ensureDir(path.dirname(destPath)); // todo: should we do nesting in the docs output or "flatten" module prefixes?
 
-    if (!result.status.success) {
-      this.logger.warn(
-        (fmt) =>
-          `Failed to collect output "documentation_md" from platform module${
-            fmt.kitPath(
-              dep.sourcePath,
-            )
-          }`,
-      );
-      this.logger.warn(result.stderr);
-    } else {
-      await fs.ensureDir(path.dirname(destPath)); // todo: should we do nesting in the docs output or "flatten" module prefixes?
+    const mdSections = [documentationMd];
 
-      const mdSections = [result.stdout];
-
-      const complianceStatementsBlock = this.generateComplianceStatementSection(
-        dep,
-        docsRepo,
-        destPath,
-      );
-      mdSections.push(complianceStatementsBlock);
+    const complianceStatementsBlock = this.generateComplianceStatementSection(
+      dep,
+      docsRepo,
+      destPath,
+    );
+    mdSections.push(complianceStatementsBlock);
 
-      const kitModuleSection = this.generateKitModuleSection(
-        dep,
-        docsRepo,
-        destPath,
-      );
-      mdSections.push(kitModuleSection);
+    const kitModuleSection = this.generateKitModuleSection(
+      dep,
+      docsRepo,
+      destPath,
+    );
+    mdSections.push(kitModuleSection);
 
-      await Deno.writeTextFile(destPath, mdSections.join("\n\n"));
+    await Deno.writeTextFile(destPath, mdSections.join("\n\n"));
 
-      this.logger.verbose(
-        (fmt) =>
-          `Wrote output "documentation_md" from platform module ${
-            fmt.kitPath(
-              dep.sourcePath,
-            )
-          } to ${fmt.kitPath(destPath)}`,
-      );
-    }
+    this.logger.verbose(
+      (fmt) =>
+        `Wrote output "documentation_md" from platform module ${
+          fmt.kitPath(
+            dep.sourcePath,
+          )
+        } to ${fmt.kitPath(destPath)}`,
+    );
   }
 
   private generateKitModuleSection(

src/docs/PlatformModuleOutputCollector.test.ts

@@ -0,0 +1,21 @@
+import { assertEquals } from "std/testing/assert";
+import { RunAllPlatformModuleOutputCollector } from "./PlatformModuleOutputCollector.ts";
+
+const stdout = `
+--- BEGIN COLLIE PLATFORM MODULE OUTPUT: logging ---
+foo
+123
+--- BEGIN COLLIE PLATFORM MODULE OUTPUT: billing ---
+bar
+xyz`;
+
+Deno.test("can split", () => {
+  const result = RunAllPlatformModuleOutputCollector.parseTerragrunt(stdout);
+
+  const expected = [
+    { module: "logging", output: "foo\n123" },
+    { module: "billing", output: "bar\nxyz" },
+  ];
+
+  assertEquals(result, expected);
+});