diff --git a/deno.lock b/deno.lock index 1da1375..fddd814 100644 --- a/deno.lock +++ b/deno.lock @@ -1,5 +1,50 @@ { - "version": "2", + "version": "3", + "packages": { + "specifiers": { + "npm:ajv-formats@2.1.0": "npm:ajv-formats@2.1.0_ajv@8.12.0", + "npm:ajv@8.12.0": "npm:ajv@8.12.0" + }, + "npm": { + "ajv-formats@2.1.0_ajv@8.12.0": { + "integrity": "sha512-USH2jBb+C/hIpwD2iRjp0pe0k+MvzG0mlSn/FIdCgQhUb9ALPRjt2KIQdfZDS9r0ZIeUAg7gOu9KL0PFqGqr5Q==", + "dependencies": { + "ajv": "ajv@8.12.0" + } + }, + "ajv@8.12.0": { + "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", + "dependencies": { + "fast-deep-equal": "fast-deep-equal@3.1.3", + "json-schema-traverse": "json-schema-traverse@1.0.0", + "require-from-string": "require-from-string@2.0.2", + "uri-js": "uri-js@4.4.1" + } + }, + "fast-deep-equal@3.1.3": { + "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", + "dependencies": {} + }, + "json-schema-traverse@1.0.0": { + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dependencies": {} + }, + "punycode@2.3.0": { + "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", + "dependencies": {} + }, + "require-from-string@2.0.2": { + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", + "dependencies": {} + }, + "uri-js@4.4.1": { + "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", + "dependencies": { + "punycode": "punycode@2.3.0" + } + } + } + }, "remote": { "https://cdn.esm.sh/v77/ajv-formats@2.1.0/deno/ajv-formats.js": "bfc5b929af623c70d65dc6f3e1e1301283a9183b488bc3852175e457bcb9ce2e", "https://cdn.esm.sh/v77/ajv-formats@2.1.0/dist/formats.d.ts": "c4592c3bec339a4cf9a4edd7c24d362798c31cc41558570c1e6f91a99aadc8ee", @@ -244,6 +289,7 @@ "https://deno.land/std@0.170.0/bytes/bytes_list.ts": "aba5e2369e77d426b10af1de0dcc4531acecec27f9b9056f4f7bfbf8ac147ab4", "https://deno.land/std@0.170.0/bytes/concat.ts": "97a1274e117510ffffc9499c4debb9541e408732bab2e0ca624869ae13103c10", "https://deno.land/std@0.170.0/bytes/copy.ts": "d14a58f188a997ee0d2ba696d0c82a42f4fb4b6705e90a4238b77d7644dae24c", + "https://deno.land/std@0.170.0/collections/chunk.ts": "10a147f3252406a011e0fa96a81a2e7e72cc79f9d15301e6de32424863a15f64", "https://deno.land/std@0.170.0/collections/distinct.ts": "278a5b36d7cbb2f8a25e09039064ac7894656e03d25860c58ac1735353fc2898", "https://deno.land/std@0.170.0/collections/group_by.ts": "ce8057c75d640491c0c81e0a6ce5524a8c0af00977a445ca5dbde24e4371d109", "https://deno.land/std@0.170.0/encoding/_yaml/dumper/dumper.ts": "5bd334372608a1aec7a2343705930889d4048f57a2c4d398f1d6d75996ecd0d3", @@ -566,50 +612,5 @@ "https://raw.githubusercontent.com/zongwei007/cli-format-deno/v3.x/src/format-config.ts": "ea3aba589931cb93c1e191747061fd1eecad06fb8a80c173b1e306c66a7ef2bf", "https://raw.githubusercontent.com/zongwei007/cli-format-deno/v3.x/src/format.ts": "bd3bd5bfc026d86f05967ba1a7edf12c9ddb05acbd03a7cdf92e794483646942", "https://raw.githubusercontent.com/zongwei007/cli-format-deno/v3.x/src/mod.ts": "c600d0e52aa06e41beade120a11971bc0a64e51edbbac12a7b1699a67d801c9c" - }, - "npm": { - "specifiers": { - "ajv-formats@2.1.0": "ajv-formats@2.1.0_ajv@8.12.0", - "ajv@8.12.0": "ajv@8.12.0" - }, - "packages": { - "ajv-formats@2.1.0_ajv@8.12.0": { - "integrity": "sha512-USH2jBb+C/hIpwD2iRjp0pe0k+MvzG0mlSn/FIdCgQhUb9ALPRjt2KIQdfZDS9r0ZIeUAg7gOu9KL0PFqGqr5Q==", - "dependencies": { - "ajv": "ajv@8.12.0" - } - }, - "ajv@8.12.0": { - "integrity": "sha512-sRu1kpcO9yLtYxBKvqfTeh9KzZEwO3STyX1HT+4CaDzC6HpTGYhIhPIzj9XuKU7KYDwnaeh5hcOwjy1QuJzBPA==", - "dependencies": { - "fast-deep-equal": "fast-deep-equal@3.1.3", - "json-schema-traverse": "json-schema-traverse@1.0.0", - "require-from-string": "require-from-string@2.0.2", - "uri-js": "uri-js@4.4.1" - } - }, - "fast-deep-equal@3.1.3": { - "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dependencies": {} - }, - "json-schema-traverse@1.0.0": { - "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dependencies": {} - }, - "punycode@2.3.0": { - "integrity": "sha512-rRV+zQD8tVFys26lAGR9WUuS4iUAngJScM+ZRSKtvl5tKeZ2t5bvdNFdNHBW9FWR4guGHlgmsZ1G7BSm2wTbuA==", - "dependencies": {} - }, - "require-from-string@2.0.2": { - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dependencies": {} - }, - "uri-js@4.4.1": { - "integrity": "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==", - "dependencies": { - "punycode": "punycode@2.3.0" - } - } - } } } diff --git a/src/api/terragrunt/TerragruntCliFacade.ts b/src/api/terragrunt/TerragruntCliFacade.ts index 0819c1c..366a850 100644 --- a/src/api/terragrunt/TerragruntCliFacade.ts +++ b/src/api/terragrunt/TerragruntCliFacade.ts @@ -99,6 +99,16 @@ export class TerragruntCliFacade { }); } + async moduleGroups(cwd: string): Promise> { + const cmds = ["terragrunt", "output-module-groups"]; + + const result = await this.quietRunner.run(cmds, { + cwd, + }); + + return JSON.parse(result.stdout); + } + async collectOutput(cwd: string, outputName: string) { const cmds = [ "terragrunt", @@ -112,4 +122,19 @@ export class TerragruntCliFacade { cwd, }); } + + async collectOutputs(cwd: string, outputName: string) { + const cmds = [ + "terragrunt", + "run-all", + "output", + "-raw", + outputName, + "--terragrunt-non-interactive", // disable terragrunt's own prompts, e.g. at the start of a terragrunt run-all run + ]; + + return await this.quietRunner.run(cmds, { + cwd, + }); + } } diff --git a/src/cli/Logger.ts b/src/cli/Logger.ts index b77455f..9a25104 100644 --- a/src/cli/Logger.ts +++ b/src/cli/Logger.ts @@ -63,8 +63,10 @@ export class Logger { console.error(colors.red(message)); } - public tip(msg: string) { - printTip(msg); + public tip(msg: string | ((fmt: FormatUtils) => string)) { + const message = typeof msg === "string" ? msg : msg(this.fmtUtils); + + printTip(message); } public tipCommand(msg: string, command: string) { diff --git a/src/docs/PlatformDocumentationGenerator.ts b/src/docs/PlatformDocumentationGenerator.ts index 4744018..0643317 100644 --- a/src/docs/PlatformDocumentationGenerator.ts +++ b/src/docs/PlatformDocumentationGenerator.ts @@ -15,6 +15,10 @@ import { PlatformConfig } from "../model/PlatformConfig.ts"; import { DocumentationRepository } from "./DocumentationRepository.ts"; import { MarkdownUtils } from "../model/MarkdownUtils.ts"; import { ComplianceControlRepository } from "../compliance/ComplianceControlRepository.ts"; +import { + RunAllPlatformModuleOutputCollector, + RunIndividualPlatformModuleOutputCollector, +} from "./PlatformModuleOutputCollector.ts"; export class PlatformDocumentationGenerator { constructor( @@ -67,30 +71,89 @@ export class PlatformDocumentationGenerator { dependencies: PlatformDependencies, docsRepo: DocumentationRepository, ) { + const platformPath = this.foundation.resolvePlatformPath( + dependencies.platform, + ); const platformProgress = new ProgressReporter( "generate documentation", - this.repo.relativePath( - this.foundation.resolvePlatformPath(dependencies.platform), - ), + this.repo.relativePath(platformPath), this.logger, ); - const tasks = dependencies.modules.map( - async (x) => - await this.generatePlatformModuleDocumentation( - x, - docsRepo, - dependencies.platform, - ), - ); + const platformModuleDocumentation = await this + .buildPlatformModuleOutputCollector(dependencies.platform); - await Promise.all(tasks); + // as a fallback process modules serially, unfortunately this is the only "safe" way to collect output + // see https://github.com/meshcloud/collie-cli/issues/265 + for (const dep of dependencies.modules) { + const documentationMd = await platformModuleDocumentation.getOutput(dep); + + await this.generatePlatformModuleDocumentation( + dep, + documentationMd, + docsRepo, + dependencies.platform, + ); + } platformProgress.done(); } + private async buildPlatformModuleOutputCollector(platform: PlatformConfig) { + const platformHclPath = this.foundation.resolvePlatformPath( + platform, + "platform.hcl", + ); + + const platformHcl = await Deno.readTextFile(platformHclPath); + + const fastModeIdentifier = + "--- BEGIN COLLIE PLATFORM MODULE OUTPUT: ${path_relative_to_include()} ---"; + + this.logger.verbose( + (fmt) => + `detecting if fast output collection is supported in ${ + fmt.kitPath( + platformHclPath, + ) + } by looking for a before_hook emitting "${fastModeIdentifier}"`, + ); + const enableFastMode = platformHcl.includes(fastModeIdentifier); + this.logger.verbose( + (_) => "fast output collection is supported: " + enableFastMode, + ); + + if (enableFastMode) { + const platformPath = this.foundation.resolvePlatformPath(platform); + const collector = new RunAllPlatformModuleOutputCollector( + this.terragrunt, + this.logger, + ); + + await collector.initialize(platformPath); + + return collector; + } else { + this.logger.tip( + (f) => + `Enable fast output collection for collie in ${ + f.kitPath( + platformHclPath, + ) + }`, + ); + + return new RunIndividualPlatformModuleOutputCollector( + this.repo, + this.terragrunt, + this.logger, + ); + } + } + private async generatePlatformModuleDocumentation( dep: KitModuleDependency, + documentationMd: string, docsRepo: DocumentationRepository, platform: PlatformConfig, ) { @@ -99,51 +162,34 @@ export class PlatformDocumentationGenerator { dep.kitModuleId, ); - const result = await this.terragrunt.collectOutput( - this.repo.resolvePath(path.dirname(dep.sourcePath)), - "documentation_md", - ); + await fs.ensureDir(path.dirname(destPath)); // todo: should we do nesting in the docs output or "flatten" module prefixes? - if (!result.status.success) { - this.logger.warn( - (fmt) => - `Failed to collect output "documentation_md" from platform module${ - fmt.kitPath( - dep.sourcePath, - ) - }`, - ); - this.logger.warn(result.stderr); - } else { - await fs.ensureDir(path.dirname(destPath)); // todo: should we do nesting in the docs output or "flatten" module prefixes? + const mdSections = [documentationMd]; - const mdSections = [result.stdout]; - - const complianceStatementsBlock = this.generateComplianceStatementSection( - dep, - docsRepo, - destPath, - ); - mdSections.push(complianceStatementsBlock); + const complianceStatementsBlock = this.generateComplianceStatementSection( + dep, + docsRepo, + destPath, + ); + mdSections.push(complianceStatementsBlock); - const kitModuleSection = this.generateKitModuleSection( - dep, - docsRepo, - destPath, - ); - mdSections.push(kitModuleSection); + const kitModuleSection = this.generateKitModuleSection( + dep, + docsRepo, + destPath, + ); + mdSections.push(kitModuleSection); - await Deno.writeTextFile(destPath, mdSections.join("\n\n")); + await Deno.writeTextFile(destPath, mdSections.join("\n\n")); - this.logger.verbose( - (fmt) => - `Wrote output "documentation_md" from platform module ${ - fmt.kitPath( - dep.sourcePath, - ) - } to ${fmt.kitPath(destPath)}`, - ); - } + this.logger.verbose( + (fmt) => + `Wrote output "documentation_md" from platform module ${ + fmt.kitPath( + dep.sourcePath, + ) + } to ${fmt.kitPath(destPath)}`, + ); } private generateKitModuleSection( diff --git a/src/docs/PlatformModuleOutputCollector.test.ts b/src/docs/PlatformModuleOutputCollector.test.ts new file mode 100644 index 0000000..3019ac7 --- /dev/null +++ b/src/docs/PlatformModuleOutputCollector.test.ts @@ -0,0 +1,21 @@ +import { assertEquals } from "std/testing/assert"; +import { RunAllPlatformModuleOutputCollector } from "./PlatformModuleOutputCollector.ts"; + +const stdout = ` +--- BEGIN COLLIE PLATFORM MODULE OUTPUT: logging --- +foo +123 +--- BEGIN COLLIE PLATFORM MODULE OUTPUT: billing --- +bar +xyz`; + +Deno.test("can split", () => { + const result = RunAllPlatformModuleOutputCollector.parseTerragrunt(stdout); + + const expected = [ + { module: "logging", output: "foo\n123" }, + { module: "billing", output: "bar\nxyz" }, + ]; + + assertEquals(result, expected); +}); diff --git a/src/docs/PlatformModuleOutputCollector.ts b/src/docs/PlatformModuleOutputCollector.ts new file mode 100644 index 0000000..94396b0 --- /dev/null +++ b/src/docs/PlatformModuleOutputCollector.ts @@ -0,0 +1,117 @@ +import * as path from "std/path"; +import { chunk } from "std/collections/chunk"; +import { TerragruntCliFacade } from "../api/terragrunt/TerragruntCliFacade.ts"; +import { KitModuleDependency } from "../kit/KitDependencyAnalyzer.ts"; +import { CollieRepository } from "../model/CollieRepository.ts"; +import { Logger } from "../cli/Logger.ts"; +import { MeshError } from "../errors.ts"; + +export interface PlatformModuleOutputCollector { + getOutput(dep: KitModuleDependency): Promise; +} + +interface PlatformModuleOutput { + module: string; + output: string; +} + +export class RunAllPlatformModuleOutputCollector + implements PlatformModuleOutputCollector { + private modules: PlatformModuleOutput[] = []; + + constructor( + private readonly terragrunt: TerragruntCliFacade, + private readonly logger: Logger, + ) {} + + async initialize(platformPath: string) { + const result = await this.terragrunt.collectOutputs( + platformPath, + "documentation_md", + ); + + if (!result.status.success) { + this.logger.error( + (fmt) => + `Failed to collect output "documentation_md" from platform ${ + fmt.kitPath( + platformPath, + ) + }`, + ); + this.logger.error(result.stderr); + + throw new MeshError( + "Failed to collect documentation output from platform modules", + ); + } + + this.modules = RunAllPlatformModuleOutputCollector.parseTerragrunt( + result.stdout, + ); + } + + // deno-lint-ignore require-await + async getOutput(dep: KitModuleDependency): Promise { + const platformModulePath = path.dirname(dep.sourcePath); + const module = this.modules.find((x) => + platformModulePath.endsWith(x.module) + ); + + if (!module) { + throw new MeshError( + "Failed to find documentation output from platform module " + + platformModulePath, + ); + } + return module?.output; + } + + public static parseTerragrunt(stdout: string): PlatformModuleOutput[] { + const regex = /^--- BEGIN COLLIE PLATFORM MODULE OUTPUT: (.+) ---$/gm; + + const sections = stdout.split(regex); + const cleanedSections = sections + .map((x) => x.trim()) + .filter((x) => x !== ""); + + return chunk(cleanedSections, 2).map(([module, output]) => ({ + module, + output, + })); + } +} + +export class RunIndividualPlatformModuleOutputCollector + implements PlatformModuleOutputCollector { + constructor( + private readonly repo: CollieRepository, + private readonly terragrunt: TerragruntCliFacade, + private readonly logger: Logger, + ) {} + + async getOutput(dep: KitModuleDependency): Promise { + const result = await this.terragrunt.collectOutput( + this.repo.resolvePath(path.dirname(dep.sourcePath)), + "documentation_md", + ); + + if (!result.status.success) { + this.logger.error( + (fmt) => + `Failed to collect output "documentation_md" from platform module ${ + fmt.kitPath( + dep.sourcePath, + ) + }`, + ); + this.logger.error(result.stderr); + + throw new MeshError( + "Failed to collect documentation output from platform modules", + ); + } + + return result.stdout; + } +} diff --git a/src/import_map.json b/src/import_map.json index 3a3d323..51b5f06 100644 --- a/src/import_map.json +++ b/src/import_map.json @@ -8,6 +8,7 @@ "std/archive/tar": "https://deno.land/std@0.170.0/archive/tar.ts", "std/archive/untar": "https://deno.land/std@0.170.0/archive/untar.ts", "std/streams/conversion": "https://deno.land/std@0.170.0/streams/conversion.ts", + "std/collections/chunk": "https://deno.land/std@0.170.0/collections/chunk.ts", "std/collections/groupBy": "https://deno.land/std@0.170.0/collections/group_by.ts", "std/collections/distinct": "https://deno.land/std@0.170.0/collections/distinct.ts", "std/fmt/colors": "https://deno.land/std@0.170.0/fmt/colors.ts",