diff --git a/pkg/tfgen/docs.go b/pkg/tfgen/docs.go index d11cbcb63..357187e55 100644 --- a/pkg/tfgen/docs.go +++ b/pkg/tfgen/docs.go @@ -740,7 +740,6 @@ func (p *tfMarkdownParser) parseSection(h2Section []string) error { !p.info.ReplaceExamplesSection() { p.sink.warn("Unexpected code snippets in section '%v' for %v '%v'. The HCL code will be converted if possible, "+ "but may not display correctly in the generated docs.", header, p.kind, p.rawname) - unexpectedSnippets++ } // Now process the content based on the H2 topic. These are mostly standard across TF's docs. @@ -1318,7 +1317,6 @@ func (g *Generator) convertExamplesInner( stripSubsectionsWithErrors bool, convertHCL func(hcl, path, exampleTitle string, languages []string) (string, error), ) (result string) { - output := &bytes.Buffer{} writeTrailingNewline := func(buf *bytes.Buffer) { @@ -1555,7 +1553,6 @@ func (g *Generator) legacyConvert( // convertHCLToString hides the implementation details of the upstream implementation for HCL conversion and provides // simplified parameters and return values func (g *Generator) convertHCLToString(hclCode, path, languageName string) (string, error) { - fileName := fmt.Sprintf("/%s.tf", strings.ReplaceAll(path, "/", "-")) var convertedHcl string @@ -1601,7 +1598,6 @@ type languages []string func (s languages) Len() int { return len(s) } func (s languages) Swap(i, j int) { s[i], s[j] = s[j], s[i] } func (s languages) Less(i, j int) bool { - notFound := -1 indexOf := func(item string, data []string) int { @@ -1708,7 +1704,6 @@ func (g *Generator) convertHCL(hcl, path, exampleTitle string, languages []strin isCompleteFailure := len(failedLangs) == len(languages) if isCompleteFailure { - hclAllLangsConversionFailures++ if exampleTitle == "" { g.warn(fmt.Sprintf("unable to convert HCL example for Pulumi entity '%s': %v. The example will be dropped "+ "from any generated docs or SDKs.", path, err)) @@ -1725,18 +1720,6 @@ func (g *Generator) convertHCL(hcl, path, exampleTitle string, languages []strin for lang := range failedLangs { failedLangsStrings = append(failedLangsStrings, lang) - - switch lang { - case convert.LanguageTypescript: - hclTypeScriptPartialConversionFailures++ - case convert.LanguagePython: - hclPythonPartialConversionFailures++ - case convert.LanguageCSharp: - hclCSharpPartialConversionFailures++ - case convert.LanguageGo: - hclGoPartialConversionFailures++ - } - if exampleTitle == "" { g.warn(fmt.Sprintf("unable to convert HCL example for Pulumi entity '%s' in the following language(s): "+ "%s. Examples for these languages will be dropped from any generated docs or SDKs.", @@ -1818,7 +1801,6 @@ func cleanupDoc( g.debug("Cleaning up text for attribute [%v] in [%v]", k, name) cleanedText, elided := reformatText(infoCtx, v, footerLinks) if elided { - elidedAttributes++ g.warn("Found in docs for attribute [%v] in [%v]. The attribute's description will be dropped "+ "in the Pulumi provider.", k, name) elidedDoc = true @@ -1838,7 +1820,6 @@ func cleanupDoc( if examples == "" { g.debug("Unable to find any examples in the description text. The entire description will be discarded.") - elidedDescriptions++ g.warn("Found in description for [%v]. The description and any examples will be dropped in the "+ "Pulumi provider.", name) elidedDoc = true @@ -1847,12 +1828,10 @@ func cleanupDoc( cleanedupExamples, examplesElided := reformatText(infoCtx, examples, footerLinks) if examplesElided { - elidedDescriptions++ g.warn("Found in description for [%v]. The description and any examples will be dropped in "+ "the Pulumi provider.", name) elidedDoc = true } else { - elidedDescriptionsOnly++ g.warn("Found in description for [%v], but was able to preserve the examples. The description "+ "proper will be dropped in the Pulumi provider.", name) cleanupText = cleanedupExamples @@ -1971,7 +1950,6 @@ func extractExamples(description string) string { // reformatText processes markdown strings from TF docs and cleans them for inclusion in Pulumi docs func reformatText(g infoContext, text string, footerLinks map[string]string) (string, bool) { - cleanupText := func(text string) (string, bool) { // Remove incorrect documentation that should have been cleaned up in our forks. if strings.Contains(text, "Terraform") || strings.Contains(text, "terraform") { diff --git a/pkg/tfgen/examples_coverage_exporter.go b/pkg/tfgen/examples_coverage_exporter.go index 53755def6..de93291ad 100644 --- a/pkg/tfgen/examples_coverage_exporter.go +++ b/pkg/tfgen/examples_coverage_exporter.go @@ -440,36 +440,34 @@ func (ce *coverageExportUtil) exportMarkdown(outputDirectory string, fileName st return os.WriteFile(targetFile, []byte(out), 0600) } -// The fourth mode, which simply gives the provider name, and success percentage. -func (ce *coverageExportUtil) exportHumanReadable(outputDirectory string, fileName string) error { - - // The Coverage Tracker data structure is flattened to gather statistics about each language - type LanguageStatistic struct { - Total int - Successes int - } +// The Coverage Tracker data structure is flattened to gather statistics about each language +type languageStatistic struct { + Total int + Successes int +} - type ProviderStatistic struct { - Name string - Examples int - TotalConversions int - Successes int - } +type providerStatistic struct { + Name string + Examples int + TotalConversions int + Successes int +} +func (ce coverageExportUtil) produceStatistics() (map[string]*languageStatistic, providerStatistic) { // Main maps for holding the overall provider summary, and each language conversion statistic - var allLanguageStatistics = make(map[string]*LanguageStatistic) - var providerStatistic = ProviderStatistic{ce.Tracker.ProviderName, 0, 0, 0} + var allLanguageStatistics = make(map[string]*languageStatistic) + var providerStats = providerStatistic{ce.Tracker.ProviderName, 0, 0, 0} // All the conversion attempts for each example are iterated by language name and // their results are added to the main map for _, page := range ce.Tracker.EncounteredPages { for _, example := range page.Examples { - providerStatistic.Examples++ + providerStats.Examples++ for languageName, conversionResult := range example.ConversionResults { - providerStatistic.TotalConversions++ + providerStats.TotalConversions++ // Obtaining the current language we will be creating statistics for - var currentLanguage *LanguageStatistic + var currentLanguage *languageStatistic if val, ok := allLanguageStatistics[languageName]; ok { // Current language already exists in main map @@ -477,27 +475,28 @@ func (ce *coverageExportUtil) exportHumanReadable(outputDirectory string, fileNa } else { // The main map doesn't yet contain this language, and it needs to be added - allLanguageStatistics[languageName] = &LanguageStatistic{0, 0} + allLanguageStatistics[languageName] = &languageStatistic{0, 0} currentLanguage = allLanguageStatistics[languageName] } // The language's entry in the summarized results is updated and any currentLanguage.Total++ if conversionResult.FailureSeverity == Success { - providerStatistic.Successes++ + providerStats.Successes++ currentLanguage.Successes++ } } } } - targetFile, err := createEmptyFile(outputDirectory, fileName) - if err != nil { - return err - } + return allLanguageStatistics, providerStats +} + +func (ce *coverageExportUtil) produceHumanReadableSummary() string { + allLanguageStatistics, providerStatistic := ce.produceStatistics() // Forming a string which will eventually be written to the target file - fileString := fmt.Sprintf("Provider: %s\nSuccess rate: %.2f%% (%d/%d)\n\n", + fileString := fmt.Sprintf("\nProvider: %s\nSuccess rate: %.2f%% (%d/%d)\n\n", providerStatistic.Name, float64(providerStatistic.Successes)/float64(providerStatistic.TotalConversions)*100.0, providerStatistic.Successes, @@ -522,6 +521,17 @@ func (ce *coverageExportUtil) exportHumanReadable(outputDirectory string, fileNa ) } + return fileString +} + +// The fourth mode, which simply gives the provider name, and success percentage. +func (ce *coverageExportUtil) exportHumanReadable(outputDirectory string, fileName string) error { + targetFile, err := createEmptyFile(outputDirectory, fileName) + if err != nil { + return err + } + fileString := ce.produceHumanReadableSummary() + return os.WriteFile(targetFile, []byte(fileString), 0600) } diff --git a/pkg/tfgen/examples_coverage_tracker.go b/pkg/tfgen/examples_coverage_tracker.go index f1d535864..f07a34a28 100644 --- a/pkg/tfgen/examples_coverage_tracker.go +++ b/pkg/tfgen/examples_coverage_tracker.go @@ -236,3 +236,8 @@ func (ct *CoverageTracker) exportResults(outputDirectory string) error { coverageExportUtil := newCoverageExportUtil(ct) return coverageExportUtil.tryExport(outputDirectory) } + +func (ct *CoverageTracker) getShortResultSummary() string { + coverageExportUtil := newCoverageExportUtil(ct) + return coverageExportUtil.produceHumanReadableSummary() +} diff --git a/pkg/tfgen/generate.go b/pkg/tfgen/generate.go index e19ab93d7..d4487420c 100644 --- a/pkg/tfgen/generate.go +++ b/pkg/tfgen/generate.go @@ -96,7 +96,7 @@ const ( func (l Language) shouldConvertExamples() bool { switch l { - case Golang, NodeJS, Python, CSharp, Schema, PCL: + case Schema: return true } return false @@ -987,9 +987,6 @@ func (g *Generator) UnstableGenerateFromSchema(genSchemaResult *GenerateSchemaRe return errors.Wrapf(err, "failed to create project file") } - // Print out some documentation stats as a summary afterwards. - printDocStats() - // Close the plugin host. g.pluginHost.Close() diff --git a/pkg/tfgen/main.go b/pkg/tfgen/main.go index 618b1c7a3..f5584f1e6 100644 --- a/pkg/tfgen/main.go +++ b/pkg/tfgen/main.go @@ -142,10 +142,8 @@ func newTFGenCmd(pkg string, version string, prov tfbridge.ProviderInfo, // Creating an item to keep track of example coverage if the // COVERAGE_OUTPUT_DIR env is set var coverageTracker *CoverageTracker - coverageOutputDir, coverageTrackingEnabled := os.LookupEnv("COVERAGE_OUTPUT_DIR") - if coverageTrackingEnabled { - coverageTracker = newCoverageTracker(prov.Name, prov.Version) - } + coverageOutputDir, coverageTrackingOutputEnabled := os.LookupEnv("COVERAGE_OUTPUT_DIR") + coverageTracker = newCoverageTracker(prov.Name, prov.Version) opts := GeneratorOptions{ Package: pkg, @@ -160,10 +158,15 @@ func newTFGenCmd(pkg string, version string, prov tfbridge.ProviderInfo, } err := gen(opts) - - // Exporting collected coverage data to the directory specified by COVERAGE_OUTPUT_DIR - if coverageTrackingEnabled { - err = coverageTracker.exportResults(coverageOutputDir) + if opts.Language.shouldConvertExamples() { + // Exporting collected coverage data to the directory specified by COVERAGE_OUTPUT_DIR + if coverageTrackingOutputEnabled { + err = coverageTracker.exportResults(coverageOutputDir) + } else { + fmt.Println("\nAdditional example conversion stats are available by setting COVERAGE_OUTPUT_DIR.") + } + fmt.Println(coverageTracker.getShortResultSummary()) + printDocStats() } return err diff --git a/pkg/tfgen/metrics.go b/pkg/tfgen/metrics.go index 94694d196..eea5b350e 100644 --- a/pkg/tfgen/metrics.go +++ b/pkg/tfgen/metrics.go @@ -16,87 +16,60 @@ package tfgen import ( "fmt" + schemaTools "github.com/pulumi/schema-tools/pkg" ) var ( - ignoredDocHeaders = make(map[string]int) - elidedDescriptions int // i.e., we discard the entire description, including examples - elidedDescriptionsOnly int // we discarded the description proper, but were able to preserve the examples - elidedArguments int - elidedNestedArguments int - elidedAttributes int - - unexpectedSnippets int - hclAllLangsConversionFailures int // examples that failed to convert in any language - - // examples that failed to convert in one, but not all, languages. This is less severe impact because users will - // at least have code in another language to reference: - hclGoPartialConversionFailures int - hclPythonPartialConversionFailures int - hclTypeScriptPartialConversionFailures int - hclCSharpPartialConversionFailures int + ignoredDocHeaders = make(map[string]int) + elidedArguments int + elidedNestedArguments int + unexpectedSnippets int // Arguments metrics: totalArgumentsFromDocs int // See comment in getNestedDescriptionFromParsedDocs for why we track this behavior: argumentDescriptionsFromAttributes int - - // General metrics: - entitiesMissingDocs int + entitiesMissingDocs int schemaStats schemaTools.PulumiSchemaStats ) // printDocStats outputs metrics relating to document parsing and conversion func printDocStats() { - fmt.Println("") - fmt.Println("General metrics:") fmt.Printf("\t%d total resources containing %d total inputs.\n", schemaStats.Resources.TotalResources, schemaStats.Resources.TotalInputProperties) fmt.Printf("\t%d total functions.\n", schemaStats.Functions.TotalFunctions) - fmt.Printf("\t%d entities are missing docs entirely because they could not be found in the upstream provider.\n", - entitiesMissingDocs) - fmt.Println("") - - fmt.Println("Description metrics:") - fmt.Printf("\t%d entity descriptions contained an reference and were dropped, including examples.\n", - elidedDescriptions) - fmt.Printf("\t%d entity descriptions contained an reference and were dropped, but examples were preserved.\n", - elidedDescriptionsOnly) - fmt.Println("") - - fmt.Println("Example conversion metrics:") - fmt.Printf("\t%d HCL examples failed to convert in all languages\n", hclAllLangsConversionFailures) - fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to TypeScript\n", - hclTypeScriptPartialConversionFailures) - fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to Python\n", - hclPythonPartialConversionFailures) - fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to Go\n", - hclGoPartialConversionFailures) - fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to C#\n", - hclCSharpPartialConversionFailures) - fmt.Printf("\t%d entity document sections contained unexpected HCL code snippets. Examples will be converted, "+ - "but may not display correctly in the registry, e.g. lacking tabs.\n", unexpectedSnippets) + if entitiesMissingDocs > 0 { + fmt.Printf("\t%d entities are missing docs entirely because they could not be found in the upstream provider.\n", + entitiesMissingDocs) + } + if unexpectedSnippets > 0 { + fmt.Printf("\t%d entity document sections contained unexpected HCL code snippets. Examples will be converted, "+ + "but may not display correctly in the registry, e.g. lacking tabs.\n", unexpectedSnippets) + } fmt.Println("") fmt.Println("Argument metrics:") fmt.Printf("\t%d argument descriptions were parsed from the upstream docs\n", totalArgumentsFromDocs) fmt.Printf("\t%d top-level input property descriptions came from an upstream attribute (as opposed to an argument). "+ "Nested arguments are not included in this count.\n", argumentDescriptionsFromAttributes) - fmt.Printf("\t%d arguments contained an reference and had their descriptions dropped.\n", - elidedArguments) - fmt.Printf("\t%d nested arguments contained an reference and had their descriptions dropped.\n", - elidedNestedArguments) - //nolint:lll - fmt.Printf("\t%d of %d resource inputs (%.2f%%) are missing descriptions in the schema\n", - schemaStats.Resources.InputPropertiesMissingDescriptions, schemaStats.Resources.TotalInputProperties, - float64(schemaStats.Resources.InputPropertiesMissingDescriptions)/float64(schemaStats.Resources.TotalInputProperties)*100) - fmt.Println("") - - fmt.Println("Attribute metrics:") - fmt.Printf("\t%d attributes contained an reference and had their descriptions dropped.\n", - elidedAttributes) + if elidedArguments > 0 || elidedNestedArguments > 0 { + fmt.Printf("\t%d arguments contained an reference and had their descriptions dropped.\n", + elidedArguments) + fmt.Printf("\t%d nested arguments contained an reference and had their descriptions dropped.\n", + elidedNestedArguments) + } + fmt.Printf( + "\t%d of %d resource inputs (%.2f%%) are missing descriptions in the schema\n", + schemaStats.Resources.InputPropertiesMissingDescriptions, + schemaStats.Resources.TotalInputProperties, + float64( + schemaStats.Resources.InputPropertiesMissingDescriptions, + )/float64( + schemaStats.Resources.TotalInputProperties, + )*100, + ) fmt.Println("") }