Replace tfgen result summaries (#1456)

Should address #1348.  
Replaces the current summaries:  
```
General metrics:
	1241 total resources containing 13068 total inputs.
	511 total functions.
	0 entities are missing docs entirely because they could not be found in the upstream provider.

Description metrics:
	0 entity descriptions contained an <elided> reference and were dropped, including examples.
	0 entity descriptions contained an <elided> reference and were dropped, but examples were preserved.

Example conversion metrics:
	95 HCL examples failed to convert in all languages
	59 HCL examples were converted in at least one language but failed to convert to TypeScript
	59 HCL examples were converted in at least one language but failed to convert to Python
	59 HCL examples were converted in at least one language but failed to convert to Go
	59 HCL examples were converted in at least one language but failed to convert to C#
	73 entity document sections contained unexpected HCL code snippets. Examples will be converted, but may not display correctly in the registry, e.g. lacking tabs.

Argument metrics:
	9594 argument descriptions were parsed from the upstream docs
	196 top-level input property descriptions came from an upstream attribute (as opposed to an argument). Nested arguments are not included in this count.
	0 arguments contained an <elided> reference and had their descriptions dropped.
	0 nested arguments contained an <elided> reference and had their descriptions dropped.
	542 of 13068 resource inputs (4.15%) are missing descriptions in the schema

Attribute metrics:
	0 attributes contained an <elided> reference and had their descriptions dropped.
```

with the summaries from `examples_coverage_tracker`:

```
Additional example conversion stats are available by setting COVERAGE_OUTPUT_DIR.
Provider:     azuread
Success rate: 97.75% (522/534)

Converted 97.75% of csharp examples (87/89)
Converted 97.75% of go examples (87/89)
Converted 98.88% of java examples (88/89)
Converted 97.75% of python examples (87/89)
Converted 97.75% of typescript examples (87/89)
Converted 96.63% of yaml examples (86/89)
```

I've also added a message about the detailed summaries since it might be
useful for people to find out about the additional stats available:
`Additional example conversion stats are available by setting
COVERAGE_OUTPUT_DIR.`.

Happy to remove it, let me know if I should.

pkg/tfgen/docs.go

@@ -716,7 +716,6 @@ func (p *tfMarkdownParser) parseSection(h2Section []string) error {
 			!p.info.ReplaceExamplesSection() {
 			p.sink.warn("Unexpected code snippets in section '%v' for %v '%v'. The HCL code will be converted if possible, "+
 				"but may not display correctly in the generated docs.", header, p.kind, p.rawname)
-			unexpectedSnippets++
 		}
 
 		// Now process the content based on the H2 topic. These are mostly standard across TF's docs.
@@ -1294,7 +1293,6 @@ func (g *Generator) convertExamplesInner(
 	stripSubsectionsWithErrors bool,
 	convertHCL func(hcl, path, exampleTitle string, languages []string) (string, error),
 ) (result string) {
-
 	output := &bytes.Buffer{}
 
 	writeTrailingNewline := func(buf *bytes.Buffer) {
@@ -1531,7 +1529,6 @@ func (g *Generator) legacyConvert(
 // convertHCLToString hides the implementation details of the upstream implementation for HCL conversion and provides
 // simplified parameters and return values
 func (g *Generator) convertHCLToString(hclCode, path, languageName string) (string, error) {
-
 	fileName := fmt.Sprintf("/%s.tf", strings.ReplaceAll(path, "/", "-"))
 
 	var convertedHcl string
@@ -1577,7 +1574,6 @@ type languages []string
 func (s languages) Len() int      { return len(s) }
 func (s languages) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
 func (s languages) Less(i, j int) bool {
-
 	notFound := -1
 
 	indexOf := func(item string, data []string) int {
@@ -1684,7 +1680,6 @@ func (g *Generator) convertHCL(hcl, path, exampleTitle string, languages []strin
 	isCompleteFailure := len(failedLangs) == len(languages)
 
 	if isCompleteFailure {
-		hclAllLangsConversionFailures++
 		if exampleTitle == "" {
 			g.warn(fmt.Sprintf("unable to convert HCL example for Pulumi entity '%s': %v. The example will be dropped "+
 				"from any generated docs or SDKs.", path, err))
@@ -1701,18 +1696,6 @@ func (g *Generator) convertHCL(hcl, path, exampleTitle string, languages []strin
 
 	for lang := range failedLangs {
 		failedLangsStrings = append(failedLangsStrings, lang)
-
-		switch lang {
-		case convert.LanguageTypescript:
-			hclTypeScriptPartialConversionFailures++
-		case convert.LanguagePython:
-			hclPythonPartialConversionFailures++
-		case convert.LanguageCSharp:
-			hclCSharpPartialConversionFailures++
-		case convert.LanguageGo:
-			hclGoPartialConversionFailures++
-		}
-
 		if exampleTitle == "" {
 			g.warn(fmt.Sprintf("unable to convert HCL example for Pulumi entity '%s' in the following language(s): "+
 				"%s. Examples for these languages will be dropped from any generated docs or SDKs.",
@@ -1794,7 +1777,6 @@ func cleanupDoc(
 		g.debug("Cleaning up text for attribute [%v] in [%v]", k, name)
 		cleanedText, elided := reformatText(infoCtx, v, footerLinks)
 		if elided {
-			elidedAttributes++
 			g.warn("Found <elided> in docs for attribute [%v] in [%v]. The attribute's description will be dropped "+
 				"in the Pulumi provider.", k, name)
 			elidedDoc = true
@@ -1814,7 +1796,6 @@ func cleanupDoc(
 		if examples == "" {
 			g.debug("Unable to find any examples in the description text. The entire description will be discarded.")
 
-			elidedDescriptions++
 			g.warn("Found <elided> in description for [%v]. The description and any examples will be dropped in the "+
 				"Pulumi provider.", name)
 			elidedDoc = true
@@ -1823,12 +1804,10 @@ func cleanupDoc(
 
 			cleanedupExamples, examplesElided := reformatText(infoCtx, examples, footerLinks)
 			if examplesElided {
-				elidedDescriptions++
 				g.warn("Found <elided> in description for [%v]. The description and any examples will be dropped in "+
 					"the Pulumi provider.", name)
 				elidedDoc = true
 			} else {
-				elidedDescriptionsOnly++
 				g.warn("Found <elided> in description for [%v], but was able to preserve the examples. The description "+
 					"proper will be dropped in the Pulumi provider.", name)
 				cleanupText = cleanedupExamples
@@ -1947,7 +1926,6 @@ func extractExamples(description string) string {
 
 // reformatText processes markdown strings from TF docs and cleans them for inclusion in Pulumi docs
 func reformatText(g infoContext, text string, footerLinks map[string]string) (string, bool) {
-
 	cleanupText := func(text string) (string, bool) {
 		// Remove incorrect documentation that should have been cleaned up in our forks.
 		if strings.Contains(text, "Terraform") || strings.Contains(text, "terraform") {

pkg/tfgen/examples_coverage_exporter.go

@@ -440,64 +440,63 @@ func (ce *coverageExportUtil) exportMarkdown(outputDirectory string, fileName st
 	return os.WriteFile(targetFile, []byte(out), 0600)
 }
 
-// The fourth mode, which simply gives the provider name, and success percentage.
-func (ce *coverageExportUtil) exportHumanReadable(outputDirectory string, fileName string) error {
-
-	// The Coverage Tracker data structure is flattened to gather statistics about each language
-	type LanguageStatistic struct {
-		Total     int
-		Successes int
-	}
+// The Coverage Tracker data structure is flattened to gather statistics about each language
+type languageStatistic struct {
+	Total     int
+	Successes int
+}
 
-	type ProviderStatistic struct {
-		Name             string
-		Examples         int
-		TotalConversions int
-		Successes        int
-	}
+type providerStatistic struct {
+	Name             string
+	Examples         int
+	TotalConversions int
+	Successes        int
+}
 
+func (ce coverageExportUtil) produceStatistics() (map[string]*languageStatistic, providerStatistic) {
 	// Main maps for holding the overall provider summary, and each language conversion statistic
-	var allLanguageStatistics = make(map[string]*LanguageStatistic)
-	var providerStatistic = ProviderStatistic{ce.Tracker.ProviderName, 0, 0, 0}
+	var allLanguageStatistics = make(map[string]*languageStatistic)
+	var providerStats = providerStatistic{ce.Tracker.ProviderName, 0, 0, 0}
 
 	// All the conversion attempts for each example are iterated by language name and
 	// their results are added to the main map
 	for _, page := range ce.Tracker.EncounteredPages {
 		for _, example := range page.Examples {
-			providerStatistic.Examples++
+			providerStats.Examples++
 			for languageName, conversionResult := range example.ConversionResults {
-				providerStatistic.TotalConversions++
+				providerStats.TotalConversions++
 
 				// Obtaining the current language we will be creating statistics for
-				var currentLanguage *LanguageStatistic
+				var currentLanguage *languageStatistic
 				if val, ok := allLanguageStatistics[languageName]; ok {
 
 					// Current language already exists in main map
 					currentLanguage = val
 				} else {
 
 					// The main map doesn't yet contain this language, and it needs to be added
-					allLanguageStatistics[languageName] = &LanguageStatistic{0, 0}
+					allLanguageStatistics[languageName] = &languageStatistic{0, 0}
 					currentLanguage = allLanguageStatistics[languageName]
 				}
 
 				// The language's entry in the summarized results is updated and any
 				currentLanguage.Total++
 				if conversionResult.FailureSeverity == Success {
-					providerStatistic.Successes++
+					providerStats.Successes++
 					currentLanguage.Successes++
 				}
 			}
 		}
 	}
 
-	targetFile, err := createEmptyFile(outputDirectory, fileName)
-	if err != nil {
-		return err
-	}
+	return allLanguageStatistics, providerStats
+}
+
+func (ce *coverageExportUtil) produceHumanReadableSummary() string {
+	allLanguageStatistics, providerStatistic := ce.produceStatistics()
 
 	// Forming a string which will eventually be written to the target file
-	fileString := fmt.Sprintf("Provider:     %s\nSuccess rate: %.2f%% (%d/%d)\n\n",
+	fileString := fmt.Sprintf("\nProvider:     %s\nSuccess rate: %.2f%% (%d/%d)\n\n",
 		providerStatistic.Name,
 		float64(providerStatistic.Successes)/float64(providerStatistic.TotalConversions)*100.0,
 		providerStatistic.Successes,
@@ -522,6 +521,17 @@ func (ce *coverageExportUtil) exportHumanReadable(outputDirectory string, fileNa
 		)
 	}
 
+	return fileString
+}
+
+// The fourth mode, which simply gives the provider name, and success percentage.
+func (ce *coverageExportUtil) exportHumanReadable(outputDirectory string, fileName string) error {
+	targetFile, err := createEmptyFile(outputDirectory, fileName)
+	if err != nil {
+		return err
+	}
+	fileString := ce.produceHumanReadableSummary()
+
 	return os.WriteFile(targetFile, []byte(fileString), 0600)
 }
 

pkg/tfgen/examples_coverage_tracker.go

@@ -236,3 +236,8 @@ func (ct *CoverageTracker) exportResults(outputDirectory string) error {
 	coverageExportUtil := newCoverageExportUtil(ct)
 	return coverageExportUtil.tryExport(outputDirectory)
 }
+
+func (ct *CoverageTracker) getShortResultSummary() string {
+	coverageExportUtil := newCoverageExportUtil(ct)
+	return coverageExportUtil.produceHumanReadableSummary()
+}

pkg/tfgen/generate.go

@@ -96,7 +96,7 @@ const (
 
 func (l Language) shouldConvertExamples() bool {
 	switch l {
-	case Golang, NodeJS, Python, CSharp, Schema, PCL:
+	case Schema:
 		return true
 	}
 	return false
@@ -987,9 +987,6 @@ func (g *Generator) UnstableGenerateFromSchema(genSchemaResult *GenerateSchemaRe
 		return errors.Wrapf(err, "failed to create project file")
 	}
 
-	// Print out some documentation stats as a summary afterwards.
-	printDocStats()
-
 	// Close the plugin host.
 	g.pluginHost.Close()
 

pkg/tfgen/main.go

@@ -142,10 +142,8 @@ func newTFGenCmd(pkg string, version string, prov tfbridge.ProviderInfo,
 			// Creating an item to keep track of example coverage if the
 			// COVERAGE_OUTPUT_DIR env is set
 			var coverageTracker *CoverageTracker
-			coverageOutputDir, coverageTrackingEnabled := os.LookupEnv("COVERAGE_OUTPUT_DIR")
-			if coverageTrackingEnabled {
-				coverageTracker = newCoverageTracker(prov.Name, prov.Version)
-			}
+			coverageOutputDir, coverageTrackingOutputEnabled := os.LookupEnv("COVERAGE_OUTPUT_DIR")
+			coverageTracker = newCoverageTracker(prov.Name, prov.Version)
 
 			opts := GeneratorOptions{
 				Package:         pkg,
@@ -160,10 +158,15 @@ func newTFGenCmd(pkg string, version string, prov tfbridge.ProviderInfo,
 			}
 
 			err := gen(opts)
-
-			// Exporting collected coverage data to the directory specified by COVERAGE_OUTPUT_DIR
-			if coverageTrackingEnabled {
-				err = coverageTracker.exportResults(coverageOutputDir)
+			if opts.Language.shouldConvertExamples() {
+				// Exporting collected coverage data to the directory specified by COVERAGE_OUTPUT_DIR
+				if coverageTrackingOutputEnabled {
+					err = coverageTracker.exportResults(coverageOutputDir)
+				} else {
+					fmt.Println("\nAdditional example conversion stats are available by setting COVERAGE_OUTPUT_DIR.")
+				}
+				fmt.Println(coverageTracker.getShortResultSummary())
+				printDocStats()
 			}
 
 			return err

pkg/tfgen/metrics.go

@@ -16,87 +16,60 @@ package tfgen
 
 import (
 	"fmt"
+
 	schemaTools "github.com/pulumi/schema-tools/pkg"
 )
 
 var (
-	ignoredDocHeaders      = make(map[string]int)
-	elidedDescriptions     int // i.e., we discard the entire description, including examples
-	elidedDescriptionsOnly int // we discarded the description proper, but were able to preserve the examples
-	elidedArguments        int
-	elidedNestedArguments  int
-	elidedAttributes       int
-
-	unexpectedSnippets            int
-	hclAllLangsConversionFailures int // examples that failed to convert in any language
-
-	// examples that failed to convert in one, but not all, languages. This is less severe impact because users will
-	// at least have code in another language to reference:
-	hclGoPartialConversionFailures         int
-	hclPythonPartialConversionFailures     int
-	hclTypeScriptPartialConversionFailures int
-	hclCSharpPartialConversionFailures     int
+	ignoredDocHeaders     = make(map[string]int)
+	elidedArguments       int
+	elidedNestedArguments int
+	unexpectedSnippets    int
 
 	// Arguments metrics:
 	totalArgumentsFromDocs int
 	// See comment in getNestedDescriptionFromParsedDocs for why we track this behavior:
 	argumentDescriptionsFromAttributes int
-
-	// General metrics:
-	entitiesMissingDocs int
+	entitiesMissingDocs                int
 
 	schemaStats schemaTools.PulumiSchemaStats
 )
 
 // printDocStats outputs metrics relating to document parsing and conversion
 func printDocStats() {
-	fmt.Println("")
-
 	fmt.Println("General metrics:")
 	fmt.Printf("\t%d total resources containing %d total inputs.\n",
 		schemaStats.Resources.TotalResources, schemaStats.Resources.TotalInputProperties)
 	fmt.Printf("\t%d total functions.\n", schemaStats.Functions.TotalFunctions)
-	fmt.Printf("\t%d entities are missing docs entirely because they could not be found in the upstream provider.\n",
-		entitiesMissingDocs)
-	fmt.Println("")
-
-	fmt.Println("Description metrics:")
-	fmt.Printf("\t%d entity descriptions contained an <elided> reference and were dropped, including examples.\n",
-		elidedDescriptions)
-	fmt.Printf("\t%d entity descriptions contained an <elided> reference and were dropped, but examples were preserved.\n",
-		elidedDescriptionsOnly)
-	fmt.Println("")
-
-	fmt.Println("Example conversion metrics:")
-	fmt.Printf("\t%d HCL examples failed to convert in all languages\n", hclAllLangsConversionFailures)
-	fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to TypeScript\n",
-		hclTypeScriptPartialConversionFailures)
-	fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to Python\n",
-		hclPythonPartialConversionFailures)
-	fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to Go\n",
-		hclGoPartialConversionFailures)
-	fmt.Printf("\t%d HCL examples were converted in at least one language but failed to convert to C#\n",
-		hclCSharpPartialConversionFailures)
-	fmt.Printf("\t%d entity document sections contained unexpected HCL code snippets. Examples will be converted, "+
-		"but may not display correctly in the registry, e.g. lacking tabs.\n", unexpectedSnippets)
+	if entitiesMissingDocs > 0 {
+		fmt.Printf("\t%d entities are missing docs entirely because they could not be found in the upstream provider.\n",
+			entitiesMissingDocs)
+	}
+	if unexpectedSnippets > 0 {
+		fmt.Printf("\t%d entity document sections contained unexpected HCL code snippets. Examples will be converted, "+
+			"but may not display correctly in the registry, e.g. lacking tabs.\n", unexpectedSnippets)
+	}
 	fmt.Println("")
 
 	fmt.Println("Argument metrics:")
 	fmt.Printf("\t%d argument descriptions were parsed from the upstream docs\n", totalArgumentsFromDocs)
 	fmt.Printf("\t%d top-level input property descriptions came from an upstream attribute (as opposed to an argument). "+
 		"Nested arguments are not included in this count.\n", argumentDescriptionsFromAttributes)
-	fmt.Printf("\t%d arguments contained an <elided> reference and had their descriptions dropped.\n",
-		elidedArguments)
-	fmt.Printf("\t%d nested arguments contained an <elided> reference and had their descriptions dropped.\n",
-		elidedNestedArguments)
-	//nolint:lll
-	fmt.Printf("\t%d of %d resource inputs (%.2f%%) are missing descriptions in the schema\n",
-		schemaStats.Resources.InputPropertiesMissingDescriptions, schemaStats.Resources.TotalInputProperties,
-		float64(schemaStats.Resources.InputPropertiesMissingDescriptions)/float64(schemaStats.Resources.TotalInputProperties)*100)
-	fmt.Println("")
-
-	fmt.Println("Attribute metrics:")
-	fmt.Printf("\t%d attributes contained an <elided> reference and had their descriptions dropped.\n",
-		elidedAttributes)
+	if elidedArguments > 0 || elidedNestedArguments > 0 {
+		fmt.Printf("\t%d arguments contained an <elided> reference and had their descriptions dropped.\n",
+			elidedArguments)
+		fmt.Printf("\t%d nested arguments contained an <elided> reference and had their descriptions dropped.\n",
+			elidedNestedArguments)
+	}
+	fmt.Printf(
+		"\t%d of %d resource inputs (%.2f%%) are missing descriptions in the schema\n",
+		schemaStats.Resources.InputPropertiesMissingDescriptions,
+		schemaStats.Resources.TotalInputProperties,
+		float64(
+			schemaStats.Resources.InputPropertiesMissingDescriptions,
+		)/float64(
+			schemaStats.Resources.TotalInputProperties,
+		)*100,
+	)
 	fmt.Println("")
 }