Emit <!--Start PulumiCodeChooser --> instead of {{% examples %}} (#…

…1689)

**IMPORTANT** This cannot be merged until the bridge has updated
pulumi/pulumi to a version that includes [these
changes](pulumi/pulumi@26bdac7).

After that, merging will be safe.


**Note**: pulumi/pulumi#15475 needs to be merged
first for these changes to be safe.

This PR aims to remove the nested "Example Usage" logic from the bridge,
[as discussed here](pulumi/registry#3498).

The issue this is aimed at fixing is
pulumi/pulumi-aws#2624 but there should be a
myriad of other issues that will be addressed by this change.

In a nutshell, this PR removes any `{{% example(s) }}` injection from
the `Description` fields of a resource and instead detects any and all
code conversions from TF/HCL --> pulumi languages, and surrounds them
with an HTML comment.
This is currently 
```
const (
	startPulumiCodeChooser = "<!--Start PulumiCodeChooser -->"
	endPulumiCodeChooser   = "<!--End PulumiCodeChooser -->"
)
```

Marking up any converted code blocks using the new format will serve
_only_ as a marker for registry docsgen to surround this converted code
with a language chooser.

I refrained from going full Markdown parser because I believe it would
be an even larger rewrite. Instead, we detect code fences in the
docstring input, and append text and converted code section to the
output, with the following caveats:

- If a code block is part of a Section (i.e. has a Markdown header), if
we encounter errors in code conversion, the new functionality strips the
entire section automatically. Non-headered code blocks with such an
error will also continue to be stripped.
- Using indices rather than line splitting allows us to get rid of the
awkward `<break>` placeholders for the Import section.
- The Import section no longer needs special handling in
`convertExamplesInner`: it is valid code and will be retained as such
- With a little bit of extra parsing of the code fence syntax
highlighter, we can capture a few more cases of perfectly valid json or
yaml that we have elided previously as "failure to convert from TF".
- With the removal of examples, titles (which never actually made it
into the schema) are now obsolete. We will refer to resource paths when
logging conversion failures. We have done so all along when there was no
`exampleTitle`, and this change affects only the wording of logging
output.
- The `isFrontMatter` variable was removed as well. To the best of my
knowledge, Hugo front matter does not get piped through the bridge, and
is generated by the registry instead.
- Handwritten docs continue to bypass `convertExamplesInner`, with an
added caveat that both third party and our own docs may add the old
example shortcodes. This change will allow us to [remove some of
these](https://github.com/pulumi/pulumi-aws/blob/master/docs/resource/aws_acm_certificate_validation.examples.md).
Registry docsgen will continue to support the shortcode way of doing
things, since native providers conform to that expectation separately.

---------

Co-authored-by: Ian Wahbe <[email protected]>

pkg/tfgen/convert_cli.go

@@ -64,14 +64,13 @@ type cliConverter struct {
 
 	generator interface {
 		convertHCL(
-			e *Example, hcl, path, exampleTitle string, languages []string,
+			e *Example, hcl, path string, languages []string,
 		) (string, error)
 		convertExamplesInner(
 			docs string,
 			path examplePath,
-			stripSubsectionsWithErrors bool,
 			convertHCL func(
-				e *Example, hcl, path, exampleTitle string, languages []string,
+				e *Example, hcl, path string, languages []string,
 			) (string, error),
 			useCoverageTracker bool,
 		) string
@@ -81,9 +80,8 @@ type cliConverter struct {
 	loader schema.Loader
 
 	convertExamplesList []struct {
-		docs                       string
-		path                       examplePath
-		stripSubsectionsWithErrors bool
+		docs string
+		path examplePath
 	}
 
 	currentPackageSpec *pschema.PackageSpec
@@ -124,20 +122,17 @@ func (g *Generator) cliConverter() *cliConverter {
 func (cc *cliConverter) StartConvertingExamples(
 	docs string,
 	path examplePath,
-	stripSubsectionsWithErrors bool,
 ) string {
 	// Record inner HCL conversions and discard the result.
 	cov := false // do not use coverage tracker yet, it will be used in the second pass.
-	cc.generator.convertExamplesInner(docs, path, stripSubsectionsWithErrors, cc.recordHCL, cov)
+	cc.generator.convertExamplesInner(docs, path, cc.recordHCL, cov)
 	// Record the convertExamples job for later.
 	e := struct {
-		docs                       string
-		path                       examplePath
-		stripSubsectionsWithErrors bool
+		docs string
+		path examplePath
 	}{
-		docs:                       docs,
-		path:                       path,
-		stripSubsectionsWithErrors: stripSubsectionsWithErrors,
+		docs: docs,
+		path: path,
 	}
 	cc.convertExamplesList = append(cc.convertExamplesList, e)
 	// Return a placeholder referencing the convertExampleJob by position.
@@ -165,8 +160,7 @@ func (cc *cliConverter) FinishConvertingExamples(p pschema.PackageSpec) pschema.
 
 		// Use coverage tracker here on the second pass.
 		useCoverageTracker := true
-		source := cc.generator.convertExamplesInner(ex.docs, ex.path,
-			ex.stripSubsectionsWithErrors, cc.generator.convertHCL, useCoverageTracker)
+		source := cc.generator.convertExamplesInner(ex.docs, ex.path, cc.generator.convertHCL, useCoverageTracker)
 		// JSON-escaping to splice into JSON string literals.
 		bytes, err := json.Marshal(source)
 		contract.AssertNoErrorf(err, "json.Masrhal(sourceCode)")
@@ -465,7 +459,7 @@ func (cc *cliConverter) convertPCL(
 
 // Act as a convertHCL stub that does not actually convert but spies on the literals involved.
 func (cc *cliConverter) recordHCL(
-	e *Example, hcl, path, exampleTitle string, languages []string,
+	e *Example, hcl, path string, languages []string,
 ) (string, error) {
 	cache := cc.generator.getOrCreateExamplesCache()