Add parameterized provider support to YAML

CHANGELOG_PENDING.md

@@ -2,6 +2,8 @@
 
 - [features] add "pulumi.organiztion" to the built-in "pulumi" variable to obtain the current organization.
 
+- [features] add support for parameterized packages.
+
 ### Bug Fixes
 
 - Parse the items property on config type declarations to prevent diagnostic messages about

go.mod

@@ -18,6 +18,8 @@ require (
 	github.com/stretchr/testify v1.9.0
 	github.com/zclconf/go-cty v1.13.2
 	google.golang.org/grpc v1.63.2
+	google.golang.org/protobuf v1.33.0
+	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -186,7 +188,6 @@ require (
 	google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20240311173647-c811ad7063a7 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20240311173647-c811ad7063a7 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/warnings.v0 v0.1.2 // indirect
 	lukechampine.com/frand v1.4.2 // indirect
 	mvdan.cc/gofumpt v0.5.0 // indirect

pkg/pulumiyaml/packages/file.go

@@ -0,0 +1,97 @@
+// Copyright 2024, Pulumi Corporation.  All rights reserved.
+
+// Package packages contains utilities for working with Pulumi package lock files.
+package packages
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/fs"
+	"os"
+	"path/filepath"
+
+	"gopkg.in/yaml.v2"
+)
+
+type ParameterizationLock struct {
+	// Name is the name of the parameterized package.
+	Name string `yaml:"name"`
+	// Version is the version of the parameterized package.
+	Version string `yaml:"version"`
+	// Value is the value of the parameter. Value is a base64 encoded byte array, use SetValue and GetValue to manipulate the actual value.
+	Value string `yaml:"value"`
+}
+
+func (p *ParameterizationLock) GetValue() ([]byte, error) {
+	return base64.StdEncoding.DecodeString(p.Value)
+}
+
+func (p *ParameterizationLock) SetValue(value []byte) {
+	p.Value = base64.StdEncoding.EncodeToString(value)
+}
+
+type PackageLock struct {
+	// Name is the name of the plugin.
+	Name string `yaml:"name"`
+	// Version is the version of the plugin.
+	Version string `yaml:"version,omitempty"`
+	// PluginDownloadURL is the URL to download the plugin from.
+	DownloadURL string `yaml:"downloadUrl,omitempty"`
+	// Parameterization is the parameterization of the package.
+	Parameterization *ParameterizationLock `yaml:"parameterization,omitempty"`
+}
+
+func (p *PackageLock) Valid() bool {
+	// All packages need a name
+	if p.Name == "" {
+		return false
+	}
+
+	// If parameterization is not nil, it must be valid.
+	if p.Parameterization != nil {
+		return p.Parameterization.Name != "" && p.Parameterization.Version != ""
+	}
+
+	return true
+}
+
+// SearchPackageLocks searches the given directory down recursively for package lock .yaml files.
+func SearchPackageLocks(directory string) ([]PackageLock, error) {
+	var packages []PackageLock
+	err := filepath.WalkDir(directory, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// If the file is a directory, skip it.
+		if d.IsDir() {
+			return nil
+		}
+
+		// If the file is not a .yaml file, skip it.
+		if filepath.Ext(path) != ".yaml" {
+			return nil
+		}
+
+		// Read the file.
+		data, err := os.ReadFile(path)
+		if err != nil {
+			return fmt.Errorf("reading %s: %w", path, err)
+		}
+
+		var packageLock PackageLock
+		if err := yaml.Unmarshal(data, &packageLock); err != nil {
+			return fmt.Errorf("unmarshalling %s: %w", path, err)
+		}
+
+		// If the file is not valid skip it
+		if !packageLock.Valid() {
+			return nil
+		}
+
+		// Else append it to the list of packages found.
+		packages = append(packages, packageLock)
+		return nil
+	})
+	return packages, err
+}

pkg/pulumiyaml/packages/file_test.go

@@ -0,0 +1,36 @@
+// Copyright 2024, Pulumi Corporation.  All rights reserved.
+
+package packages
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSearchPackageLocks(t *testing.T) {
+	t.Parallel()
+
+	expected := []PackageLock{
+		{
+			Name: "pkg",
+		},
+		{
+			Name:        "pkg2",
+			Version:     "1.2",
+			DownloadURL: "github://api.github.com/pulumiverse",
+		},
+		{
+			Name: "base",
+			Parameterization: &ParameterizationLock{
+				Name:    "pkg",
+				Version: "1.0.0",
+				Value:   "cGtn",
+			},
+		},
+	}
+
+	actual, err := SearchPackageLocks("testdata")
+	require.NoError(t, err)
+	require.ElementsMatch(t, expected, actual)
+}

pkg/pulumiyaml/packages/testdata/bad.yaml

@@ -0,0 +1,2 @@
+# no name so this is invalid
+version: 1.1

pkg/pulumiyaml/packages/testdata/bad_param.yaml

@@ -0,0 +1,5 @@
+name: pkg
+parameterization:
+  name: pkg
+  # no version so this is invalid
+  value: cGtn

pkg/pulumiyaml/packages/testdata/good.yaml

@@ -0,0 +1 @@
+name: pkg

pkg/pulumiyaml/packages/testdata/good_param.yaml

@@ -0,0 +1,5 @@
+name: base
+parameterization:
+  name: pkg
+  version: 1.0.0
+  value: cGtn

pkg/pulumiyaml/packages/testdata/nested/nested.yaml

@@ -0,0 +1,3 @@
+name: pkg2
+version: 1.2
+downloadUrl: github://api.github.com/pulumiverse

pkg/pulumiyaml/run.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/ast"
 	ctypes "github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/config"
+	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/packages"
 	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/syntax"
 	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/syntax/encoding"
 )
@@ -659,6 +660,7 @@ type Evaluator interface {
 type programEvaluator struct {
 	*evalContext
 	pulumiCtx *pulumi.Context
+	packages  []packages.PackageLock
 }
 
 func (e *programEvaluator) error(expr ast.Expr, summary string) (interface{}, bool) {
@@ -767,7 +769,11 @@ func (e programEvaluator) EvalOutput(r *Runner, node ast.PropertyMapEntry) bool
 
 func (r *Runner) Evaluate(ctx *pulumi.Context) syntax.Diagnostics {
 	eCtx := r.newContext(nil)
-	return r.Run(programEvaluator{evalContext: eCtx, pulumiCtx: ctx})
+	packages, err := packages.SearchPackageLocks(r.cwd)
+	if err != nil {
+		return syntax.Diagnostics{syntax.Error(nil, err.Error(), "")}
+	}
+	return r.Run(programEvaluator{evalContext: eCtx, pulumiCtx: ctx, packages: packages})
 }
 
 func getConfNodesFromMap(project string, configPropertyMap resource.PropertyMap) []configNode {

pkg/server/server.go

@@ -19,18 +19,23 @@ package server
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
+	"path/filepath"
 
 	pbempty "github.com/golang/protobuf/ptypes/empty"
+	"github.com/pulumi/pulumi/pkg/v3/codegen/schema"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/resource/plugin"
 	"github.com/pulumi/pulumi/sdk/v3/go/common/version"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 	pulumirpc "github.com/pulumi/pulumi/sdk/v3/proto/go"
+	"gopkg.in/yaml.v2"
 
 	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml"
 	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/ast"
+	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/packages"
 	"github.com/pulumi/pulumi-yaml/pkg/pulumiyaml/syntax"
 )
 
@@ -227,3 +232,85 @@ func (host *yamlLanguageHost) RuntimeOptionsPrompts(context.Context, *pulumirpc.
 func (host *yamlLanguageHost) About(ctx context.Context, req *pulumirpc.AboutRequest) (*pulumirpc.AboutResponse, error) {
 	return &pulumirpc.AboutResponse{}, nil
 }
+
+func (host *yamlLanguageHost) GeneratePackage(ctx context.Context, req *pulumirpc.GeneratePackageRequest) (*pulumirpc.GeneratePackageResponse, error) {
+	// YAML doesn't generally have "SDKs" per-se but we can write out a "lock file" for a given package name and
+	// version, and if using a parameterized package this is necessary so that we have somewhere to save the parameter
+	// value.
+
+	if len(req.ExtraFiles) > 0 {
+		return nil, errors.New("overlays are not supported for Go")
+	}
+
+	loader, err := schema.NewLoaderClient(req.LoaderTarget)
+	if err != nil {
+		return nil, err
+	}
+
+	var spec schema.PackageSpec
+	err = json.Unmarshal([]byte(req.Schema), &spec)
+	if err != nil {
+		return nil, err
+	}
+
+	pkg, diags, err := schema.BindSpec(spec, loader)
+	if err != nil {
+		return nil, err
+	}
+	rpcDiagnostics := plugin.HclDiagnosticsToRPCDiagnostics(diags)
+	if diags.HasErrors() {
+		return &pulumirpc.GeneratePackageResponse{
+			Diagnostics: rpcDiagnostics,
+		}, nil
+	}
+
+	// Generate the a package lock file in the given directory. This is just a simple YAML file that contains the name,
+	// version, and any parameter values.
+	lock := packages.PackageLock{}
+	// The format of the lock file differs based on if this is a parameterized package or not.
+	if pkg.Parameterization == nil {
+		lock.Name = pkg.Name
+		if pkg.Version != nil {
+			lock.Version = pkg.Version.String()
+		}
+		lock.DownloadURL = pkg.PluginDownloadURL
+	} else {
+		lock.Name = pkg.Parameterization.BaseProvider.Name
+		lock.Version = pkg.Parameterization.BaseProvider.Version.String()
+		lock.DownloadURL = pkg.PluginDownloadURL
+		if pkg.Version == nil {
+			return nil, errors.New("parameterized package must have a version")
+		}
+		lock.Parameterization = &packages.ParameterizationLock{
+			Name:    pkg.Name,
+			Version: pkg.Version.String(),
+		}
+		lock.Parameterization.SetValue(pkg.Parameterization.Parameter)
+	}
+
+	// Write out a yaml file for this package
+	var version string
+	if pkg.Version != nil {
+		version = fmt.Sprintf("-%s", pkg.Version.String())
+	}
+	dest := filepath.Join(req.Directory, fmt.Sprintf("%s%s.yaml", pkg.Name, version))
+
+	data, err := yaml.Marshal(lock)
+	if err != nil {
+		return nil, err
+	}
+
+	err = os.MkdirAll(req.Directory, 0o700)
+	if err != nil {
+		return nil, fmt.Errorf("could not create output directory %s: %w", req.Directory, err)
+	}
+
+	err = os.WriteFile(dest, data, 0o600)
+	if err != nil {
+		return nil, fmt.Errorf("could not write output file %s: %w", dest, err)
+	}
+
+	return &pulumirpc.GeneratePackageResponse{
+		Diagnostics: rpcDiagnostics,
+	}, nil
+}

pkg/tests/integration_test.go

@@ -3,12 +3,15 @@
 package tests
 
 import (
+	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
+	ptesting "github.com/pulumi/pulumi/sdk/v3/go/common/testing"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
 func integrationDir(dir string) string {
@@ -121,3 +124,27 @@ func TestEnvVarsKeepConflictingValues(t *testing.T) {
 	}
 	integration.ProgramTest(t, &testOptions)
 }
+
+// Test a paramaterized provider.
+//
+//nolint:paralleltest // ProgramTest calls t.Parallel()
+func TestParameterized(t *testing.T) {
+	e := ptesting.NewEnvironment(t)
+	// We can't use ImportDirectory here because we need to run this in the right directory such that the relative paths
+	// work. This also means we don't delete the directory after the test runs.
+	var err error
+	e.CWD, err = filepath.Abs("testdata/parameterized")
+	require.NoError(t, err)
+
+	err = os.RemoveAll(filepath.Join("testdata", "parameterized", "sdk"))
+	require.NoError(t, err)
+
+	_, _ = e.RunCommand("pulumi", "package", "gen-sdk", "../../testprovider", "pkg", "--language", "yaml", "--local")
+
+	integration.ProgramTest(t, &integration.ProgramTestOptions{
+		Dir: filepath.Join("testdata", "parameterized"),
+		LocalProviders: []integration.LocalDependency{
+			{Package: "testprovider", Path: "testprovider"},
+		},
+	})
+}

pkg/tests/testdata/parameterized/Pulumi.yaml

@@ -0,0 +1,8 @@
+name: parameterized
+description: An integration test showing the use of a parameterized package
+runtime: yaml
+resources:
+  res1:
+    type: pkg:index:Random
+  res2:
+    type: pkg:index:Echo

pkg/tests/testdata/parameterized/sdk/yaml/pkg-1.0.0.yaml

@@ -0,0 +1,6 @@
+name: testprovider
+version: 0.0.1
+parameterization:
+  name: pkg
+  version: 1.0.0
+  value: cGtn

pkg/tests/testprovider/.gitignore

@@ -0,0 +1,2 @@
+pulumi-resource-testprovider
+schema-testprovider.json

pkg/tests/testprovider/PulumiPlugin.yaml

@@ -0,0 +1 @@
+runtime: go