Adds support for cloud-run-v2, modal work pool metadata (#279)

* Adds cloud run v2 to worker_metadata * Fixes test in TestAccDatasource_worker_metadata * Use the same endpoints as the Prefect client The endpoint to list the base job configs differs between OSS and Cloud. This change calculates the correct endpoint, and updates the test to refer to the specific evergreen workspace used for testing. * Generate Terraform Docs * Support Modal push workers * Generate Terraform Docs --------- Co-authored-by: Mitchell Nielsen <[email protected]> Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
PrefectHQ · Oct 8, 2024 · e237d13 · e237d13
1 parent 5933437
commit e237d13
Show file tree

Hide file tree

Showing 5 changed files with 88 additions and 9 deletions.
diff --git a/docs/data-sources/worker_metadata.md b/docs/data-sources/worker_metadata.md
@@ -50,6 +50,11 @@ resource "prefect_work_pool" "process" {
 <!-- schema generated by tfplugindocs -->
 ## Schema
 
+### Optional
+
+- `account_id` (String) Account ID (UUID), defaults to the account set in the provider
+- `workspace_id` (String) Workspace ID (UUID), defaults to the workspace set in the provider
+
 ### Read-Only
 
 - `base_job_configs` (Attributes) A map of default base job configurations (JSON) for each of the primary worker types (see [below for nested schema](#nestedatt--base_job_configs))
@@ -63,10 +68,13 @@ Read-Only:
 - `azure_container_instances_push` (String) Default base job configuration for Azure Container Instances Push workers
 - `cloud_run` (String) Default base job configuration for Cloud Run workers
 - `cloud_run_push` (String) Default base job configuration for Cloud Run Push workers
+- `cloud_run_v2` (String) Default base job configuration for Cloud Run V2 workers
+- `cloud_run_v2_push` (String) Default base job configuration for Cloud Run V2 Push workers
 - `docker` (String) Default base job configuration for Docker workers
 - `ecs` (String) Default base job configuration for ECS workers
 - `ecs_push` (String) Default base job configuration for ECS Push workers
 - `kubernetes` (String) Default base job configuration for Kubernetes workers
+- `modal_push` (String) Default base job configuration for Modal Push workers
 - `prefect_agent` (String) Default base job configuration for Prefect Agent workers
 - `process` (String) Default base job configuration for Process workers
 - `vertex_ai` (String) Default base job configuration for Vertex AI workers
diff --git a/internal/api/client.go b/internal/api/client.go
@@ -12,7 +12,7 @@ type PrefectClient interface {
 	BlockDocuments(accountID uuid.UUID, workspaceID uuid.UUID) (BlockDocumentClient, error)
 	BlockSchemas(accountID uuid.UUID, workspaceID uuid.UUID) (BlockSchemaClient, error)
 	BlockTypes(accountID uuid.UUID, workspaceID uuid.UUID) (BlockTypeClient, error)
-	Collections() (CollectionsClient, error)
+	Collections(accountID uuid.UUID, workspaceID uuid.UUID) (CollectionsClient, error)
 	Deployments(accountID uuid.UUID, workspaceID uuid.UUID) (DeploymentsClient, error)
 	Teams(accountID uuid.UUID) (TeamsClient, error)
 	Flows(accountID uuid.UUID, workspaceID uuid.UUID) (FlowsClient, error)

diff --git a/internal/client/collections.go b/internal/client/collections.go
@@ -7,7 +7,9 @@ import (
 	"io"
 	"net/http"
 
+	"github.com/google/uuid"
 	"github.com/prefecthq/terraform-provider-prefect/internal/api"
+	"github.com/prefecthq/terraform-provider-prefect/internal/provider/helpers"
 )
 
 var _ = api.CollectionsClient(&CollectionsClient{})
@@ -21,18 +23,37 @@ type CollectionsClient struct {
 // Collections returns an CollectionsClient.
 //
 //nolint:ireturn // required to support PrefectClient mocking
-func (c *Client) Collections() (api.CollectionsClient, error) {
+func (c *Client) Collections(accountID, workspaceID uuid.UUID) (api.CollectionsClient, error) {
+	if accountID == uuid.Nil {
+		accountID = c.defaultAccountID
+	}
+
+	if workspaceID == uuid.Nil {
+		workspaceID = c.defaultWorkspaceID
+	}
+
+	if helpers.IsCloudEndpoint(c.endpoint) && (accountID == uuid.Nil || workspaceID == uuid.Nil) {
+		return nil, fmt.Errorf("prefect Cloud endpoints require an account_id and workspace_id to be set on either the provider or the resource")
+	}
+
 	return &CollectionsClient{
 		hc:          c.hc,
 		apiKey:      c.apiKey,
-		routePrefix: fmt.Sprintf("%s/collections", c.endpoint),
+		routePrefix: getWorkspaceScopedURL(c.endpoint, accountID, workspaceID, "collections"),
 	}, nil
 }
 
 // GetWorkerMetadataViews returns a map of worker metadata views by prefect package name.
 // This endpoint serves base job configurations for the primary worker types.
 func (c *CollectionsClient) GetWorkerMetadataViews(ctx context.Context) (api.WorkerTypeByPackage, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/views/aggregate-worker-metadata", c.routePrefix), http.NoBody)
+	routeSuffix := "views/aggregate-worker-metadata"
+	if helpers.IsCloudEndpoint(c.routePrefix) {
+		routeSuffix = "work_pool_types"
+	}
+
+	url := fmt.Sprintf("%s/%s", c.routePrefix, routeSuffix)
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
 	if err != nil {
 		return nil, fmt.Errorf("error creating request: %w", err)
 	}

diff --git a/internal/provider/datasources/worker_metadata.go b/internal/provider/datasources/worker_metadata.go
@@ -10,6 +10,7 @@ import (
 	"github.com/hashicorp/terraform-plugin-framework/datasource/schema"
 	"github.com/hashicorp/terraform-plugin-framework/types"
 	"github.com/prefecthq/terraform-provider-prefect/internal/api"
+	"github.com/prefecthq/terraform-provider-prefect/internal/provider/customtypes"
 	"github.com/prefecthq/terraform-provider-prefect/internal/provider/helpers"
 )
 
@@ -18,6 +19,9 @@ type WorkerMetadataDataSource struct {
 }
 
 type WorkerMetadataDataSourceModel struct {
+	AccountID   customtypes.UUIDValue `tfsdk:"account_id"`
+	WorkspaceID customtypes.UUIDValue `tfsdk:"workspace_id"`
+
 	BaseJobConfigs types.Object `tfsdk:"base_job_configs"`
 }
 
@@ -58,6 +62,16 @@ Get metadata information about the common Worker types, such as Kubernetes, ECS,
 Use this data source to get the default base job configurations for those common Worker types.
 `,
 		Attributes: map[string]schema.Attribute{
+			"account_id": schema.StringAttribute{
+				CustomType:  customtypes.UUIDType{},
+				Description: "Account ID (UUID), defaults to the account set in the provider",
+				Optional:    true,
+			},
+			"workspace_id": schema.StringAttribute{
+				CustomType:  customtypes.UUIDType{},
+				Description: "Workspace ID (UUID), defaults to the workspace set in the provider",
+				Optional:    true,
+			},
 			"base_job_configs": schema.SingleNestedAttribute{
 				Computed:    true,
 				Description: "A map of default base job configurations (JSON) for each of the primary worker types",
@@ -87,6 +101,11 @@ Use this data source to get the default base job configurations for those common
 						Description: "Default base job configuration for Cloud Run workers",
 						CustomType:  jsontypes.NormalizedType{},
 					},
+					"cloud_run_v2": schema.StringAttribute{
+						Computed:    true,
+						Description: "Default base job configuration for Cloud Run V2 workers",
+						CustomType:  jsontypes.NormalizedType{},
+					},
 					"vertex_ai": schema.StringAttribute{
 						Computed:    true,
 						Description: "Default base job configuration for Vertex AI workers",
@@ -112,11 +131,21 @@ Use this data source to get the default base job configurations for those common
 						Description: "Default base job configuration for Cloud Run Push workers",
 						CustomType:  jsontypes.NormalizedType{},
 					},
+					"cloud_run_v2_push": schema.StringAttribute{
+						Computed:    true,
+						Description: "Default base job configuration for Cloud Run V2 Push workers",
+						CustomType:  jsontypes.NormalizedType{},
+					},
 					"ecs_push": schema.StringAttribute{
 						Computed:    true,
 						Description: "Default base job configuration for ECS Push workers",
 						CustomType:  jsontypes.NormalizedType{},
 					},
+					"modal_push": schema.StringAttribute{
+						Computed:    true,
+						Description: "Default base job configuration for Modal Push workers",
+						CustomType:  jsontypes.NormalizedType{},
+					},
 				},
 			},
 		},
@@ -133,7 +162,7 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read
 		return
 	}
 
-	client, err := d.client.Collections()
+	client, err := d.client.Collections(model.AccountID.ValueUUID(), model.WorkspaceID.ValueUUID())
 	if err != nil {
 		resp.Diagnostics.Append(helpers.CreateClientErrorDiagnostic("Collections", err))
 
@@ -157,31 +186,38 @@ func (d *WorkerMetadataDataSource) Read(ctx context.Context, req datasource.Read
 	}
 
 	// https://developer.hashicorp.com/terraform/plugin/framework/handling-data/types/object#setting-values
+	// https://docs.prefect.io/latest/deploy/infrastructure-concepts/work-pools#work-pool-types
 	attributeTypes := map[string]attr.Type{
 		"kubernetes":                     jsontypes.NormalizedType{},
 		"ecs":                            jsontypes.NormalizedType{},
 		"azure_container_instances":      jsontypes.NormalizedType{},
 		"docker":                         jsontypes.NormalizedType{},
 		"cloud_run":                      jsontypes.NormalizedType{},
+		"cloud_run_v2":                   jsontypes.NormalizedType{},
 		"vertex_ai":                      jsontypes.NormalizedType{},
 		"prefect_agent":                  jsontypes.NormalizedType{},
 		"process":                        jsontypes.NormalizedType{},
 		"azure_container_instances_push": jsontypes.NormalizedType{},
 		"cloud_run_push":                 jsontypes.NormalizedType{},
+		"cloud_run_v2_push":              jsontypes.NormalizedType{},
 		"ecs_push":                       jsontypes.NormalizedType{},
+		"modal_push":                     jsontypes.NormalizedType{},
 	}
 	attributeValues := map[string]attr.Value{
 		"kubernetes":                     jsontypes.NewNormalizedValue(string(remap["kubernetes"])),
 		"ecs":                            jsontypes.NewNormalizedValue(string(remap["ecs"])),
 		"azure_container_instances":      jsontypes.NewNormalizedValue(string(remap["azure-container-instance"])),
 		"docker":                         jsontypes.NewNormalizedValue(string(remap["docker"])),
 		"cloud_run":                      jsontypes.NewNormalizedValue(string(remap["cloud-run"])),
+		"cloud_run_v2":                   jsontypes.NewNormalizedValue(string(remap["cloud-run-v2"])),
 		"vertex_ai":                      jsontypes.NewNormalizedValue(string(remap["vertex-ai"])),
 		"prefect_agent":                  jsontypes.NewNormalizedValue(string(remap["prefect-agent"])),
 		"process":                        jsontypes.NewNormalizedValue(string(remap["process"])),
 		"azure_container_instances_push": jsontypes.NewNormalizedValue(string(remap["azure-container-instance:push"])),
 		"cloud_run_push":                 jsontypes.NewNormalizedValue(string(remap["cloud-run:push"])),
+		"cloud_run_v2_push":              jsontypes.NewNormalizedValue(string(remap["cloud-run-v2:push"])),
 		"ecs_push":                       jsontypes.NewNormalizedValue(string(remap["ecs:push"])),
+		"modal_push":                     jsontypes.NewNormalizedValue(string(remap["modal:push"])),
 	}
 
 	obj, diag := types.ObjectValue(attributeTypes, attributeValues)

diff --git a/internal/provider/datasources/worker_metadata_test.go b/internal/provider/datasources/worker_metadata_test.go
@@ -1,16 +1,27 @@
 package datasources_test
 
 import (
+	"fmt"
+	"os"
 	"testing"
 
 	"github.com/hashicorp/terraform-plugin-testing/helper/resource"
 	"github.com/prefecthq/terraform-provider-prefect/internal/testutils"
 )
 
 func fixtureAccWorkerMetadtata() string {
-	return `
-data "prefect_worker_metadata" "default" {}
-`
+	aID := os.Getenv("PREFECT_CLOUD_ACCOUNT_ID")
+
+	return fmt.Sprintf(`
+data "prefect_workspace" "evergreen" {
+	handle = "github-ci-tests"
+}
+
+data "prefect_worker_metadata" "default" {
+  account_id = "%s"
+  workspace_id = data.prefect_workspace.evergreen.id
+}
+`, aID)
 }
 
 //nolint:paralleltest // we use the resource.ParallelTest helper instead
@@ -24,17 +35,20 @@ func TestAccDatasource_worker_metadata(t *testing.T) {
 			{
 				Config: fixtureAccWorkerMetadtata(),
 				Check: resource.ComposeAggregateTestCheckFunc(
-					resource.TestCheckResourceAttr(datasourceName, "base_job_configs.%", "11"),
+					resource.TestCheckResourceAttr(datasourceName, "base_job_configs.%", "14"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.kubernetes"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.ecs"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.azure_container_instances"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.docker"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run"),
+					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_v2"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.vertex_ai"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.prefect_agent"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.process"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.azure_container_instances_push"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_push"),
+					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.cloud_run_v2_push"),
+					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.modal_push"),
 					resource.TestCheckResourceAttrSet(datasourceName, "base_job_configs.ecs_push"),
 				),
 			},