Merge pull request #785 from cloudflare/skip-known-incomplete-resources

internal/app/cf-terraforming/cmd/generate.go

@@ -174,7 +174,9 @@ func generateResources() func(cmd *cobra.Command, args []string) {
 					}).Debug("no result found")
 					continue
 				}
-				err = json.Unmarshal([]byte(value.String()), &jsonStructData)
+
+				modifiedJSON := modifyResponsePayload(resourceType, value)
+				err = json.Unmarshal([]byte(modifiedJSON), &jsonStructData)
 				if err != nil {
 					log.Fatalf("failed to unmarshal result: %s", err)
 				}
@@ -1418,7 +1420,7 @@ func generateResources() func(cmd *cobra.Command, args []string) {
 			log.WithFields(logrus.Fields{
 				"count":    resourceCount,
 				"resource": resourceType,
-			}).Debug("found resources to generate output for")
+			}).Debug("generating resource output")
 
 			// If we don't have any resources to generate, just bail out early.
 			if resourceCount == 0 {

internal/app/cf-terraforming/cmd/import.go

@@ -20,6 +20,7 @@ import (
 	"github.com/hashicorp/hc-install/releases"
 	"github.com/hashicorp/hcl/v2/hclwrite"
 	"github.com/hashicorp/terraform-exec/tfexec"
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/tidwall/gjson"
@@ -121,7 +122,9 @@ func runImport() func(cmd *cobra.Command, args []string) {
 
 		_, providerVersion, err := tf.Version(context.Background(), true)
 		providerVersionString = providerVersion[providerRegistryHostname+"/cloudflare/cloudflare"].String()
-		log.Debugf("detected provider version: %s", providerVersionString)
+		log.WithFields(logrus.Fields{
+			"version": providerVersionString,
+		}).Debug("detected provider")
 
 		var jsonStructData []interface{}
 
@@ -152,14 +155,27 @@ func runImport() func(cmd *cobra.Command, args []string) {
 				placeholderReplacer := strings.NewReplacer("{account_id}", accountID, "{zone_id}", zoneID)
 				endpoint = placeholderReplacer.Replace(endpoint)
 
+				if strings.Contains(endpoint, "{") {
+					log.WithFields(logrus.Fields{
+						"resource": resourceType,
+						"endpoint": endpoint,
+					}).Debug("failed to substitute all path placeholders due to unknown parameters")
+
+					continue
+				}
+
 				client := cloudflare.NewClient()
 
 				err := client.Get(context.Background(), endpoint, nil, &result)
 				if err != nil {
 					var apierr *cloudflare.Error
 					if errors.As(err, &apierr) {
 						if apierr.StatusCode == http.StatusNotFound {
-							log.Debugf("no resources found at %s. skipping...", endpoint)
+							log.WithFields(logrus.Fields{
+								"resource": resourceType,
+								"endpoint": endpoint,
+							}).Debug("no resources found")
+
 							continue
 						}
 					}
@@ -173,7 +189,11 @@ func runImport() func(cmd *cobra.Command, args []string) {
 
 				value := gjson.Get(string(body), "result")
 				if value.Type == gjson.Null {
-					log.Debugf("no result found at %s. skipping...", endpoint)
+					log.WithFields(logrus.Fields{
+						"resource": resourceType,
+						"endpoint": endpoint,
+					}).Debug("no result found")
+
 					continue
 				}
 				err = json.Unmarshal([]byte(value.String()), &jsonStructData)

internal/app/cf-terraforming/cmd/resource_to_endpoint_mapping.go

@@ -175,10 +175,6 @@ var resourceToEndpoint = map[string]map[string]string{
 		"list": "/zones/{zone_id}/firewall/lockdowns",
 		"get":  "/zones/{zone_id}/firewall/lockdowns/{lock_downs_id}",
 	},
-	"cloudflare_firewall_rule": {
-		"list": "/zones/{zone_id}/firewall/rules",
-		"get":  "/zones/{zone_id}/firewall/rules/{rule_id}",
-	},
 	"cloudflare_access_rule": {
 		"list": "/{account_or_zone}/{account_or_zone_id}/firewall/access_rules/rules",
 		"get":  "/{account_or_zone}/{account_or_zone_id}/firewall/access_rules/rules/{rule_id}",
@@ -243,10 +239,6 @@ var resourceToEndpoint = map[string]map[string]string{
 		"list": "/zones/{zone_id}/web3/hostnames",
 		"get":  "/zones/{zone_id}/web3/hostnames/{identifier}",
 	},
-	"cloudflare_workers_script": {
-		"list": "/accounts/{account_id}/workers/scripts",
-		"get":  "/accounts/{account_id}/workers/scripts/{script_name}",
-	},
 	"cloudflare_workers_script_subdomain": {
 		"list": "",
 		"get":  "/accounts/{account_id}/workers/scripts/{script_name}/subdomain",
@@ -691,10 +683,6 @@ var resourceToEndpoint = map[string]map[string]string{
 		"list": "",
 		"get":  "/accounts/{account_identifier}/cloudforce-one/requests/{request_identifier}/asset/{asset_identifer}",
 	},
-	"cloudflare_cloud_connector_rules": {
-		"list": "/zones/{zone_id}/cloud_connector/rules",
-		"get":  "",
-	},
 	"cloudflare_leaked_credential_check": {
 		"list": "",
 		"get":  "/zones/{zone_id}/leaked-credential-checks",

internal/app/cf-terraforming/cmd/util.go

@@ -16,6 +16,7 @@ import (
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	"github.com/tidwall/gjson"
 	"github.com/zclconf/go-cty/cty"
 )
 
@@ -318,3 +319,10 @@ func boolToEnabledOrDisabled(value bool) string {
 	}
 	return "disabled"
 }
+
+// modifyResponsePayload takes the current resource and the `gjson.Result`
+// to run arbitary modifications to the JSON before passing it to be overlayed
+// the provider schema.
+func modifyResponsePayload(resourceName string, value gjson.Result) string {
+	return value.String()
+}

scripts/build-resource-to-endpoint-mapping

@@ -11,6 +11,20 @@ config = YAML.load_file(ARGV[0])
 mappingPath = "internal/app/cf-terraforming/cmd/resource_to_endpoint_mapping.go"
 output = ""
 
+def skipped_resource?(terraform_name)
+  [
+    # deprecated and backend service support automatic migrations
+    "firewall_rule",
+    "firewall_filter",
+
+    # doesn't work with outer wrap parameters
+    "cloud_connector_rules",
+
+    # terraform can't get the content (may be bundled, etc)
+    "workers_script"
+  ].include?(terraform_name)
+end
+
 def terraform_name(v)
   if v.is_a?(Hash)
     # don't generate if we've explicitly disabled the resource
@@ -80,13 +94,12 @@ def build_output(output, config)
   terraform_name = terraform_name(config)
   endpoint = list_endpoint(config)
 
-  if !terraform_name.nil? && !endpoint.nil?
+  if !terraform_name.nil? && !endpoint.nil? && !skipped_resource?(terraform_name)
     output << "
     \"cloudflare_#{terraform_name}\": map[string]string{
       \"list\":\"#{endpoint[:list]}\",
       \"get\":\"#{endpoint[:get]}\",
     },"
-    # output << ": \"\",\n"
   end
 
   return if config["subresources"].nil?