source tfplan feature complete

cmd/terraform-operator/stateSourcePending.go

@@ -95,23 +95,52 @@ func getSourceData(parent *tftype.Terraform, desiredChildren *[]interface{}, pod
 			gcsObjects = append(gcsObjects, source.GCS)
 		}
 
-		if source.TFApply != "" {
+		if source.TFApply != "" || source.TFPlan != "" {
+
 			if source.TFApply == parent.Name && parent.Kind == "TerraformApply" {
 				return sourceData, fmt.Errorf("Circular reference to TerraformApply source[%d]: %s", i, source.TFApply)
 			}
 
-			myLog(parent, "INFO", fmt.Sprintf("Including TerraformApply source[%d]: %s", i, source.TFApply))
-			tfapply, err := getTerraformApply(parent.Namespace, source.TFApply)
-			if err != nil {
-				return sourceData, fmt.Errorf("Waiting for source TerraformApply: %s", source.TFApply)
+			if source.TFPlan == parent.Name && parent.Kind == "TerraformPlan" {
+				return sourceData, fmt.Errorf("Circular reference to TerraformPlan source[%d]: %s", i, source.TFPlan)
+			}
+
+			sourceKind := "TerraformApply"
+			sourceName := source.TFApply
+
+			var tf tftype.Terraform
+
+			tfapply, tfapplyErr := getTerraform("tfapply", parent.Namespace, source.TFApply)
+			tfplan, tfplanErr := getTerraform("tfplan", parent.Namespace, source.TFPlan)
+
+			if source.TFApply != "" && source.TFPlan != "" && tfapplyErr != nil && tfplanErr != nil {
+				// no source available yet.
+				return sourceData, fmt.Errorf("Waiting for either source TerraformPlan: '%s', or source TerraformApply: '%s'", source.TFPlan, source.TFApply)
+			}
+
+			if tfapplyErr == nil {
+				// Prefer tfapply if both were specified.
+				tf = tfapply
+				myLog(parent, "INFO", fmt.Sprintf("Including %s source[%d]: %s", sourceKind, i, source.TFApply))
+			} else if tfplanErr == nil {
+				tf = tfplan
+				sourceKind = "TerraformPlan"
+				sourceName = source.TFPlan
+				myLog(parent, "INFO", fmt.Sprintf("Including %s source[%d]: %s", sourceKind, i, source.TFPlan))
+			} else {
+				if source.TFPlan != "" {
+					return sourceData, fmt.Errorf("Waiting for source TerraformPlan: %s", source.TFPlan)
+				} else {
+					return sourceData, fmt.Errorf("Waiting for source TerraformApply: %s", source.TFApply)
+				}
 			}
 
 			// ConfigMaps generated from embedded source.
-			for _, configMapName := range tfapply.Status.Sources.EmbeddedConfigMaps {
-				configMapData, err := getConfigMapSourceData(tfapply.ObjectMeta.Namespace, configMapName)
+			for _, configMapName := range tf.Status.Sources.EmbeddedConfigMaps {
+				configMapData, err := getConfigMapSourceData(tf.ObjectMeta.Namespace, configMapName)
 				if err != nil {
 					// Wait for configmap to become available.
-					return sourceData, fmt.Errorf("Waiting for TerraformApply %s source embedded ConfigMap: %s", source.TFApply, configMapName)
+					return sourceData, fmt.Errorf("Waiting for %s %s source embedded ConfigMap: %s", sourceKind, sourceName, configMapName)
 				}
 
 				configMapHash, err := toSha1(configMapData)
@@ -126,24 +155,24 @@ func getSourceData(parent *tftype.Terraform, desiredChildren *[]interface{}, pod
 					configMapKeys = append(configMapKeys, tuple)
 				}
 
-				myLog(parent, "INFO", fmt.Sprintf("Including TerraformApply %s embedded ConfigMap source with %d keys: %s", source.TFApply, len(configMapData), configMapName))
+				myLog(parent, "INFO", fmt.Sprintf("Including %s %s embedded ConfigMap source with %d keys: %s", sourceKind, sourceName, len(configMapData), configMapName))
 			}
 
-			for j, tfsource := range tfapply.Spec.Sources {
+			for j, tfsource := range tf.Spec.Sources {
 
 				// ConfigMap source
 				if tfsource.ConfigMap.Name != "" {
 					configMapName := tfsource.ConfigMap.Name
 
-					configMapData, err := getConfigMapSourceData(tfapply.ObjectMeta.Namespace, configMapName)
+					configMapData, err := getConfigMapSourceData(tf.ObjectMeta.Namespace, configMapName)
 					if err != nil {
 						// Wait for configmap to become available.
-						return sourceData, fmt.Errorf("Waiting for TerraformApply %s source ConfigMap: %s", source.TFApply, configMapName)
+						return sourceData, fmt.Errorf("Waiting for %s %s source ConfigMap: %s", sourceKind, sourceName, configMapName)
 					}
 
 					err = validateConfigMapSource(configMapData)
 					if err != nil {
-						return sourceData, fmt.Errorf("TerraformApply %s ConfigMap source %s data is invalid: %v", source.TFApply, configMapName, err)
+						return sourceData, fmt.Errorf("%s %s ConfigMap source %s data is invalid: %v", sourceKind, sourceName, configMapName, err)
 					}
 
 					configMapHash, err := toSha1(configMapData)
@@ -158,12 +187,12 @@ func getSourceData(parent *tftype.Terraform, desiredChildren *[]interface{}, pod
 						configMapKeys = append(configMapKeys, tuple)
 					}
 
-					myLog(parent, "INFO", fmt.Sprintf("Including TerraformApply %s ConfigMap source[%d] with %d keys: %s", source.TFApply, j, len(configMapData), configMapName))
+					myLog(parent, "INFO", fmt.Sprintf("Including %s %s ConfigMap source[%d] with %d keys: %s", sourceKind, sourceName, j, len(configMapData), configMapName))
 				}
 
 				// GCS source
 				if tfsource.GCS != "" {
-					myLog(parent, "INFO", fmt.Sprintf("Including TerraformApply %s GCS source[%d]: %s", source.TFApply, j, tfsource.GCS))
+					myLog(parent, "INFO", fmt.Sprintf("Including %s %s GCS source[%d]: %s", sourceKind, sourceName, j, tfsource.GCS))
 					gcsObjects = append(gcsObjects, tfsource.GCS)
 				}
 			}

cmd/terraform-operator/stateTFInputPending.go

@@ -14,7 +14,7 @@ func getTFInputs(parent *tftype.Terraform) (TerraformInputVars, error) {
 
 	// Wait for tfinputs
 	for _, tfinput := range parent.Spec.TFInputs {
-		tfapply, err := getTerraformApply(parent.ObjectMeta.Namespace, tfinput.Name)
+		tfapply, err := getTerraform("tfapply", parent.ObjectMeta.Namespace, tfinput.Name)
 		if err != nil {
 			return tfInputVars, fmt.Errorf("Waiting for TerraformApply/%s", tfinput.Name)
 		} else {
@@ -45,12 +45,12 @@ func getTFInputs(parent *tftype.Terraform) (TerraformInputVars, error) {
 	return tfInputVars, nil
 }
 
-func getTerraformApply(namespace string, name string) (tftype.Terraform, error) {
+func getTerraform(kind string, namespace string, name string) (tftype.Terraform, error) {
 	var tfapply tftype.Terraform
 	var stdout bytes.Buffer
 	var stderr bytes.Buffer
 
-	cmd := exec.Command("kubectl", "get", "tfapply", "-n", namespace, name, "-o", "yaml")
+	cmd := exec.Command("kubectl", "get", kind, "-n", namespace, name, "-o", "yaml")
 	cmd.Stdout = &stdout
 	cmd.Stderr = &stderr
 

cmd/terraform-operator/stateTFPlanPending.go

@@ -1,12 +1,9 @@
 package main
 
 import (
-	"bytes"
 	"fmt"
-	"os/exec"
 
 	tftype "github.com/danisla/terraform-operator/pkg/types"
-	"github.com/ghodss/yaml"
 )
 
 func getTFPlanFile(parent *tftype.Terraform) (string, error) {
@@ -18,7 +15,7 @@ func getTFPlanFile(parent *tftype.Terraform) (string, error) {
 
 	if parent.Spec.TFPlan != "" {
 
-		tfplan, err = getTerraformPlan(parent.ObjectMeta.Namespace, parent.Spec.TFPlan)
+		tfplan, err = getTerraform("tfplan", parent.ObjectMeta.Namespace, parent.Spec.TFPlan)
 		if err != nil {
 			return tfplanFile, fmt.Errorf("Waiting for TerraformPlan/%s", parent.Spec.TFPlan)
 		}
@@ -34,22 +31,3 @@ func getTFPlanFile(parent *tftype.Terraform) (string, error) {
 
 	return tfplanFile, nil
 }
-
-func getTerraformPlan(namespace string, name string) (tftype.Terraform, error) {
-	var tfplan tftype.Terraform
-	var stdout bytes.Buffer
-	var stderr bytes.Buffer
-
-	cmd := exec.Command("kubectl", "get", "tfplan", "-n", namespace, name, "-o", "yaml")
-	cmd.Stdout = &stdout
-	cmd.Stderr = &stderr
-
-	err := cmd.Run()
-	if err != nil {
-		return tfplan, fmt.Errorf("Failed to run kubectl: %s\n%v", stderr.String(), err)
-	}
-
-	err = yaml.Unmarshal(stdout.Bytes(), &tfplan)
-
-	return tfplan, err
-}

cmd/terraform-operator/stateTFVarsFromPending.go

@@ -13,7 +13,7 @@ func getTFVarsFrom(parent *tftype.Terraform) (TerraformInputVars, error) {
 		if varsFrom.TFApply != "" {
 			tfApplyName := varsFrom.TFApply
 
-			tfapply, err := getTerraformApply(parent.Namespace, tfApplyName)
+			tfapply, err := getTerraform("tfapply", parent.Namespace, tfApplyName)
 			if err != nil {
 				return tfVars, fmt.Errorf("Waiting for tfvarsFrom TerraformApply: %s", tfApplyName)
 			}

pkg/types/types.go

@@ -41,6 +41,7 @@ type TerraformConfigSource struct {
 	ConfigMap TerraformSourceConfigMap `json:"configMap,omitempty"`
 	Embedded  string                   `json:"embedded,omitempty"`
 	GCS       string                   `json:"gcs,omitempty"`
+	TFPlan    string                   `json:"tfplan,omitempty"`
 	TFApply   string                   `json:"tfapply,omitempty"`
 }
 

test.mk

@@ -1,6 +1,6 @@
 TEST_PLAN_ARTIFACTS := job1-cm.yaml job1-cm-tfplan.yaml job2-src-tfplan.yaml job3-cm-tfplan-inputs.yaml job4-gcs-tfplan.yaml job5-src-b64-tfplan.yaml
 TEST_APPLY_ARTIFACTS := job1-cm.yaml job1-cm-tfapply.yaml job1-cm-tfapply-tfplan.yaml job2-src-tfapply.yaml job4-gcs-tfapply.yaml
-TEST_DESTROY_ARTIFACTS := job1-cm.yaml job1-cm-tfdestroy.yaml job2-src-tfdestroy.yaml job4-gcs-tfdestroy.yaml
+TEST_DESTROY_ARTIFACTS := job1-cm.yaml job1-cm-tfdestroy.yaml job2-src-tfdestroy.yaml job4-gcs-tfdestroy.yaml job2-tfplan-tfdestroy.yaml job2-tfapply-tfdestroy.yaml job2-tfplan-tfapply-tfdestroy.yaml
 
 IMAGE := "gcr.io/cloud-solutions-group/terraform-pod:latest"
 
@@ -155,6 +155,26 @@ spec:
     region: us-central1
 endef
 
+define TEST_JOB_TF_SRC
+apiVersion: ctl.isla.solutions/v1
+kind: {{KIND}}
+metadata:
+  name: {{NAME}}
+spec:
+  image: {{IMAGE}}
+  imagePullPolicy: Always
+  backendBucket: {{BACKEND_BUCKET}}
+  backendPrefix: {{BACKEND_PREFIX}}
+  providerConfig:
+    google:
+      secretName: {{GOOGLE_PROVIDER_SECRET_NAME}}
+  sources:
+  {{TFPLAN_SRC}}
+  {{TFAPPLY_SRC}}
+  tfvars:
+    region: us-central1
+endef
+
 credentials: $(GOOGLE_CREDENTIALS_SA_KEY) project
 	kubectl create secret generic $(GOOGLE_PROVIDER_SECRET_NAME) --from-literal=GOOGLE_PROJECT=$(PROJECT) --from-file=GOOGLE_CREDENTIALS=$(GOOGLE_CREDENTIALS_SA_KEY)
 
@@ -229,7 +249,7 @@ tests/job%-cm-tfdestroy.yaml: backend_bucket
 export TEST_JOB_SRC
 tests/job%-src-tfplan.yaml: backend_bucket tests/main.tf
 	@mkdir -p tests
-	echo "$${TEST_JOB_SRC}" | \
+	@echo "$${TEST_JOB_SRC}" | \
 	sed -e "s/{{KIND}}/TerraformPlan/g" \
 	    -e "s/{{NAME}}/job$*/g" \
 	    -e "s|{{IMAGE}}|$(IMAGE)|g" \
@@ -354,6 +374,50 @@ tests/job%-gcs-tfdestroy.yaml: backend_bucket tests/job%-bundle.tgz
 
 ### END Tests with GCS tarball source ###
 
+### BEGIN Tests with tfplan or tfapply source ###
+export TEST_JOB_TF_SRC
+tests/job%-tfplan-tfdestroy.yaml: backend_bucket
+	@mkdir -p tests
+	@echo "$${TEST_JOB_TF_SRC}" | \
+	sed -e "s/{{KIND}}/TerraformDestroy/g" \
+	    -e "s/{{NAME}}/job$*/g" \
+	    -e "s|{{IMAGE}}|$(IMAGE)|g" \
+	    -e "s/{{BACKEND_BUCKET}}/$(BACKEND_BUCKET)/g" \
+	    -e "s/{{BACKEND_PREFIX}}/terraform/g" \
+	    -e "s/{{GOOGLE_PROVIDER_SECRET_NAME}}/$(GOOGLE_PROVIDER_SECRET_NAME)/g" \
+	    -e "s/{{TFPLAN_SRC}}/- tfplan: job$*/g" \
+			-e "s/{{TFAPPLY_SRC}}//g" \
+	> $@
+
+export TEST_JOB_TF_SRC
+tests/job%-tfplan-tfapply-tfdestroy.yaml: backend_bucket
+	@mkdir -p tests
+	@echo "$${TEST_JOB_TF_SRC}" | \
+	sed -e "s/{{KIND}}/TerraformDestroy/g" \
+	    -e "s/{{NAME}}/job$*/g" \
+	    -e "s|{{IMAGE}}|$(IMAGE)|g" \
+	    -e "s/{{BACKEND_BUCKET}}/$(BACKEND_BUCKET)/g" \
+	    -e "s/{{BACKEND_PREFIX}}/terraform/g" \
+	    -e "s/{{GOOGLE_PROVIDER_SECRET_NAME}}/$(GOOGLE_PROVIDER_SECRET_NAME)/g" \
+	    -e "s/{{TFPLAN_SRC}}/- tfplan: job$*/g" \
+	    -e "s/{{TFAPPLY_SRC}}/  tfapply: job$*/g" \
+	> $@
+
+export TEST_JOB_TF_SRC
+tests/job%-tfapply-tfdestroy.yaml: backend_bucket
+	@mkdir -p tests
+	@echo "$${TEST_JOB_TF_SRC}" | \
+	sed -e "s/{{KIND}}/TerraformDestroy/g" \
+	    -e "s/{{NAME}}/job$*/g" \
+	    -e "s|{{IMAGE}}|$(IMAGE)|g" \
+	    -e "s/{{BACKEND_BUCKET}}/$(BACKEND_BUCKET)/g" \
+	    -e "s/{{BACKEND_PREFIX}}/terraform/g" \
+	    -e "s/{{GOOGLE_PROVIDER_SECRET_NAME}}/$(GOOGLE_PROVIDER_SECRET_NAME)/g" \
+	    -e "s/{{TFPLAN_SRC}}//g" \
+			-e "s/{{TFAPPLY_SRC}}/- tfapply: job$*/g" \
+	> $@
+### END Tests with tfplan or tfapply source ###
+
 test-artifacts: $(addprefix tests/,$(TEST_ARTIFACTS))
 
 test: $(addprefix tests/,$(TEST_PLAN_ARTIFACTS))