diff --git a/Dockerfile b/Dockerfile index 74eb9d74..a2fe7c8d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # Build the manager binary -FROM golang:1.13 as builder +FROM golang:1.16 as builder WORKDIR /workspace # Copy the Go Modules manifests @@ -19,9 +19,14 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -a -o manager # Use distroless as minimal base image to package the manager binary # Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/static:nonroot +FROM golang:1.16 WORKDIR / COPY --from=builder /workspace/manager . -USER nonroot:nonroot +#USER nonroot:nonroot + +# COPY terraform binary +COPY bin/terraform /usr/bin/terraform +#RUN chmod +x /usr/bin/terraform +RUN apt-get install git ENTRYPOINT ["/manager"] diff --git a/Makefile b/Makefile index 39bc1fda..5d8ec356 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ # Image URL to use all building/pushing image targets -IMG ?= oamdev/terraform-controller:0.1.17 +IMG ?= oamdev/terraform-controller:0.1.19 # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true" @@ -42,7 +42,7 @@ deploy: manifests # Generate manifests e.g. CRD, RBAC etc. manifests: controller-gen $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=tf-api-role webhook paths="./..." output:crd:artifacts:config=chart/crds - mv config/rbac/role.yaml chart/templates/tf_api_role.yaml + # mv config/rbac/role.yaml chart/templates/tf_api_role.yaml # Run go fmt against code fmt: goimports diff --git a/controllers/configuration/terraform/linux/terraform b/bin/terraform similarity index 100% rename from controllers/configuration/terraform/linux/terraform rename to bin/terraform diff --git a/chart/Chart.yaml b/chart/Chart.yaml index ab1a741c..36826d72 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 name: terraform-controller -version: 0.1.17 +version: 0.1.19 description: A Kubernetes Terraform controller home: https://github.com/oam-dev/terraform-controller diff --git a/chart/templates/read_provider_creds_role.yml b/chart/templates/read_provider_creds_role.yml index 842bb3d2..f66bda63 100644 --- a/chart/templates/read_provider_creds_role.yml +++ b/chart/templates/read_provider_creds_role.yml @@ -2,6 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: read-provider-creds + namespace: {{ .Release.Namespace }} rules: - apiGroups: - "" diff --git a/chart/templates/tf_api_role.yaml b/chart/templates/tf_api_role.yaml index 22aa2132..fad44546 100644 --- a/chart/templates/tf_api_role.yaml +++ b/chart/templates/tf_api_role.yaml @@ -1,48 +1,67 @@ - --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: creationTimestamp: null name: tf-api-role + namespace: {{ .Release.Namespace }} rules: -- apiGroups: - - terraform.core.oam.dev - resources: - - configurations - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - terraform.core.oam.dev - resources: - - configurations/status - verbs: - - get - - patch - - update -- apiGroups: - - terraform.core.oam.dev - resources: - - providers - verbs: - - create - - delete - - get - - list - - patch - - update - - watch -- apiGroups: - - terraform.core.oam.dev - resources: - - providers/status - verbs: - - get - - patch - - update + - apiGroups: + - terraform.core.oam.dev + resources: + - configurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - terraform.core.oam.dev + resources: + - configurations/status + verbs: + - get + - patch + - update + - apiGroups: + - terraform.core.oam.dev + resources: + - providers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch + - apiGroups: + - terraform.core.oam.dev + resources: + - providers/status + verbs: + - get + - patch + - update + - apiGroups: + - "" + resources: + - "secrets" + verbs: + - "get" + - "list" + - "create" + - "update" + - "delete" + - apiGroups: + - "coordination.k8s.io" + resources: + - "leases" + verbs: + - "create" + - "update" + - "get" + - "delete" diff --git a/chart/templates/tf_api_role_binding.yaml b/chart/templates/tf_api_role_binding.yaml index cdba3f0c..b6bd5c61 100644 --- a/chart/templates/tf_api_role_binding.yaml +++ b/chart/templates/tf_api_role_binding.yaml @@ -9,4 +9,4 @@ roleRef: subjects: - kind: ServiceAccount name: tf-controller-service-account - namespace: {{.Release.Namespace}} + namespace: {{ .Release.Namespace }} diff --git a/chart/templates/tf_controller_clusterrole.yml b/chart/templates/tf_controller_clusterrole.yml index f4a36bf2..a5571fec 100644 --- a/chart/templates/tf_controller_clusterrole.yml +++ b/chart/templates/tf_controller_clusterrole.yml @@ -2,6 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: tf-controller-clusterrole + namespace: {{ .Release.Namespace }} rules: - apiGroups: - "" @@ -20,6 +21,7 @@ rules: - "create" - "update" - "watch" + - "delete" - apiGroups: - "batch" resources: diff --git a/chart/templates/tf_controller_role.yaml b/chart/templates/tf_controller_role.yaml index 38ca6f31..6634e07b 100644 --- a/chart/templates/tf_controller_role.yaml +++ b/chart/templates/tf_controller_role.yaml @@ -2,6 +2,7 @@ apiVersion: rbac.authorization.k8s.io/v1 kind: Role metadata: name: tf-controller-role + namespace: {{ .Release.Namespace }} rules: - apiGroups: - "" diff --git a/chart/templates/tf_executor_role.yml b/chart/templates/tf_executor_role.yml index 3ca58320..ecbaf92c 100644 --- a/chart/templates/tf_executor_role.yml +++ b/chart/templates/tf_executor_role.yml @@ -14,6 +14,7 @@ rules: - "list" - "create" - "update" + - "delete" - apiGroups: - "coordination.k8s.io" resources: diff --git a/chart/values.yaml b/chart/values.yaml index 5fa6eb57..299a70c4 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -1,7 +1,7 @@ replicaCount: 1 -version: 0.1.17 +version: 0.1.19 image: - name: oamdev/terraform-controller:0.1.17 + name: oamdev/terraform-controller:0.1.19 imagePullPolicy: Always diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml new file mode 100644 index 00000000..22aa2132 --- /dev/null +++ b/config/rbac/role.yaml @@ -0,0 +1,48 @@ + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + creationTimestamp: null + name: tf-api-role +rules: +- apiGroups: + - terraform.core.oam.dev + resources: + - configurations + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - terraform.core.oam.dev + resources: + - configurations/status + verbs: + - get + - patch + - update +- apiGroups: + - terraform.core.oam.dev + resources: + - providers + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - terraform.core.oam.dev + resources: + - providers/status + verbs: + - get + - patch + - update diff --git a/controllers/configuration/configuration.go b/controllers/configuration/configuration.go index 8fe524ff..21cf478e 100644 --- a/controllers/configuration/configuration.go +++ b/controllers/configuration/configuration.go @@ -4,18 +4,17 @@ import ( "fmt" "os" "os/exec" - "path/filepath" - "runtime" "strings" "github.com/google/go-cmp/cmp" "github.com/google/go-cmp/cmp/cmpopts" - "github.com/oam-dev/terraform-controller/controllers/util" "github.com/pkg/errors" v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" "github.com/oam-dev/terraform-controller/api/types" "github.com/oam-dev/terraform-controller/api/v1beta1" + "github.com/oam-dev/terraform-controller/controllers/util" ) // ValidConfigurationObject will validate a Configuration @@ -35,14 +34,11 @@ func ValidConfigurationObject(configuration *v1beta1.Configuration) (types.Confi return "", nil } -// ComposeConfiguration will compose the Terraform configuration with hcl/json and backend -// and will also check whether configuration is changed -func ComposeConfiguration(configuration *v1beta1.Configuration, controllerNamespace string, - configurationType types.ConfigurationType, cm *v1.ConfigMap) (string, bool, error) { - var configurationChanged bool +// RenderConfiguration will compose the Terraform configuration with hcl/json and backend +func RenderConfiguration(configuration *v1beta1.Configuration, controllerNamespace string, configurationType types.ConfigurationType) (string, error) { switch configurationType { case types.ConfigurationJSON: - return configuration.Spec.JSON, configurationChanged, nil + return configuration.Spec.JSON, nil case types.ConfigurationHCL: if configuration.Spec.Backend != nil { if configuration.Spec.Backend.SecretSuffix == "" { @@ -57,23 +53,40 @@ func ComposeConfiguration(configuration *v1beta1.Configuration, controllerNamesp } backendTF, err := util.RenderTemplate(configuration.Spec.Backend, controllerNamespace) if err != nil { - return "", configurationChanged, errors.Wrap(err, "failed to prepare Terraform backend configuration") + return "", errors.Wrap(err, "failed to prepare Terraform backend configuration") } completedConfiguration := configuration.Spec.HCL + "\n" + backendTF + return completedConfiguration, nil + + } + return "", errors.New("unknown issue") +} + +// CheckWhetherConfigurationChanges will check whether configuration is changed +func CheckWhetherConfigurationChanges(configurationType types.ConfigurationType, cm *v1.ConfigMap, completedConfiguration string) (bool, error) { + var configurationChanged bool + switch configurationType { + case types.ConfigurationJSON: + return configurationChanged, nil + case types.ConfigurationHCL: if cm != nil { configurationChanged = cm.Data[types.TerraformHCLConfigurationName] != completedConfiguration + if configurationChanged { + klog.InfoS("Configuration HCL changed", "ConfigMap", cm.Data[types.TerraformHCLConfigurationName], + "RenderedCompletedConfiguration", completedConfiguration) + } } else { // If the ConfigMap doesn't exist, we can surely say the configuration hcl/json changed configurationChanged = true } - return completedConfiguration, configurationChanged, nil + return configurationChanged, nil } - return "", configurationChanged, errors.New("unknown issue") + return configurationChanged, errors.New("unknown issue") } // CompareTwoContainerEnvs compares two slices of v1.EnvVar @@ -85,24 +98,38 @@ func CompareTwoContainerEnvs(s1 []v1.EnvVar, s2 []v1.EnvVar) bool { } // checkTerraformSyntax checks the syntax error for a HCL/JSON configuration -func checkTerraformSyntax(configuration string) error { - abs, _ := filepath.Abs(".") - var terraform = "terraform/darwin/terraform" - switch runtime.GOOS { - case "linux": - terraform = "terraform/linux/terraform" - case "windows": - terraform = "terraform/windows/terraform.exe" +func checkTerraformSyntax(name, configuration string) error { + klog.InfoS("About to check the syntax issue", "configuration", configuration) + dir, osErr := os.MkdirTemp("", fmt.Sprintf("tf-validate-%s-", name)) + if osErr != nil { + klog.ErrorS(osErr, "Failed to create folder", "Dir", dir) + return osErr } - terraform = filepath.Join(abs, "controllers", "configuration", terraform) - dir, _ := os.MkdirTemp(".", "tf-validate-") + klog.InfoS("Validate dir", "Dir", dir) defer os.RemoveAll(dir) //nolint:errcheck tfFile := fmt.Sprintf("%s/main.tf", dir) - if err := os.WriteFile(tfFile, []byte(configuration), 0400); err != nil { + if err := os.WriteFile(tfFile, []byte(configuration), 0777); err != nil { //nolint + klog.ErrorS(err, "Failed to write Configuration hcl to main.tf", "HCL", configuration) + return err + } + if err := os.Chdir(dir); err != nil { + klog.ErrorS(err, "Failed to change dir", "dir", dir) return err } - cmd := fmt.Sprintf("cd %s && %s init && %s validate", dir, terraform, terraform) - output, _ := exec.Command("bash", "-c", cmd).CombinedOutput() //nolint:gosec + + var ( + output []byte + err error + ) + output, err = exec.Command("terraform", "init").CombinedOutput() + if err != nil { + klog.ErrorS(err, "The command execution isn't successful", "cmd", "terraform init", "output", string(output)) + } else { + output, err = exec.Command("terraform", "validate").CombinedOutput() + if err != nil { + klog.ErrorS(err, "The command execution isn't successful", "cmd", "terraform validate", "output", string(output)) + } + } if strings.Contains(string(output), "Success!") { return nil } @@ -118,5 +145,5 @@ func CheckConfigurationSyntax(configuration *v1beta1.Configuration, configuratio case types.ConfigurationJSON: template = configuration.Spec.JSON } - return checkTerraformSyntax(template) + return checkTerraformSyntax(configuration.Name, template) } diff --git a/controllers/configuration/configuration_test.go b/controllers/configuration/configuration_test.go index c9389595..c9539ac4 100644 --- a/controllers/configuration/configuration_test.go +++ b/controllers/configuration/configuration_test.go @@ -78,10 +78,12 @@ func TestCompareTwoContainerEnvs(t *testing.T) { func TestCheckTFConfiguration(t *testing.T) { cases := map[string]struct { + name string configuration string subStr string }{ "Invalid": { + name: "bad", configuration: `resource2 "alicloud_oss_bucket" "bucket-acl" { bucket = var.bucket acl = var.acl @@ -106,6 +108,7 @@ variable "acl" { subStr: "Error:", }, "valid": { + name: "good", configuration: `resource "alicloud_oss_bucket" "bucket-acl" { bucket = var.bucket acl = var.acl @@ -134,7 +137,7 @@ variable "acl" { os.Chdir("../../") for name, tc := range cases { t.Run(name, func(t *testing.T) { - err := checkTerraformSyntax(tc.configuration) + err := checkTerraformSyntax(tc.name, tc.configuration) if err != nil { if !strings.Contains(err.Error(), tc.subStr) { t.Errorf("\ncheckTFConfiguration(...) %s\n", cmp.Diff(err.Error(), tc.subStr)) diff --git a/controllers/configuration/terraform/darwin/terraform b/controllers/configuration/terraform/darwin/terraform deleted file mode 100755 index 7783b3fc..00000000 Binary files a/controllers/configuration/terraform/darwin/terraform and /dev/null differ diff --git a/controllers/configuration/terraform/windows/terraform.exe b/controllers/configuration/terraform/windows/terraform.exe deleted file mode 100755 index 123f4384..00000000 Binary files a/controllers/configuration/terraform/windows/terraform.exe and /dev/null differ diff --git a/controllers/configuration_controller.go b/controllers/configuration_controller.go index aa61f7fb..9601f115 100644 --- a/controllers/configuration_controller.go +++ b/controllers/configuration_controller.go @@ -31,7 +31,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/klog/v2" - "k8s.io/utils/pointer" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -136,23 +135,23 @@ func (r *ConfigurationReconciler) Reconcile(req ctrl.Request) (ctrl.Result, erro } } - configurationChanged, err := r.preCheck(ctx, req.Namespace, &configuration, tfInputConfigMapsName) + configurationChanged, err := r.preCheck(ctx, &configuration, tfInputConfigMapsName) if err != nil { return ctrl.Result{}, err } if !configuration.ObjectMeta.DeletionTimestamp.IsZero() { // terraform destroy - if controllerutil.ContainsFinalizer(&configuration, configurationFinalizer) { - klog.InfoS("performing Terraform Destroy", "Namespace", req.Namespace, "Name", req.Name) - destroyJobName := req.Name + "-" + string(TerraformDestroy) - klog.InfoS("Terraform destroy job", "Namespace", req.Namespace, "Name", destroyJobName) - if err := r.terraformDestroy(ctx, req.Namespace, configuration, configurationChanged, destroyJobName, tfInputConfigMapsName, applyJobName); err != nil { - if err.Error() == MessageDestroyJobNotCompleted { - return ctrl.Result{RequeueAfter: 3 * time.Second}, nil - } - return ctrl.Result{RequeueAfter: 3 * time.Second}, errors.Wrap(err, "continue reconciling to destroy cloud resource") + klog.InfoS("Performing Configuration Destroy", "Namespace", req.Namespace, "Name", req.Name) + destroyJobName := req.Name + "-" + string(TerraformDestroy) + klog.InfoS("Terraform destroy job", "Namespace", req.Namespace, "Name", destroyJobName) + if err := r.terraformDestroy(ctx, req.Namespace, configuration, configurationChanged, destroyJobName, tfInputConfigMapsName, applyJobName); err != nil { + if err.Error() == MessageDestroyJobNotCompleted { + return ctrl.Result{RequeueAfter: 3 * time.Second}, nil } + return ctrl.Result{RequeueAfter: 3 * time.Second}, errors.Wrap(err, "continue reconciling to destroy cloud resource") + } + if controllerutil.ContainsFinalizer(&configuration, configurationFinalizer) { controllerutil.RemoveFinalizer(&configuration, configurationFinalizer) if err := r.Update(ctx, &configuration); err != nil { return ctrl.Result{RequeueAfter: 3 * time.Second}, errors.Wrap(err, "failed to remove finalizer") @@ -216,7 +215,6 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, namespac destroyJob batchv1.Job k8sClient = r.Client ) - if configuration.Status.Apply.State == types.ConfigurationProvisioningAndChecking { warning := fmt.Sprintf("Destroy could not complete and needs to wait for Provision to complet first: %s", MessageCloudResourceProvisioningAndChecking) klog.Warning(warning) @@ -225,8 +223,10 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, namespac if err := k8sClient.Get(ctx, client.ObjectKey{Name: destroyJobName, Namespace: controllerNamespace}, &destroyJob); err != nil { if kerrors.IsNotFound(err) { - if err = assembleAndTriggerJob(ctx, k8sClient, configuration.Name, &configuration, tfInputConfigMapsName, namespace, r.ProviderName, TerraformDestroy); err != nil { - return err + if err := r.Client.Get(ctx, client.ObjectKey{Name: configuration.Name, Namespace: configuration.Namespace}, &v1beta1.Configuration{}); err == nil { + if err = assembleAndTriggerJob(ctx, k8sClient, configuration.Name, &configuration, tfInputConfigMapsName, namespace, r.ProviderName, TerraformDestroy); err != nil { + return err + } } } } @@ -242,7 +242,7 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, namespac } // When the deletion Job process succeeded, clean up work is starting. - if destroyJob.Status.Succeeded == *pointer.Int32Ptr(1) { + if destroyJob.Status.Succeeded == int32(1) { // 1. delete Terraform input Configuration ConfigMap if err := deleteConfigMap(ctx, k8sClient, tfInputConfigMapsName); err != nil { return err @@ -266,25 +266,19 @@ func (r *ConfigurationReconciler) terraformDestroy(ctx context.Context, namespac } // 4. delete destroy job - return k8sClient.Delete(ctx, &destroyJob, client.PropagationPolicy(metav1.DeletePropagationBackground)) + var j batchv1.Job + if err := r.Client.Get(ctx, client.ObjectKey{Name: destroyJob.Name, Namespace: destroyJob.Namespace}, &j); err == nil { + return r.Client.Delete(ctx, &j, client.PropagationPolicy(metav1.DeletePropagationBackground)) + } } return errors.New(MessageDestroyJobNotCompleted) } -func (r *ConfigurationReconciler) preCheck(ctx context.Context, namespace string, configuration *v1beta1.Configuration, tfInputConfigMapsName string) (bool, error) { +func (r *ConfigurationReconciler) preCheck(ctx context.Context, configuration *v1beta1.Configuration, tfInputConfigMapsName string) (bool, error) { var ( k8sClient = r.Client ) - var inputConfigurationCM v1.ConfigMap - if err := r.Client.Get(ctx, client.ObjectKey{Name: tfInputConfigMapsName, Namespace: namespace}, &inputConfigurationCM); err != nil { - if kerrors.IsNotFound(err) { - klog.InfoS("The input Configuration ConfigMaps doesn't exist", "Namespace", namespace, "Name", tfInputConfigMapsName) - } else { - return false, err - } - } - // Validation: 1) validate Configuration itself configurationType, err := cfgvalidator.ValidConfigurationObject(configuration) if err != nil { @@ -299,8 +293,23 @@ func (r *ConfigurationReconciler) preCheck(ctx context.Context, namespace string updateStatus(ctx, k8sClient, *configuration, types.ConfigurationSyntaxGood, "") //nolint:errcheck } - // Compose configuration with backend, and check whether configuration(hcl/json) is changed - completeConfiguration, configurationChanged, err := cfgvalidator.ComposeConfiguration(configuration, controllerNamespace, configurationType, &inputConfigurationCM) + // Render configuration with backend + completeConfiguration, err := cfgvalidator.RenderConfiguration(configuration, controllerNamespace, configurationType) + if err != nil { + return false, err + } + + var inputConfigurationCM v1.ConfigMap + if err := r.Client.Get(ctx, client.ObjectKey{Name: tfInputConfigMapsName, Namespace: controllerNamespace}, &inputConfigurationCM); err != nil { + if kerrors.IsNotFound(err) { + klog.InfoS("The input Configuration ConfigMaps doesn't exist", "Namespace", controllerNamespace, "Name", tfInputConfigMapsName) + } else { + return false, err + } + } + + // Check whether configuration(hcl/json) is changed + configurationChanged, err := cfgvalidator.CheckWhetherConfigurationChanges(configurationType, &inputConfigurationCM, completeConfiguration) if err != nil { return false, err } @@ -369,7 +378,10 @@ func (r *ConfigurationReconciler) updateTerraformJobIfNeeded(ctx context.Context // if either one changes, delete the job if envChanged || configurationChanged { - return r.Client.Delete(ctx, &job, client.PropagationPolicy(metav1.DeletePropagationBackground)) + var j batchv1.Job + if err := r.Client.Get(ctx, client.ObjectKey{Name: job.Name, Namespace: job.Namespace}, &j); err == nil { + return r.Client.Delete(ctx, &job, client.PropagationPolicy(metav1.DeletePropagationBackground)) + } } return nil } diff --git a/go.mod b/go.mod index 7b791422..87f16133 100644 --- a/go.mod +++ b/go.mod @@ -17,6 +17,5 @@ require ( k8s.io/client-go v0.18.8 k8s.io/klog/v2 v2.4.0 k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29 // indirect - k8s.io/utils v0.0.0-20200603063816-c1c6865ac451 sigs.k8s.io/controller-runtime v0.6.2 ) diff --git a/tf-validate-3316748360/main.tf b/tf-validate-3316748360/main.tf new file mode 100644 index 00000000..0a6bf5a2 --- /dev/null +++ b/tf-validate-3316748360/main.tf @@ -0,0 +1,20 @@ +resource "alicloud_oss_bucket" "bucket-acl" { + bucket = var.bucket + acl = var.acl +} + +output "BUCKET_NAME" { + value = "${alicloud_oss_bucket.bucket-acl.bucket}.${alicloud_oss_bucket.bucket-acl.extranet_endpoint}" +} + +variable "bucket" { + description = "OSS bucket name" + default = "vela-website" + type = string +} + +variable "acl" { + description = "OSS bucket ACL, supported 'private', 'public-read', 'public-read-write'" + default = "private" + type = string +}