Recursively diff Kustomizations

Signed-off-by: Boris Kreitchman <[email protected]>

cmd/flux/diff_kustomization.go

@@ -44,7 +44,12 @@ flux diff kustomization my-app --path ./path/to/local/manifests \
 # Exclude files by providing a comma separated list of entries that follow the .gitignore pattern fromat.
 flux diff kustomization my-app --path ./path/to/local/manifests \
 	--kustomization-file ./path/to/local/my-app.yaml \
-	--ignore-paths "/to_ignore/**/*.yaml,ignore.yaml"`,
+	--ignore-paths "/to_ignore/**/*.yaml,ignore.yaml
+
+# Run recursively on all encountered Kustomizations
+flux diff kustomization my-app --path ./path/to/local/manifests \
+    --recursive \
+    --local-sources GitRepository/flux-system/my-repo=./path/to/local/git"`,
 	ValidArgsFunction: resourceNamesCompletionFunc(kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)),
 	RunE:              diffKsCmdRun,
 }
@@ -55,6 +60,8 @@ type diffKsFlags struct {
 	ignorePaths       []string
 	progressBar       bool
 	strictSubst       bool
+	recursive         bool
+	localSources      map[string]string
 }
 
 var diffKsArgs diffKsFlags
@@ -66,6 +73,8 @@ func init() {
 	diffKsCmd.Flags().StringVar(&diffKsArgs.kustomizationFile, "kustomization-file", "", "Path to the Flux Kustomization YAML file.")
 	diffKsCmd.Flags().BoolVar(&diffKsArgs.strictSubst, "strict-substitute", false,
 		"When enabled, the post build substitutions will fail if a var without a default value is declared in files but is missing from the input vars.")
+	diffKsCmd.Flags().BoolVarP(&diffKsArgs.recursive, "recursive", "r", false, "Recursively diff Kustomizations")
+	diffKsCmd.Flags().StringToStringVar(&diffKsArgs.localSources, "local-sources", nil, "Comma-separated list of repositories in format: Kind/namespace/name=path")
 	diffCmd.AddCommand(diffKsCmd)
 }
 
@@ -101,6 +110,8 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
 			build.WithProgressBar(),
 			build.WithIgnore(diffKsArgs.ignorePaths),
 			build.WithStrictSubstitute(diffKsArgs.strictSubst),
+			build.WithRecursive(diffKsArgs.recursive),
+			build.WithLocalSources(diffKsArgs.localSources),
 		)
 	} else {
 		builder, err = build.NewBuilder(name, diffKsArgs.path,
@@ -109,6 +120,8 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
 			build.WithKustomizationFile(diffKsArgs.kustomizationFile),
 			build.WithIgnore(diffKsArgs.ignorePaths),
 			build.WithStrictSubstitute(diffKsArgs.strictSubst),
+			build.WithRecursive(diffKsArgs.recursive),
+			build.WithLocalSources(diffKsArgs.localSources),
 		)
 	}
 
@@ -138,6 +151,12 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
 
 	select {
 	case <-sigc:
+		if diffKsArgs.progressBar {
+			err := builder.StopSpinner()
+			if err != nil {
+				return err
+			}
+		}
 		fmt.Println("Build cancelled... exiting.")
 		return builder.Cancel()
 	case err := <-errChan:

internal/build/build.go

@@ -51,13 +51,14 @@ import (
 )
 
 const (
-	controllerName      = "kustomize-controller"
-	controllerGroup     = "kustomize.toolkit.fluxcd.io"
-	mask                = "**SOPS**"
-	dockercfgSecretType = "kubernetes.io/dockerconfigjson"
-	typeField           = "type"
-	dataField           = "data"
-	stringDataField     = "stringData"
+	controllerName       = "kustomize-controller"
+	controllerGroup      = "kustomize.toolkit.fluxcd.io"
+	mask                 = "**SOPS**"
+	dockercfgSecretType  = "kubernetes.io/dockerconfigjson"
+	typeField            = "type"
+	dataField            = "data"
+	stringDataField      = "stringData"
+	spinnerDryRunMessage = "running dry-run"
 )
 
 var defaultTimeout = 80 * time.Second
@@ -81,6 +82,8 @@ type Builder struct {
 	spinner       *yacspin.Spinner
 	dryRun        bool
 	strictSubst   bool
+	recursive     bool
+	localSources  map[string]string
 }
 
 // BuilderOptionFunc is a function that configures a Builder
@@ -110,7 +113,7 @@ func WithProgressBar() BuilderOptionFunc {
 			CharSet:         yacspin.CharSets[59],
 			Suffix:          "Kustomization diffing...",
 			SuffixAutoColon: true,
-			Message:         "running dry-run",
+			Message:         spinnerDryRunMessage,
 			StopCharacter:   "✓",
 			StopColors:      []string{"fgGreen"},
 		}
@@ -175,6 +178,37 @@ func WithIgnore(ignore []string) BuilderOptionFunc {
 	}
 }
 
+// WithRecursive sets the recurvice flag
+func WithRecursive(recursive bool) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.recursive = recursive
+		return nil
+	}
+}
+
+// WithLocalSources sets the localSources field
+func WithLocalSources(localSources map[string]string) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.localSources = localSources
+		return nil
+	}
+}
+
+func withClientConfigFrom(in *Builder) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.client = in.client
+		b.restMapper = in.restMapper
+		return nil
+	}
+}
+
+func withSpinnerFrom(in *Builder) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.spinner = in.spinner
+		return nil
+	}
+}
+
 // NewBuilder returns a new Builder
 // It takes a kustomization name and a path to the resources
 // It also takes a list of BuilderOptionFunc to configure the builder
@@ -583,20 +617,15 @@ func (b *Builder) Cancel() error {
 	b.mu.Lock()
 	defer b.mu.Unlock()
 
-	err := b.stopSpinner()
-	if err != nil {
-		return err
-	}
-
-	err = kustomize.CleanDirectory(b.resourcesPath, b.action)
+	err := kustomize.CleanDirectory(b.resourcesPath, b.action)
 	if err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (b *Builder) startSpinner() error {
+func (b *Builder) StartSpinner() error {
 	if b.spinner == nil {
 		return nil
 	}
@@ -609,7 +638,7 @@ func (b *Builder) startSpinner() error {
 	return nil
 }
 
-func (b *Builder) stopSpinner() error {
+func (b *Builder) StopSpinner() error {
 	if b.spinner == nil {
 		return nil
 	}

internal/build/diff.go

@@ -33,6 +33,7 @@ import (
 	"github.com/homeport/dyff/pkg/dyff"
 	"github.com/lucasb-eyer/go-colorful"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/errors"
 	"sigs.k8s.io/yaml"
@@ -57,6 +58,22 @@ func (b *Builder) Manager() (*ssa.ResourceManager, error) {
 }
 
 func (b *Builder) Diff() (string, bool, error) {
+	err := b.StartSpinner()
+	if err != nil {
+		return "", false, err
+	}
+
+	output, createdOrDrifted, diffErr := b.diff()
+
+	err = b.StopSpinner()
+	if err != nil {
+		return "", false, err
+	}
+
+	return output, createdOrDrifted, diffErr
+}
+
+func (b *Builder) diff() (string, bool, error) {
 	output := strings.Builder{}
 	createdOrDrifted := false
 	objects, err := b.Build()
@@ -77,11 +94,6 @@ func (b *Builder) Diff() (string, bool, error) {
 	ctx, cancel := context.WithTimeout(context.Background(), b.timeout)
 	defer cancel()
 
-	err = b.startSpinner()
-	if err != nil {
-		return "", false, err
-	}
-
 	var diffErrs []error
 	// create an inventory of objects to be reconciled
 	newInventory := newInventory()
@@ -127,6 +139,30 @@ func (b *Builder) Diff() (string, bool, error) {
 		}
 
 		addObjectsToInventory(newInventory, change)
+
+		if b.recursive && isKustomization(obj) && change.Action != ssa.CreatedAction {
+			kustomization, err := toKustomization(obj)
+			if err != nil {
+				return "", createdOrDrifted, err
+			}
+
+			if !kustomizationsEqual(kustomization, b.kustomization) {
+				subOutput, subCreatedOrDrifted, err := b.kustomizationDiff(kustomization)
+				if err != nil {
+					diffErrs = append(diffErrs, err)
+				}
+				if subCreatedOrDrifted {
+					createdOrDrifted = true
+					output.WriteString(bunt.Sprint(fmt.Sprintf("📁 %s changed\n", ssautil.FmtUnstructured(obj))))
+					output.WriteString(subOutput)
+				}
+
+				// finished with Kustomization diff
+				if b.spinner != nil {
+					b.spinner.Message(spinnerDryRunMessage)
+				}
+			}
+		}
 	}
 
 	if b.spinner != nil {
@@ -149,12 +185,63 @@ func (b *Builder) Diff() (string, bool, error) {
 		}
 	}
 
-	err = b.stopSpinner()
+	return output.String(), createdOrDrifted, errors.Reduce(errors.Flatten(errors.NewAggregate(diffErrs)))
+}
+
+func isKustomization(object *unstructured.Unstructured) bool {
+	return object.GetKind() == "Kustomization" && strings.HasPrefix(object.GetAPIVersion(), controllerGroup)
+}
+
+func toKustomization(object *unstructured.Unstructured) (*kustomizev1.Kustomization, error) {
+	kustomization := &kustomizev1.Kustomization{}
+	obj, err := runtime.DefaultUnstructuredConverter.ToUnstructured(object)
 	if err != nil {
-		return "", createdOrDrifted, err
+		return nil, fmt.Errorf("failed to convert to unstructured: %w", err)
 	}
+	err = runtime.DefaultUnstructuredConverter.FromUnstructured(obj, kustomization)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert to kustomization: %w", err)
+	}
+	return kustomization, nil
+}
 
-	return output.String(), createdOrDrifted, errors.Reduce(errors.Flatten(errors.NewAggregate(diffErrs)))
+func kustomizationsEqual(k1 *kustomizev1.Kustomization, k2 *kustomizev1.Kustomization) bool {
+	return k1.Name == k2.Name && k1.Namespace == k2.Namespace
+}
+
+func (b *Builder) kustomizationDiff(kustomization *kustomizev1.Kustomization) (string, bool, error) {
+	if b.spinner != nil {
+		b.spinner.Message(fmt.Sprintf("%s in %s", spinnerDryRunMessage, kustomization.Name))
+	}
+
+	sourceRef := kustomization.Spec.SourceRef.DeepCopy()
+	if sourceRef.Namespace == "" {
+		sourceRef.Namespace = kustomization.Namespace
+	}
+
+	sourceKey := sourceRef.String()
+	localPath, ok := b.localSources[sourceKey]
+	if !ok {
+		return "", false, fmt.Errorf("cannot get local path for %s of kustomization %s", sourceKey, kustomization.Name)
+	}
+
+	resourcesPath := filepath.Join(localPath, kustomization.Spec.Path)
+	subBuilder, err := NewBuilder(kustomization.Name, resourcesPath,
+		// use same client and spinner
+		withClientConfigFrom(b),
+		withSpinnerFrom(b),
+		WithTimeout(b.timeout),
+		WithIgnore(b.ignore),
+		WithStrictSubstitute(b.strictSubst),
+		WithRecursive(b.recursive),
+		WithLocalSources(b.localSources),
+		WithNamespace(kustomization.Namespace),
+	)
+	if err != nil {
+		return "", false, err
+	}
+
+	return subBuilder.diff()
 }
 
 func writeYamls(liveObject, mergedObject *unstructured.Unstructured) (string, string, string, error) {