From a199e4b6a63be55327cbd12ba5459481c333319e Mon Sep 17 00:00:00 2001 From: Ahmed AbouZaid <6760103+aabouzaid@users.noreply.github.com> Date: Sun, 10 Sep 2023 16:39:55 +0200 Subject: [PATCH] changed: tidy up the main example --- .gitignore | 1 + Makefile | 12 ++++++ README.md | 36 +++++++++++++---- examples/krm-and-kustomize/input/base.yaml | 8 ++++ examples/krm-and-kustomize/input/dev.yaml | 8 ++++ examples/krm-and-kustomize/input/stage.yaml | 8 ++++ examples/krm-and-kustomize/kustomization.yaml | 5 +++ examples/krm-and-kustomize/merger.yaml | 34 ++++++++++++++++ examples/krm-and-kustomize/resourcelist.yaml | 39 +++++++++++++++++++ .../krm-without-kustomize/input/common.yaml | 3 -- examples/krm-without-kustomize/input/dev.yaml | 3 -- .../krm-without-kustomize/input/stage.yaml | 3 -- .../krm-without-kustomize/resourcelist.yaml | 24 ------------ pkg/merger/merge.go | 7 +++- ...ators.kustomize.aabouzaid.com_mergers.yaml | 3 ++ pkg/merger/types.go | 4 +- 16 files changed, 154 insertions(+), 44 deletions(-) create mode 100644 examples/krm-and-kustomize/input/base.yaml create mode 100644 examples/krm-and-kustomize/input/dev.yaml create mode 100644 examples/krm-and-kustomize/input/stage.yaml create mode 100644 examples/krm-and-kustomize/kustomization.yaml create mode 100644 examples/krm-and-kustomize/merger.yaml create mode 100644 examples/krm-and-kustomize/resourcelist.yaml delete mode 100644 examples/krm-without-kustomize/input/common.yaml delete mode 100644 examples/krm-without-kustomize/input/dev.yaml delete mode 100644 examples/krm-without-kustomize/input/stage.yaml delete mode 100644 examples/krm-without-kustomize/resourcelist.yaml diff --git a/.gitignore b/.gitignore index 3b735ec..abd9ee0 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,7 @@ *.dll *.so *.dylib +dist/* # Test binary, built with `go test -c` *.test diff --git a/Makefile b/Makefile index ea5fb71..734a3d5 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,10 @@ CONTROLLER_GEN_HOME := /tmp/controller-gen CONTROLLER_GEN_VERSION := v0.13.0 +# +# Schema targets. +# + .PHONY: schema.install-tools schema.install-tools: mkdir -p $(CONTROLLER_GEN_HOME);\ @@ -16,6 +20,14 @@ schema.generate: -e 's|apiextensions.k8s.io/v1|config.kubernetes.io/v1alpha1|g' \ -i pkg/merger/schema/generators.kustomize.aabouzaid.com_mergers.yaml +# +# Golang targets. +# + +.PHONY: go.build +go.build: + go build -o 'dist' . + .PHONY: go.format go.format: gofumpt -l -w . diff --git a/README.md b/README.md index a1e0b2a..10a6954 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ A Kustomize generator plugin to merge YAML files seamlessly for real-world use c - [1. Generate multiple manifests from a single base](#1-generate-multiple-manifests-from-a-single-base) - [2. Merge lists in manifests without schema or a unique identifier](#2-merge-lists-in-manifests-without-schema-or-a-unique-identifier) - [3. Structure long manifests into smaller ones](#3-structure-long-manifests-into-smaller-ones) -- [TODO](#todo) +- [TO-DO](#to-do) - [Project status](#project-status) - [Contributing](#contributing) - [License](#license) @@ -36,7 +36,9 @@ Kustomize's default patch strategy for the lists (arrays) is `replace`, and to c for the Kubernetes `Custom Resource`, you must provide the OpenAPI schema of that custom resource, which is only helpful if the lists of that CR have a unique id. -For more details, please read: +Many people are asking for such functionalities as [easy merging CRs](https://stackoverflow.com/q/73655002/4547221), +[creating a ConfigMap from arbitrary YAML files](https://stackoverflow.com/q/74547569/4547221), +and for more details on the challenge of providing OpenAPI schema to merge files, please read the following post: [Set OpenAPI patch strategy for Kubernetes Custom Resources](https://tech.aabouzaid.com/2022/11/set-openapi-patch-strategy-for-kubernetes-custom-resources-kustomize.html). @@ -44,7 +46,8 @@ For more details, please read: - Generate multiple resources/manifests from a single base without copying the resources multiple times. - Merge any manifests (even CustomResources) without needing their OpenAPI schema. -- Merge manifests with a list of maps without a unique identifier (when using `x-kubernetes-patch-merge-key` is not possible). +- Merge manifests with a list of maps without a unique identifier + (when using `x-kubernetes-patch-merge-key` is not possible). - Merge YAML files with different merge strategies (StrategicMerge). - Merge applications configuration YAML files into a ConfigMap or Secret (WIP). @@ -57,6 +60,19 @@ apiVersion: generators.kustomize.aabouzaid.com/v1alpha1 kind: Merger metadata: name: merge + annotations: + # Containerized KRM function. + config.kubernetes.io/function: | + container: + image: ghcr.io/aabouzaid/kustomize-generator-merger + mounts: + - type: bind + src: ./ + dst: /mnt + # Exec KRM functions. + # config.kubernetes.io/function: | + # exec: + # path: ./kustomize-plugin-merger spec: resources: - name: example @@ -66,6 +82,8 @@ spec: # - Patch: Produce a single output by merging all sources together then with the destination. method: overlay files: + # The same as in the KRM container above. + root: /mnt sources: - src01.yaml - src02.yaml @@ -88,7 +106,8 @@ This section shows a couple of use cases where Merger can help. ### 1. Generate multiple manifests from a single base -In this case, you have multiple `CronJobs`, all of them share the same body, but each one has a different command or other config. +In this case, you have multiple `CronJobs`, all of them share the same body, +but each one has a different command or other config. [Use case full example](./examples/generate-multiple-manifests/README.md). @@ -96,7 +115,7 @@ In this case, you have multiple `CronJobs`, all of them share the same body, but Currently, in Kustomize, it's not possible to merge resources without a unique identifier, even with Open API schema. -It's possible to do that using the merge strategy `append` in Merger (later on, `combine` will also be supported). +It's possible to do that using the merge strategy `append` in Merger (later on, `combineWithKey` will also be supported). [Use case full example](./examples/merge-lists-without-schema/README.md). @@ -109,15 +128,16 @@ and use the Merger `patch` input method to make it a single manifest again. [Use case full example](./examples/structure-long-manifests/README.md). -## TODO +## TO-DO - Support `ConfigMap` or `Secret` as an output. - Support `combine` merge strategy with an identifier key (similar to `x-kubernetes-patch-merge-key`). +- Provide better docs for Merger options. ## Project status -Please note that this project is still under development and could be breaking changes, +Please note that this project is still under development and could have breaking changes, but it will follow the SemVer convention. @@ -130,4 +150,4 @@ or [create a PR](https://github.com/aabouzaid/kustomize-plugin-merger/pulls). ## License -This is open-source software licensed using the Apache License 2.0. Please see [LICENSE](LICENSE) for details. +Merger is an open-source software licensed using the Apache License 2.0. Please see [LICENSE](LICENSE) for details. diff --git a/examples/krm-and-kustomize/input/base.yaml b/examples/krm-and-kustomize/input/base.yaml new file mode 100644 index 0000000..7ae17ba --- /dev/null +++ b/examples/krm-and-kustomize/input/base.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Pod +metadata: + name: base +spec: + containers: + - name: sidecar + image: sidecar diff --git a/examples/krm-and-kustomize/input/dev.yaml b/examples/krm-and-kustomize/input/dev.yaml new file mode 100644 index 0000000..a887280 --- /dev/null +++ b/examples/krm-and-kustomize/input/dev.yaml @@ -0,0 +1,8 @@ +metadata: + name: dev +spec: + containers: + - name: dev + image: dev:1.0.0 + ports: + - containerPort: 80 diff --git a/examples/krm-and-kustomize/input/stage.yaml b/examples/krm-and-kustomize/input/stage.yaml new file mode 100644 index 0000000..cffb165 --- /dev/null +++ b/examples/krm-and-kustomize/input/stage.yaml @@ -0,0 +1,8 @@ +metadata: + name: stage +spec: + containers: + - name: stage + image: stage:1.0.0 + ports: + - containerPort: 80 diff --git a/examples/krm-and-kustomize/kustomization.yaml b/examples/krm-and-kustomize/kustomization.yaml new file mode 100644 index 0000000..2880438 --- /dev/null +++ b/examples/krm-and-kustomize/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +generators: +- merger.yaml diff --git a/examples/krm-and-kustomize/merger.yaml b/examples/krm-and-kustomize/merger.yaml new file mode 100644 index 0000000..fcc81a6 --- /dev/null +++ b/examples/krm-and-kustomize/merger.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: generators.kustomize.aabouzaid.com/v1alpha1 +kind: Merger +metadata: + name: merge + annotations: + # Containerized KRM function. + config.kubernetes.io/function: | + container: + image: ghcr.io/aabouzaid/kustomize-generator-merger + mounts: + - type: bind + src: ./ + dst: /mnt + # Exec KRM functions. + # config.kubernetes.io/function: | + # exec: + # path: ./kustomize-plugin-merger +spec: + resources: + - name: my-envs + input: + method: overlay + files: + # The same as in the KRM container above. + root: /mnt + sources: + - input/dev.yaml + - input/stage.yaml + destination: input/base.yaml + merge: + strategy: append + output: + format: raw diff --git a/examples/krm-and-kustomize/resourcelist.yaml b/examples/krm-and-kustomize/resourcelist.yaml new file mode 100644 index 0000000..7fd49a3 --- /dev/null +++ b/examples/krm-and-kustomize/resourcelist.yaml @@ -0,0 +1,39 @@ +--- +apiVersion: config.kubernetes.io/v1 +kind: ResourceList +metadata: + name: krm-function-input +functionConfig: + apiVersion: generators.kustomize.aabouzaid.com/v1alpha1 + kind: Merger + metadata: + name: merge + annotations: + # Containerized KRM function. + config.kubernetes.io/function: | + container: + image: ghcr.io/aabouzaid/kustomize-generator-merger + mounts: + - type: bind + src: ./ + dst: /mnt + # Exec KRM functions. + # config.kubernetes.io/function: | + # exec: + # path: ./kustomize-plugin-merger + spec: + resources: + - name: my-envs + input: + method: overlay + files: + # The same as in the KRM container above. + root: /mnt + sources: + - input/dev.yaml + - input/stage.yaml + destination: input/base.yaml + merge: + strategy: append + output: + format: raw \ No newline at end of file diff --git a/examples/krm-without-kustomize/input/common.yaml b/examples/krm-without-kustomize/input/common.yaml deleted file mode 100644 index 2b6e46a..0000000 --- a/examples/krm-without-kustomize/input/common.yaml +++ /dev/null @@ -1,3 +0,0 @@ -versions: - labels: - foo: bar diff --git a/examples/krm-without-kustomize/input/dev.yaml b/examples/krm-without-kustomize/input/dev.yaml deleted file mode 100644 index 9df437b..0000000 --- a/examples/krm-without-kustomize/input/dev.yaml +++ /dev/null @@ -1,3 +0,0 @@ -versions: - env: dev - dev: true diff --git a/examples/krm-without-kustomize/input/stage.yaml b/examples/krm-without-kustomize/input/stage.yaml deleted file mode 100644 index cf3863f..0000000 --- a/examples/krm-without-kustomize/input/stage.yaml +++ /dev/null @@ -1,3 +0,0 @@ -versions: - env: stage - stage: true \ No newline at end of file diff --git a/examples/krm-without-kustomize/resourcelist.yaml b/examples/krm-without-kustomize/resourcelist.yaml deleted file mode 100644 index c2e206a..0000000 --- a/examples/krm-without-kustomize/resourcelist.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -apiVersion: config.kubernetes.io/v1 -kind: ResourceList -metadata: - name: krm-function-input -functionConfig: - apiVersion: generators.kustomize.aabouzaid.com/v1alpha1 - kind: Merger - metadata: - name: merge - spec: - resources: - - name: my-envs - input: - method: overlay - files: - sources: - - example/input/dev.yaml - - example/input/stage.yaml - destination: example/input/common.yaml - merge: - strategy: combine - output: - format: raw diff --git a/pkg/merger/merge.go b/pkg/merger/merge.go index fa771f7..1d6e45b 100644 --- a/pkg/merger/merge.go +++ b/pkg/merger/merge.go @@ -29,13 +29,16 @@ func (r *mergerResource) setInputFilesOverlay() { for _, inputFileSource := range r.Input.Files.Sources { r.Input.items = append(r.Input.items, resourceInputFiles{ - Sources: []string{inputFileSource}, - Destination: r.Input.Files.Destination, + Sources: []string{r.Input.Files.Root + inputFileSource}, + Destination: r.Input.Files.Root + r.Input.Files.Destination, }) } } func (r *mergerResource) setInputFilesPatch() { + for index, inputFileSource := range r.Input.Files.Sources { + r.Input.Files.Sources[index] = r.Input.Files.Root + inputFileSource + } r.Input.items = append(r.Input.items, r.Input.Files) } diff --git a/pkg/merger/schema/generators.kustomize.aabouzaid.com_mergers.yaml b/pkg/merger/schema/generators.kustomize.aabouzaid.com_mergers.yaml index ccb521b..0de139d 100644 --- a/pkg/merger/schema/generators.kustomize.aabouzaid.com_mergers.yaml +++ b/pkg/merger/schema/generators.kustomize.aabouzaid.com_mergers.yaml @@ -20,6 +20,7 @@ spec: - name: v1alpha1 schema: openAPIV3Schema: + description: Merger manifest. properties: apiVersion: description: 'APIVersion defines the versioned schema of this representation @@ -44,6 +45,8 @@ spec: properties: destination: type: string + root: + type: string sources: items: type: string diff --git a/pkg/merger/types.go b/pkg/merger/types.go index d6d6c34..67cbcfd 100644 --- a/pkg/merger/types.go +++ b/pkg/merger/types.go @@ -20,7 +20,7 @@ const ( ResourceKind string = "Merger" ) -// Merger manifest body. +// Merger manifest. type Merger struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata"` @@ -49,6 +49,8 @@ type mergerResource struct { // type resourceInputFiles struct { + // +optional + Root string `yaml:"root" json:"root"` Sources []string `yaml:"sources" json:"sources"` Destination string `yaml:"destination" json:"destination"` }