Provider helm chart (#343)

sky-uk · Jul 24, 2024 · d39b920 · d39b920
1 parent b748dcf
commit d39b920
Show file tree

Hide file tree

Showing 34 changed files with 435 additions and 326 deletions.
diff --git a/Makefile b/Makefile
@@ -79,7 +79,7 @@ test-argo:
 	$(MAKE) -C argo/kfp-compiler test
 	$(MAKE) -C argo/providers test
 
-test-all: test helm-test test-argo
+test-all: test helm-test-operator helm-test-provider test-argo
 
 integration-test-all: integration-test
 	$(MAKE) -C argo/kfp-compiler integration-test
@@ -133,19 +133,35 @@ kustomize: ## Download kustomize locally if necessary.
 	$(call go-install,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/[email protected])
 
 ##@ Package
-
-helm-package: helm-cmd helm-test
+helm-package-operator: helm-cmd helm-test-operator
 	$(HELM) package helm/kfp-operator --version $(VERSION) --app-version $(VERSION) -d dist
 
-helm-install: helm-package values.yaml
+helm-package-provider: helm-cmd helm-test-provider
+	$(HELM) package helm/provider --version $(VERSION) --app-version $(VERSION) -d dist
+
+helm-package: helm-package-operator helm-package-provider
+
+helm-install-operator: helm-package-operator values.yaml
 	$(HELM) install -f values.yaml kfp-operator dist/kfp-operator-$(VERSION).tgz
 
-helm-uninstall:
+helm-uninstall-operator:
 	$(HELM) uninstall kfp-operator
 
-helm-upgrade: helm-package values.yaml
+helm-upgrade-operator: helm-package-operator values.yaml
 	$(HELM) upgrade -f values.yaml kfp-operator dist/kfp-operator-$(VERSION).tgz
 
+# NAME needs to be passed as an argument to the make target to point at the specific values file for the provider being installed
+helm-install-provider: helm-package-provider
+	$(HELM) install -f $(NAME).yaml provider-$(NAME) dist/provider-$(VERSION).tgz
+
+# NAME needs to be passed as an argument to the make target to point at the specific values file for the provider being installed
+helm-uninstall-provider:
+	$(HELM) uninstall provider-$(NAME)
+
+# NAME needs to be passed as an argument to the make target to point at the specific values file for the provider being installed
+helm-upgrade-provider: helm-package-provider
+	$(HELM) upgrade -f $(NAME).yaml provider-$(NAME) dist/provider-$(VERSION).tgz
+
 ifeq ($(HELM_REPOSITORIES)$(OSS_HELM_REPOSITORIES),)
 helm-publish:
 	$(error OSS_HELM_REPOSITORIES or HELM_REPOSITORIES must be provided as space-separated lists of URLs)
@@ -161,15 +177,17 @@ define helm-upload
 @echo "Publishing Helm chart to $(1)"
 @if [[ "$(1)" == "oci://"* ]]; then \
 	helm push dist/kfp-operator-$(VERSION).tgz $(1)/kfp-operator; \
+	helm push dist/provider-$(VERSION).tgz $(1)/provider; \
 else \
 	curl --fail --netrc-file $(NETRC_FILE) -T dist/kfp-operator-$(VERSION).tgz $(1); \
+	curl --fail --netrc-file $(NETRC_FILE) -T dist/provider-$(VERSION).tgz $(1); \
 fi
 $(NEWLINE)
 endef
 endif
 
 INDEXED_YAML := $(YQ) e '{([.metadata.name, .kind] | join("-")): .}'
-helm-test: manifests helm-cmd kustomize yq dyff
+helm-test-operator: manifests helm-cmd kustomize yq dyff
 	$(eval TMP := $(shell mktemp -d))
 
 	# Create yaml files with helm and kustomize.
@@ -181,6 +199,10 @@ helm-test: manifests helm-cmd kustomize yq dyff
 	$(DYFF) between --set-exit-code $(TMP)/helm_indexed $(TMP)/kustomize_indexed
 	rm -rf $(TMP)
 
+helm-test-provider: helm-cmd
+	$(eval TMP := $(shell mktemp -d))
+	$(HELM) template helm/provider -f helm/provider/test/values.yaml > $(TMP)/helm
+
 ##@ Containers
 
 include docker-targets.mk

diff --git a/config/manager/controller_manager_config.yaml b/config/manager/controller_manager_config.yaml
@@ -11,10 +11,6 @@ controller:
     leaderElect: true
     resourceName: kfp-operator-lock
 spec:
-  defaultBeamArgs:
-  - name: project
-    value: ${DATAFLOW_PROJECT}
   multiversion: true
-  pipelineStorage: ${PIPELINE_STORAGE}
   workflowNamespace: kfp-operator-system
   workflowTemplatePrefix: kfp-operator-
diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml
@@ -1,6 +1,5 @@
 resources:
 - manager.yaml
-- providers.yaml
 - workflows/common.yaml
 - workflows/compiled.yaml
 - workflows/rbac.yaml

diff --git a/config/manager/providers.yaml b/config/manager/providers.yaml
diff --git a/docs-gen/content/docs/getting-started/installation.md b/docs-gen/content/docs/getting-started/installation.md
@@ -11,9 +11,12 @@ This guide assumes you are familiar with [Helm](https://helm.sh/).
 
 - [Argo 3.1.6-3.3](https://argoproj.github.io/argo-workflows/installation/) installed cluster-wide or into the namespace where the operator's workflows run (see [configuration](../../reference/configuration)).
 - [Argo-Events 1.7.4+](https://argoproj.github.io/argo-events/installation/) installed cluster-wide (see [configuration](../../reference/configuration)).
-- The KFP-Operator supports configurable provider backends. Currently, Kubeflow Pipelines and Vertex AI are supported. Please refer to the [respective configuration section](../../reference/configuration/#provider-configuration) before proceeding.
 
-## Build and Install
+## KFP-Operator
+
+To get a working installation you will need to install both the KFP-Operator and at least one provider ([see below]({{< ref "#providers" >}} "Providers"))
+
+### Build and Install
 
 Create basic `values.yaml` with the following content:
 
@@ -29,8 +32,7 @@ helm install oci://ghcr.io/kfp-operator/kfp-operator -f values.yaml
 
 You will need to configure service accounts and roles required by your chosen `Provider`, [see here for reference]({{< ref "#provider-rbac" >}} "Provider RBAC Reference").
 
-
-## Configuration Values
+### Configuration Values
 
 Valid configuration options to override the [Default `values.yaml`]({{< ghblob "/helm/kfp-operator/values.yaml" >}}) are:
 
@@ -65,21 +67,39 @@ Valid configuration options to override the [Default `values.yaml`]({{< ghblob "
 | `manager.provider.type`                                   | Provider type (`kfp` for Kubeflow Pipelines or `vai` for Vertex AI Pipelines)                                                                                                                                       |
 | `manager.provider.configuration`                          | Configuration block for the specific provider (see [Provider Configuration](../../reference/configuration#provider-configuration)), automatically mounted as a file                                                 |
 | `logging.verbosity`                                       | Logging verbosity for all components - see the [logging documentation]({{< param "github_project_repo" >}}/blob/master/CONTRIBUTING.md#logging) for valid values                                                    |
-| `eventsourceServer.metadata`                              | [Object Metadata](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/object-meta/#ObjectMeta) for the eventsource server's pods                                                                 |
-| `eventsourceServer.rbac.create`                           | Create roles and rolebindings for the eventsource server                                                                                                                                                            |
-| `eventsourceServer.serviceAccount.name`                   | Eventsource server's service account                                                                                                                                                                                |
-| `eventsourceServer.serviceAccount.create`                 | Create the eventsource server's service account or expect it to be created externally                                                                                                                               |
-| `eventsourceServer.resources`                             | Eventsource server resources as per [k8s documentation](https://kubernetes.io/docs/reference/kubernetes-api/workload-resources/pod-v1/#resources)                                                                   |
-| `providers`                                               | Dictionary of providers (see below)                                                                                                                                                                                 |
+| `statusFeedback.enabled`                                  | Whether run completion eventing and status update feedback loop should be installed - defaults to `false`                                                                                                           |
 
 Examples for these values can be found in the [test configuration]({{< ghblob "/helm/kfp-operator/test/values.yaml" >}})
 
-### Providers
+## Providers
+
+Supported providers are:
+- Kubeflow Pipelines
+- Vertex AI
+
+Install one or more by following these instructions. Please refer to the [respective configuration section](../../reference/configuration/#provider-configuration) before proceeding.
+
+### Build and Install
+
+Create basic `kfp.yaml` value file with the following content:
+
+```yaml
+{{% readfile file="includes/quickstart/resources/kfp.yaml" %}}
+```
+
+Install the latest version of the provider
+
+```sh
+helm install oci://ghcr.io/kfp-operator/provider -f kfp.yaml
+```
 
-The `providers` block contains a dictionary of provider names to provider configurations:
+### Configuration
+
+The `provider` block contains provider configurations:
 
 | Parameter name            | Description                                                                                                                                                 |
 |---------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `name`                    | Name given to this provider                                                                                                                                 |
 | `type`                    | Provider type (`kfp` or `vai`)                                                                                                                              |
 | `serviceAccount.name`     | Name of the service account to run provider-specific operations                                                                                             |
 | `serviceAccount.create`   | Create the service account (or assume it has been created externally)                                                                                       |
@@ -89,23 +109,13 @@ The `providers` block contains a dictionary of provider names to provider config
 Example:
 
 ```yaml
-providers:
-  kfp:
-    type: kfp
-    serviceAccount:
-      name: kfp-operator-kfp
-      create: false
-    configuration:
-      ...
-  vai:
-    type: vai
-    serviceAccount: 
-      name: kfp-operator-kfp
-      create: true
-      metadata:
-        annotations:
-          iam.gke.io/gcp-service-account: [email protected]
-    configuration:
+provider:
+  name: kfp-provider
+  type: kfp
+  executionMode: v1
+  serviceAccount:
+    name: kfp-operator-kfp
+    create: false
       ...
 ```
 

diff --git a/docs-gen/content/docs/getting-started/overview.md b/docs-gen/content/docs/getting-started/overview.md
@@ -48,7 +48,7 @@ The Kubeflow Pipelines operator can optionally be installed with [Argo-Events](h
 
 Currently, we support the following eventsources:
 
-- [Run Completion Eventsource](../reference/run-completion)
+- [Run Completion Eventsource](../../reference/run-completion)
 
 ## Architecture Overview
 

diff --git a/docs-gen/includes/quickstart/resources/kfp.yaml b/docs-gen/includes/quickstart/resources/kfp.yaml
@@ -0,0 +1,16 @@
+provider:
+  name: kfp-provider
+  type: kfp
+  executionMode: v1
+  serviceAccount:
+    name: kfp-operator-kfp
+    create: false
+  configuration:
+    kfpNamespace: kubeflow
+    restKfpApiUrl: http://ml-pipeline.kubeflow:8888
+    grpcMetadataStoreAddress: metadata-grpc-service.kubeflow:8080
+    grpcKfpApiAddress: ml-pipeline.kubeflow:8887
+    defaultBeamArgs:
+      - name: project
+        value: ${DATAFLOW_PROJECT}
+    pipelineRootStorage: ${PIPELINE_STORAGE}
diff --git a/docs-gen/includes/quickstart/resources/values.yaml b/docs-gen/includes/quickstart/resources/values.yaml
@@ -4,11 +4,3 @@ manager:
     serviceAccount: pipeline-runner
   configuration:
     defaultExperiment: Default
-    pipelineStorage: {STORAGE_LOCATION}
-  provider:
-    type: kfp
-    configuration:
-      kfpNamespace: kubeflow
-      restKfpApiUrl: http://ml-pipeline.kubeflow:8888
-      grpcKfpApiAddress: ml-pipeline.kubeflow:8887
-      grpcMetadataStoreAddress: metadata-grpc-service.kubeflow:8080