diff --git a/charts/flyte-core/README.md b/charts/flyte-core/README.md index fecd58959f3..a61d1b7d218 100644 --- a/charts/flyte-core/README.md +++ b/charts/flyte-core/README.md @@ -92,9 +92,9 @@ helm install gateway bitnami/contour -n flyte | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI | | configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.10.7-b1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration | | configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.10.7-b1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/flyteplugins@v0.5.28/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) | -| configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | +| configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration | | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). | -| configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | +| configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). | | configmap.datacatalogServer | object | `{"application":{"grpcPort":8089,"grpcServerReflection":true,"httpPort":8080},"datacatalog":{"heartbeat-grace-period-multiplier":3,"max-reservation-heartbeat":"30s","metrics-scope":"datacatalog","profiler-port":10254,"storage-prefix":"metadata/datacatalog"}}` | Datacatalog server config | | configmap.domain | object | `{"domains":[{"id":"development","name":"development"},{"id":"staging","name":"staging"},{"id":"production","name":"production"}]}` | Domains configuration for Flyte projects. This enables the specified number of domains across all projects in Flyte. | | configmap.enabled_plugins.tasks | object | `{"task-plugins":{"default-for-task-types":{"container":"container","container_array":"k8s-array","sidecar":"sidecar"},"enabled-plugins":["container","sidecar","k8s-array"]}}` | Tasks specific configuration [structure](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/nodes/task/config#GetConfig) | @@ -255,7 +255,7 @@ helm install gateway bitnami/contour -n flyte | sparkoperator.enabled | bool | `false` | - enable or disable Sparkoperator deployment installation | | sparkoperator.plugin_config | object | `{"plugins":{"spark":{"spark-config-default":[{"spark.hadoop.fs.s3a.aws.credentials.provider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain"},{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]}}}` | Spark plugin configuration | | sparkoperator.plugin_config.plugins.spark.spark-config-default | list | `[{"spark.hadoop.fs.s3a.aws.credentials.provider":"com.amazonaws.auth.DefaultAWSCredentialsProviderChain"},{"spark.hadoop.mapreduce.fileoutputcommitter.algorithm.version":"2"},{"spark.kubernetes.allocation.batch.size":"50"},{"spark.hadoop.fs.s3a.acl.default":"BucketOwnerFullControl"},{"spark.hadoop.fs.s3n.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3n.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.impl":"org.apache.hadoop.fs.s3a.S3AFileSystem"},{"spark.hadoop.fs.AbstractFileSystem.s3a.impl":"org.apache.hadoop.fs.s3a.S3A"},{"spark.hadoop.fs.s3a.multipart.threshold":"536870912"},{"spark.blacklist.enabled":"true"},{"spark.blacklist.timeout":"5m"},{"spark.task.maxfailures":"8"}]` | Spark default configuration | -| storage | object | `{"bucketName":"my-s3-bucket","custom":{},"enableMultiContainer":false,"gcs":null,"limits":{"maxDownloadMBs":10},"s3":{"accessKey":"","authType":"iam","region":"us-east-1","secretKey":""},"type":"sandbox"}` | ---------------------------------------------------- STORAGE SETTINGS | +| storage | object | `{"bucketName":"my-s3-bucket","custom":{},"enableMultiContainer":false,"gcs":null,"limits":{"maxDownloadMBs":10},"s3":{"accessKey":"","authType":"iam","region":"us-east-1","secretKey":""},"type":"sandbox","userBucketName":"my-s3-bucket"}` | ---------------------------------------------------- STORAGE SETTINGS | | storage.bucketName | string | `"my-s3-bucket"` | bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. | | storage.custom | object | `{}` | Settings for storage type custom. See https://github.com/graymeta/stow for supported storage providers/settings. | | storage.enableMultiContainer | bool | `false` | toggles multi-container storage config | diff --git a/charts/flyte-core/templates/_helpers.tpl b/charts/flyte-core/templates/_helpers.tpl index 0221874d5cb..6c631e3666d 100755 --- a/charts/flyte-core/templates/_helpers.tpl +++ b/charts/flyte-core/templates/_helpers.tpl @@ -17,6 +17,26 @@ flyteadmin {{- end -}} +{{- define "flytepropeller-userstorage" -}} +propeller: + rawoutput-prefix: {{ include "flyte-core.storage.userDataPrefix" . }} +{{- end -}} + +{{/* +Get the Flyte user data prefix. +*/}} +{{- define "flyte-core.storage.userDataPrefix" -}} +{{- $userBucketName := required "User data container required" .Values.storage.userBucketName -}} +{{- if eq "s3" .Values.storage.type -}} +{{- printf "s3://%s/data" $userBucketName -}} +{{- else if eq "gcs" .Values.storage.type -}} +{{- printf "gs://%s/data" $userBucketName -}} +{{- else if eq "azure" .Values.storage.type -}} +{{- printf "abfs://%s/data" $userBucketName -}} +{{- end -}} +{{- end -}} + + {{- define "flyteadmin.selectorLabels" -}} app.kubernetes.io/name: {{ template "flyteadmin.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} @@ -157,6 +177,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end }} {{- define "storage.base" -}} +{{ include "flytepropeller-userstorage" .}} storage: {{- if eq .Values.storage.type "s3" }} type: s3 diff --git a/charts/flyte-core/values-eks.yaml b/charts/flyte-core/values-eks.yaml index b05cd1f61cc..38e1e5429d2 100644 --- a/charts/flyte-core/values-eks.yaml +++ b/charts/flyte-core/values-eks.yaml @@ -5,6 +5,7 @@ userSettings: dbPassword: rdsHost: bucketName: + userBucketName: logGroup: redisHostUrl: redisHostKey: @@ -186,6 +187,8 @@ storage: type: s3 # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. bucketName: "{{ .Values.userSettings.bucketName }}" + # -- userBucketName can be the same as bucketName. It defines the bucket that Flyte will use to store Raw data generated by Tasks (Inputs/Outputs) + userBucketName: "{{ .Values.userSettings.userBucketName }}" s3: region: "{{ .Values.userSettings.accountRegion }}" diff --git a/charts/flyte-core/values-gcp.yaml b/charts/flyte-core/values-gcp.yaml index 4051fb23b92..f50ed6be813 100644 --- a/charts/flyte-core/values-gcp.yaml +++ b/charts/flyte-core/values-gcp.yaml @@ -6,6 +6,7 @@ userSettings: dbHost: dbPassword: bucketName: + userBucketName: hostName: # @@ -198,6 +199,8 @@ storage: type: gcs # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. bucketName: "{{ .Values.userSettings.bucketName }}" + # -- userBucketName can be the same as bucketName. It defines the bucket that Flyte will use to store Raw data generated by Tasks (Inputs/Outputs) + userBucketName: "{{ .Values.userSettings.userBucketName }}" # -- settings for storage type s3 gcs: # -- GCP project ID. Required for storage type gcs. diff --git a/charts/flyte-core/values.yaml b/charts/flyte-core/values.yaml index 6fdb7777639..304dde4fae9 100755 --- a/charts/flyte-core/values.yaml +++ b/charts/flyte-core/values.yaml @@ -444,6 +444,9 @@ storage: type: sandbox # -- bucketName defines the storage bucket flyte will use. Required for all types except for sandbox. bucketName: my-s3-bucket + #-- userBucketName is a required setting and defines a separate bucket to be used by Flyte to store all the Raw data produced by Tasks + #-- Use the same value as for bucketName to store metadata and raw data in the same bucket. + userBucketName: my-s3-bucket # -- settings for storage type s3 s3: region: us-east-1 @@ -666,7 +669,6 @@ configmap: pod-template-name: "flytepropeller-template" # -- follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). propeller: - rawoutput-prefix: s3://my-s3-bucket/ metadata-prefix: metadata/propeller workers: 4 max-workflow-retries: 30 diff --git a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml index 05fd72bdca0..0cbeff3eddf 100644 --- a/deployment/eks/flyte_aws_scheduler_helm_generated.yaml +++ b/deployment/eks/flyte_aws_scheduler_helm_generated.yaml @@ -179,6 +179,8 @@ data: signedUrls: durationMinutes: 3 storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -388,6 +390,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -500,6 +504,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -847,7 +853,7 @@ spec: template: metadata: annotations: - configChecksum: "2b5c85969f2bd85bb51a084f9fd72c20c3aca94be99e53cb4c4e9f78e77ebc5" + configChecksum: "4d5a990a82cbe032ea8d812a11455c556f0ffd5758f76de1754017c3d512e29" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -1135,7 +1141,7 @@ spec: template: metadata: annotations: - configChecksum: "59ef5b555bd41c3e854a315f21031c76dfa876455ff8069b989cb6c28ec1f17" + configChecksum: "8a96b2f36f440c1c0ecda8d41fe1ebf1acc9acb645e902553d20063cef472b6" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -1226,7 +1232,7 @@ spec: template: metadata: annotations: - configChecksum: "bc166f55ba3c1c28dd69e3ed5515185a366105541f3168dbb7eec901ff7fba8" + configChecksum: "97e208eee574e6c1cf055668193a3ef791e2e6c6f7598b7394992698ddd26ab" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1308,7 +1314,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b1 annotations: - configChecksum: "bc166f55ba3c1c28dd69e3ed5515185a366105541f3168dbb7eec901ff7fba8" + configChecksum: "97e208eee574e6c1cf055668193a3ef791e2e6c6f7598b7394992698ddd26ab" spec: securityContext: fsGroup: 65534 diff --git a/deployment/eks/flyte_helm_controlplane_generated.yaml b/deployment/eks/flyte_helm_controlplane_generated.yaml index 4eaea054ab5..b05a2f4abb4 100644 --- a/deployment/eks/flyte_helm_controlplane_generated.yaml +++ b/deployment/eks/flyte_helm_controlplane_generated.yaml @@ -160,6 +160,8 @@ data: signedUrls: durationMinutes: 3 storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -354,6 +356,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -553,7 +557,7 @@ spec: template: metadata: annotations: - configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f" + configChecksum: "a66378327daf755a8cea8f042c9985b075f4ff1b7d41035f441301083ae3355" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -841,7 +845,7 @@ spec: template: metadata: annotations: - configChecksum: "59ef5b555bd41c3e854a315f21031c76dfa876455ff8069b989cb6c28ec1f17" + configChecksum: "8a96b2f36f440c1c0ecda8d41fe1ebf1acc9acb645e902553d20063cef472b6" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -932,7 +936,7 @@ spec: template: metadata: annotations: - configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f" + configChecksum: "a66378327daf755a8cea8f042c9985b075f4ff1b7d41035f441301083ae3355" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte diff --git a/deployment/eks/flyte_helm_dataplane_generated.yaml b/deployment/eks/flyte_helm_dataplane_generated.yaml index 0b96d5abfae..1a01506c1f8 100644 --- a/deployment/eks/flyte_helm_dataplane_generated.yaml +++ b/deployment/eks/flyte_helm_dataplane_generated.yaml @@ -165,6 +165,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -427,7 +429,7 @@ spec: template: metadata: annotations: - configChecksum: "bc166f55ba3c1c28dd69e3ed5515185a366105541f3168dbb7eec901ff7fba8" + configChecksum: "97e208eee574e6c1cf055668193a3ef791e2e6c6f7598b7394992698ddd26ab" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -509,7 +511,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b1 annotations: - configChecksum: "bc166f55ba3c1c28dd69e3ed5515185a366105541f3168dbb7eec901ff7fba8" + configChecksum: "97e208eee574e6c1cf055668193a3ef791e2e6c6f7598b7394992698ddd26ab" spec: securityContext: fsGroup: 65534 diff --git a/deployment/eks/flyte_helm_generated.yaml b/deployment/eks/flyte_helm_generated.yaml index c198baf1a02..3a39a4fa794 100644 --- a/deployment/eks/flyte_helm_generated.yaml +++ b/deployment/eks/flyte_helm_generated.yaml @@ -191,6 +191,8 @@ data: signedUrls: durationMinutes: 3 storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -385,6 +387,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -531,6 +535,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: s3:///data storage: type: s3 container: "" @@ -878,7 +884,7 @@ spec: template: metadata: annotations: - configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f" + configChecksum: "a66378327daf755a8cea8f042c9985b075f4ff1b7d41035f441301083ae3355" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -1166,7 +1172,7 @@ spec: template: metadata: annotations: - configChecksum: "59ef5b555bd41c3e854a315f21031c76dfa876455ff8069b989cb6c28ec1f17" + configChecksum: "8a96b2f36f440c1c0ecda8d41fe1ebf1acc9acb645e902553d20063cef472b6" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -1257,7 +1263,7 @@ spec: template: metadata: annotations: - configChecksum: "053b20ebc40227f6ed8ddc61f5997ee7997c604158f773779f20ec61af11a2f" + configChecksum: "a66378327daf755a8cea8f042c9985b075f4ff1b7d41035f441301083ae3355" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte @@ -1345,7 +1351,7 @@ spec: template: metadata: annotations: - configChecksum: "bc166f55ba3c1c28dd69e3ed5515185a366105541f3168dbb7eec901ff7fba8" + configChecksum: "97e208eee574e6c1cf055668193a3ef791e2e6c6f7598b7394992698ddd26ab" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1427,7 +1433,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b1 annotations: - configChecksum: "bc166f55ba3c1c28dd69e3ed5515185a366105541f3168dbb7eec901ff7fba8" + configChecksum: "97e208eee574e6c1cf055668193a3ef791e2e6c6f7598b7394992698ddd26ab" spec: securityContext: fsGroup: 65534 diff --git a/deployment/gcp/flyte_helm_controlplane_generated.yaml b/deployment/gcp/flyte_helm_controlplane_generated.yaml index afb124e36a4..b027a27f8f2 100644 --- a/deployment/gcp/flyte_helm_controlplane_generated.yaml +++ b/deployment/gcp/flyte_helm_controlplane_generated.yaml @@ -162,6 +162,8 @@ data: namespace_mapping: template: '{{ domain }}' storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -364,6 +366,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -568,7 +572,7 @@ spec: template: metadata: annotations: - configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3" + configChecksum: "01ac368ead289483871595436b5db773d7f295ed87d111279a86aae652e2d7e" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -856,7 +860,7 @@ spec: template: metadata: annotations: - configChecksum: "bc69ed841506b28a42ac19bd0884d483472b3d11fe85fe7e546b879aeb30a85" + configChecksum: "05eacba2dc5c343c0a009d8fef2eab876dfce57f63fc4fbe823cb4c3d0d0790" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -947,7 +951,7 @@ spec: template: metadata: annotations: - configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3" + configChecksum: "01ac368ead289483871595436b5db773d7f295ed87d111279a86aae652e2d7e" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte diff --git a/deployment/gcp/flyte_helm_dataplane_generated.yaml b/deployment/gcp/flyte_helm_dataplane_generated.yaml index c8c00f23674..c2417481b62 100644 --- a/deployment/gcp/flyte_helm_dataplane_generated.yaml +++ b/deployment/gcp/flyte_helm_dataplane_generated.yaml @@ -165,6 +165,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -435,7 +437,7 @@ spec: template: metadata: annotations: - configChecksum: "d326dde27803fb8570a879090d4b9d49b249522825ee5c2360d1da4a76ca99b" + configChecksum: "d7b6d4313af0e02de8ab27efad03f018efa8a5180eff01057785ead569a2dd4" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -516,7 +518,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b1 annotations: - configChecksum: "d326dde27803fb8570a879090d4b9d49b249522825ee5c2360d1da4a76ca99b" + configChecksum: "d7b6d4313af0e02de8ab27efad03f018efa8a5180eff01057785ead569a2dd4" spec: securityContext: fsGroup: 65534 diff --git a/deployment/gcp/flyte_helm_generated.yaml b/deployment/gcp/flyte_helm_generated.yaml index 6e47caa492f..37c6185adf8 100644 --- a/deployment/gcp/flyte_helm_generated.yaml +++ b/deployment/gcp/flyte_helm_generated.yaml @@ -193,6 +193,8 @@ data: namespace_mapping: template: '{{ domain }}' storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -395,6 +397,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -544,6 +548,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: gs:///data storage: type: stow stow: @@ -901,7 +907,7 @@ spec: template: metadata: annotations: - configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3" + configChecksum: "01ac368ead289483871595436b5db773d7f295ed87d111279a86aae652e2d7e" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -1189,7 +1195,7 @@ spec: template: metadata: annotations: - configChecksum: "bc69ed841506b28a42ac19bd0884d483472b3d11fe85fe7e546b879aeb30a85" + configChecksum: "05eacba2dc5c343c0a009d8fef2eab876dfce57f63fc4fbe823cb4c3d0d0790" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -1280,7 +1286,7 @@ spec: template: metadata: annotations: - configChecksum: "2e169a911a8234dd42d06ca0887279093f4ed36033d0543749ce126b26b50f3" + configChecksum: "01ac368ead289483871595436b5db773d7f295ed87d111279a86aae652e2d7e" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte @@ -1368,7 +1374,7 @@ spec: template: metadata: annotations: - configChecksum: "d326dde27803fb8570a879090d4b9d49b249522825ee5c2360d1da4a76ca99b" + configChecksum: "d7b6d4313af0e02de8ab27efad03f018efa8a5180eff01057785ead569a2dd4" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -1449,7 +1455,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b1 annotations: - configChecksum: "d326dde27803fb8570a879090d4b9d49b249522825ee5c2360d1da4a76ca99b" + configChecksum: "d7b6d4313af0e02de8ab27efad03f018efa8a5180eff01057785ead569a2dd4" spec: securityContext: fsGroup: 65534 diff --git a/deployment/sandbox/flyte_helm_generated.yaml b/deployment/sandbox/flyte_helm_generated.yaml index 1464917a335..739c671a55f 100644 --- a/deployment/sandbox/flyte_helm_generated.yaml +++ b/deployment/sandbox/flyte_helm_generated.yaml @@ -312,6 +312,8 @@ data: signedUrls: durationMinutes: 3 storage.yaml: | + propeller: + rawoutput-prefix: storage: type: minio container: "my-s3-bucket" @@ -498,6 +500,8 @@ data: profiler-port: 10254 storage-prefix: metadata/datacatalog storage.yaml: | + propeller: + rawoutput-prefix: storage: type: minio container: "my-s3-bucket" @@ -666,6 +670,8 @@ data: resourcemanager: type: noop storage.yaml: | + propeller: + rawoutput-prefix: storage: type: minio container: "my-s3-bucket" @@ -6690,7 +6696,7 @@ spec: template: metadata: annotations: - configChecksum: "45f0232531c0d1494809cf83387a95b2fc802019ea095de7a24ccd4f8de86ec" + configChecksum: "ade654f94da9745666f0c6bed442bb54040c295c808e75a4d27bd3391fff38b" labels: app.kubernetes.io/name: flyteadmin app.kubernetes.io/instance: flyte @@ -6960,7 +6966,7 @@ spec: template: metadata: annotations: - configChecksum: "8feeaa7f7ec6506426db0d3e3cda6bc3ac0049a7eeba49d6cce62b3e1c7c424" + configChecksum: "ce036804ec3b92c47410a40cb3dacc06ee7c76c78360edc5480b50f0f21e221" labels: app.kubernetes.io/name: datacatalog app.kubernetes.io/instance: flyte @@ -7040,7 +7046,7 @@ spec: template: metadata: annotations: - configChecksum: "45f0232531c0d1494809cf83387a95b2fc802019ea095de7a24ccd4f8de86ec" + configChecksum: "ade654f94da9745666f0c6bed442bb54040c295c808e75a4d27bd3391fff38b" labels: app.kubernetes.io/name: flytescheduler app.kubernetes.io/instance: flyte @@ -7124,7 +7130,7 @@ spec: template: metadata: annotations: - configChecksum: "12a9f70a50c75d12a2a6990a7772c72326b1595f5cc7e2355bc21b51d139db5" + configChecksum: "88e59413d347cb54a425b94ffb42d4b19879ed6a952ca76a5b2bb6b7549a5af" labels: app.kubernetes.io/name: flytepropeller app.kubernetes.io/instance: flyte @@ -7198,7 +7204,7 @@ spec: app.kubernetes.io/name: flyte-pod-webhook app.kubernetes.io/version: v1.10.7-b1 annotations: - configChecksum: "12a9f70a50c75d12a2a6990a7772c72326b1595f5cc7e2355bc21b51d139db5" + configChecksum: "88e59413d347cb54a425b94ffb42d4b19879ed6a952ca76a5b2bb6b7549a5af" spec: securityContext: fsGroup: 65534