make helm

Signed-off-by: Kevin Su <[email protected]>

.gitattributes

@@ -1 +1,5 @@
 docs/**/*html linguist-generated=true
+flyteidl/gen/** linguist-generated=true
+flyteidl/protos/**/*.rst linguist-generated=true
+flyteidl/clients/go/assets/admin.swagger.json linguist-generated=true
+

charts/flyteagent/README.md

@@ -20,7 +20,7 @@ A Helm chart for Flyte agent
 | fullnameOverride | string | `""` |  |
 | image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | image.repository | string | `"ghcr.io/flyteorg/flyteagent"` | Docker image for flyteagent deployment |
-| image.tag | string | `"1.10.7"` | Docker image tag |
+| image.tag | string | `"1.10.8b4"` | Docker image tag |
 | nameOverride | string | `""` |  |
 | nodeSelector | object | `{}` | nodeSelector for flyteagent deployment |
 | podAnnotations | object | `{}` | Annotations for flyteagent pods |
@@ -30,6 +30,7 @@ A Helm chart for Flyte agent
 | ports.containerPort | int | `8000` |  |
 | ports.name | string | `"agent-grpc"` |  |
 | priorityClassName | string | `""` | Sets priorityClassName for datacatalog pod(s). |
+| readinessProbe | object | `{"grpc":{"port":8000},"initialDelaySeconds":1,"periodSeconds":3}` | https://kubernetes.io/blog/2022/05/13/grpc-probes-now-in-beta/#trying-the-feature-out |
 | replicaCount | int | `1` | Replicas count for flyteagent deployment |
 | resources | object | `{"limits":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"200Mi"},"requests":{"cpu":"500m","ephemeral-storage":"200Mi","memory":"200Mi"}}` | Default resources requests and limits for flyteagent deployment |
 | securityContext | object | `{"allowPrivilegeEscalation":false}` | Security context for container |

charts/flyteagent/templates/agent/deployment.yaml

@@ -39,6 +39,10 @@ spec:
         ports:
         - containerPort: {{ .Values.ports.containerPort }}
           name: {{ .Values.ports.name }}
+        readinessProbe:
+        {{- with .Values.readinessProbe -}}
+          {{ tpl (toYaml .) $ | nindent 10 }}
+        {{- end }}
         securityContext:
           {{- toYaml .Values.securityContext | nindent 12 }}
         resources: {{- toYaml .Values.resources | nindent 10 }}

charts/flyteagent/values.yaml

@@ -23,7 +23,7 @@ image:
   # -- Docker image for flyteagent deployment
   repository: ghcr.io/flyteorg/flyteagent
   # -- Docker image tag
-  tag: 1.10.7 # FLYTEAGENT_TAG
+  tag: 1.10.8b4 # FLYTEAGENT_TAG
   # -- Docker image pull policy
   pullPolicy: IfNotPresent
 ports:
@@ -56,6 +56,13 @@ serviceAccount:
   imagePullSecrets: []
 # -- Security context for pod
 podSecurityContext: {}
+# -- Readiness probe for flyteagent. Use readinessProbe: {} if agent doesn't implement grpc-health-checking service.
+# -- https://kubernetes.io/blog/2022/05/13/grpc-probes-now-in-beta/#trying-the-feature-out
+readinessProbe:
+  grpc:
+    port: 8000
+  initialDelaySeconds: 1
+  periodSeconds: 3
 # -- Security context for container
 securityContext:
   allowPrivilegeEscalation: false

deployment/agent/flyte_agent_helm_generated.yaml

@@ -78,7 +78,7 @@ spec:
         - pyflyte
         - serve
         - agent
-        image: "ghcr.io/flyteorg/flyteagent:1.10.7"
+        image: "ghcr.io/flyteorg/flyteagent:1.10.8b4"
         imagePullPolicy: "IfNotPresent"
         name: flyteagent
         volumeMounts:
@@ -87,6 +87,11 @@ spec:
         ports:
         - containerPort: 8000
           name: agent-grpc
+        readinessProbe:
+          grpc:
+            port: 8000
+          initialDelaySeconds: 1
+          periodSeconds: 3
         securityContext:
             allowPrivilegeEscalation: false
         resources:

docker/sandbox-bundled/manifests/complete-agent.yaml

@@ -816,7 +816,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: RjRjV21uckEzcFZzWDRBQw==
+  haSharedSecret: ZVNtQlpWSTRkZnFtRVp0cw==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1412,7 +1412,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 680ea727017c7841261388cfc9f48a11cac5b0a9055bfb43b5e596c1efb9b1eb
+        checksum/secret: e540906f7530a7b8f9c9e7488bfe922999294201b1f0f75ab310162ef6441d96
       labels:
         app: docker-registry
         release: flyte-sandbox
@@ -1755,12 +1755,17 @@ spec:
           value: minio
         - name: FLYTE_AWS_SECRET_ACCESS_KEY
           value: miniostorage
-        image: ghcr.io/flyteorg/flyteagent:1.10.7
+        image: ghcr.io/flyteorg/flyteagent:1.10.8b4
         imagePullPolicy: IfNotPresent
         name: flyteagent
         ports:
         - containerPort: 8000
           name: agent-grpc
+        readinessProbe:
+          grpc:
+            port: 8000
+          initialDelaySeconds: 1
+          periodSeconds: 3
         resources:
           limits:
             cpu: 500m

docker/sandbox-bundled/manifests/complete.yaml

@@ -796,7 +796,7 @@ type: Opaque
 ---
 apiVersion: v1
 data:
-  haSharedSecret: OXNiUmdaZ0lkaGNySUxlcg==
+  haSharedSecret: VnRRWDlOZXVWcXNmZFRXRQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -1360,7 +1360,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: b001cf9b50dbaa84325834d7000859a7715c70d322563b785bdb027a3922d2c7
+        checksum/secret: 1a60b7cc9612c65c9a369c55629f6054dfed24c40b064269274d0fbb7c6d0823
       labels:
         app: docker-registry
         release: flyte-sandbox

docker/sandbox-bundled/manifests/dev.yaml

@@ -499,7 +499,7 @@ metadata:
 ---
 apiVersion: v1
 data:
-  haSharedSecret: bkhERElpYjVvU1lKVTRyUg==
+  haSharedSecret: dlY1cWxrMFBJOU8yazFyZQ==
   proxyPassword: ""
   proxyUsername: ""
 kind: Secret
@@ -934,7 +934,7 @@ spec:
     metadata:
       annotations:
         checksum/config: 8f50e768255a87f078ba8b9879a0c174c3e045ffb46ac8723d2eedbe293c8d81
-        checksum/secret: 636bca82951124e8b2191a43c5ec9e05a45c60efb0adeea909f5a3427bea570f
+        checksum/secret: b86db743a29c0ae883fd1fd39fb1a6e81288c8451f3ebac001c7f2f1893d3f2a
       labels:
         app: docker-registry
         release: flyte-sandbox

docs/deployment/configuration/general.rst

@@ -202,10 +202,10 @@ An example PodTemplate is shown:
     template:
       metadata:
         labels:
-          - foo
+          foo: from-pod-template
         annotations:
-          - foo: initial-value
-          - bar: initial-value
+          foo: initial-value
+          bar: initial-value
       spec:
         containers:
           - name: default
@@ -221,10 +221,10 @@ Pod Labels, Annotations, and enables the host networking.
     plugins:
        k8s:
         default-labels:
-          - bar
+          bar: from-default-label
         default-annotations:
-          - foo: overridden-value
-          - baz: non-overridden-value
+          foo: overridden-value
+          baz: non-overridden-value
         enable-host-networking-pod: true
 
 To construct a Pod, FlytePropeller initializes a Pod definition using the default
@@ -241,19 +241,19 @@ The resultant Pod using the above default PodTemplate and K8s Plugin configurati
       name: example-pod
       namespace: flytesnacks-development
       labels:
-        - foo // maintained initial value
-        - bar // value appended by k8s plugin configuration
+        foo: from-pod-template # maintained initial value
+        bar: from-default-label # value appended by k8s plugin configuration
       annotations:
-        - foo: overridden-value // value overridden by k8s plugin configuration
-        - bar: initial-value // maintained initial value
-        - baz: non-overridden-value // value added by k8s plugin configuration
+        foo: overridden-value # value overridden by k8s plugin configuration
+        bar: initial-value # maintained initial value
+        baz: non-overridden-value # value added by k8s plugin configuration
     spec:
       containers:
         - name: ax9kd5xb4p8r45bpdv7v-n0-0
           image: ghcr.io/flyteorg/flytecookbook:core-bfee7e549ad749bfb55922e130f4330a0ebc25b0
           terminationMessagePath: "/dev/foo"
-          // remaining container configuration omitted
-      hostNetwork: true // overridden by the k8s plugin configuration
+          # remaining container configuration omitted
+      hostNetwork: true # overridden by the k8s plugin configuration
 
 The last step in constructing a Pod is to apply any task-specific configuration.
 These options follow the same rules as merging the default PodTemplate and K8s
@@ -280,8 +280,8 @@ of the task. For example:
     template:
       metadata:
         annotations:
-          - annotation_1: initial-value
-          - bar: initial-value
+          annotation_1: initial-value
+          bar: initial-value
       spec:
         containers:
           - name: default
@@ -328,12 +328,12 @@ The resultant Pod is as follows:
       name: example-pod
       namespace: flytesnacks-development
       labels:
-        - label_1: value-1  # from Compile-time value
-        - label_2: value-2  # from Compile-time value
+        label_1: value-1  # from Compile-time value
+        label_2: value-2  # from Compile-time value
       annotations:
-        - annotation_1: value-1  # value overridden by Compile-time PodTemplate
-        - annotation_2: value-2  # from Compile-time PodTemplate
-        - bar: initial-value  # from Runtime PodTemplate
+        annotation_1: value-1  # value overridden by Compile-time PodTemplate
+        annotation_2: value-2  # from Compile-time PodTemplate
+        bar: initial-value  # from Runtime PodTemplate
     spec:
       containers:
         - name: default
@@ -343,7 +343,7 @@ The resultant Pod is as follows:
           image: a.b.c/image:v1
           command: cmd
           args: []
-          // remaining container configuration omitted
+          # remaining container configuration omitted
 
 Notice how options follow the same merging rules, i.e. lists append and maps override.
 
@@ -398,12 +398,12 @@ And a Runtime PodTemplate:
     template:
       metadata:
         labels:
-          - label_1: value-runtime
-          - label_2: value-runtime
-          - label_3: value-runtime
+          label_1: value-runtime
+          label_2: value-runtime
+          label_3: value-runtime
         annotations:
-          - foo: value-runtime
-          - bar: value-runtime
+          foo: value-runtime
+          bar: value-runtime
       spec:
         containers:
           - name: default
@@ -418,10 +418,10 @@ And the following K8s Plugin Configuration:
     plugins:
        k8s:
         default-labels:
-          - label_1: value-plugin
+          label_1: value-plugin
         default-annotations:
-          - annotation_1: value-plugin
-          - baz: value-plugin
+          annotation_1: value-plugin
+          baz: value-plugin
 
 The resultant pod for that task is as follows:
 
@@ -433,14 +433,14 @@ The resultant pod for that task is as follows:
       name: example-pod
       namespace: flytesnacks-development
       labels:
-        - label_1: value-plugin
-        - label_2: value-compile
+        label_1: value-plugin
+        label_2: value-compile
       annotations:
-        - annotation_1: value-plugin
-        - annotation_2: value-compile
-        - foo: value-runtime
-        - bar: value-runtime
-        - baz: value-plugin
+        annotation_1: value-plugin
+        annotation_2: value-compile
+        foo: value-runtime
+        bar: value-runtime
+        baz: value-plugin
     spec:
       containers:
         - name: default
@@ -450,4 +450,4 @@ The resultant pod for that task is as follows:
           image: a.b.c/image:v1
           command: cmd
           args: []
-          // remaining container configuration omitted
+          # remaining container configuration omitted

flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context.go

@@ -5,6 +5,7 @@ import (
 	pluginsCore "github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/core"
 	"github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/utils"
 	"github.com/flyteorg/flyte/flyteplugins/go/tasks/pluginmachinery/utils/secrets"
+	k8sUtils "github.com/flyteorg/flyte/flytepropeller/pkg/utils"
 )
 
 const executionIdentityVariable = "execution-identity"
@@ -60,7 +61,8 @@ func newTaskExecutionMetadata(tCtx pluginsCore.TaskExecutionMetadata, taskTmpl *
 
 	id := tCtx.GetSecurityContext().RunAs.ExecutionIdentity
 	if len(id) > 0 {
-		injectLabels[executionIdentityVariable] = id
+		sanitizedID := k8sUtils.SanitizeLabelValue(id)
+		injectLabels[executionIdentityVariable] = sanitizedID
 	}
 
 	return TaskExecutionMetadata{

flytepropeller/pkg/controller/nodes/task/k8s/task_exec_context_test.go

@@ -86,6 +86,25 @@ func Test_newTaskExecutionMetadata(t *testing.T) {
 		assert.Equal(t, 2, len(actual.GetLabels()))
 		assert.Equal(t, "test-exec-identity", actual.GetLabels()[executionIdentityVariable])
 	})
+	t.Run("Inject exec identity K8s label sanitation", func(t *testing.T) {
+
+		existingMetadata := &mocks.TaskExecutionMetadata{}
+		existingAnnotations := map[string]string{}
+		existingMetadata.OnGetAnnotations().Return(existingAnnotations)
+
+		existingMetadata.OnGetSecurityContext().Return(core.SecurityContext{RunAs: &core.Identity{ExecutionIdentity: "[email protected]"}})
+
+		existingLabels := map[string]string{
+			"existingLabel": "existingLabelValue",
+		}
+		existingMetadata.OnGetLabels().Return(existingLabels)
+
+		actual, err := newTaskExecutionMetadata(existingMetadata, &core.TaskTemplate{})
+		assert.NoError(t, err)
+
+		assert.Equal(t, 2, len(actual.GetLabels()))
+		assert.Equal(t, "name-company-com", actual.GetLabels()[executionIdentityVariable])
+	})
 }
 
 func Test_newTaskExecutionContext(t *testing.T) {