Merge branch 'master' of github.com:flyteorg/flyte into agent-secret

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,11 +1,13 @@
 ## Tracking issue
-_https://github.com/flyteorg/flyte/issues/<number>_
-
-<!-- If your PR fixes an open issue, use `Closes #999` to link your PR with the issue. #999 stands for the issue number you are fixing -->
+<!--
+If your PR fixes an open issue, use `Closes #999` to link your PR with the issue.
+Example: Closes #999
 
-<!-- Remove this section if not applicable -->
+If your PR is related to an issue or PR, use `Related to #999` to link your PR.
+Example: Related to #999
 
-<!-- Example: Closes #31 -->
+Remove this section if not applicable
+-->
 
 ## Why are the changes needed?
 

.github/workflows/checks.yml

@@ -61,7 +61,7 @@ jobs:
       component: ${{ matrix.component }}
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
     secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   docker-build:
     strategy:
       fail-fast: false

.github/workflows/codespell.yml

@@ -26,3 +26,4 @@ jobs:
         uses: codespell-project/actions-codespell@v2
         with:
           skip: "*.pb,monodocs-environment.lock.yaml"
+          ignore_words_list: crate

.github/workflows/flyteidl-checks.yml

@@ -39,7 +39,7 @@ jobs:
       component: flyteidl
       go-version: ${{ needs.unpack-envvars.outputs.go-version }}
     secrets:
-      FLYTE_BOT_PAT: ${{ secrets.FLYTE_BOT_PAT }}
+      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
   generate:
     name: Check Go Generate
     needs:

.github/workflows/tests.yml

@@ -44,6 +44,7 @@ jobs:
         working-directory: ${{ github.workspace }}/flyte
         run: |
           conda install -c conda-forge conda-lock
+          cat monodocs-environment.lock.yaml
           conda-lock install -n monodocs-env monodocs-environment.lock.yaml
       - shell: bash -el {0}
         working-directory: ${{ github.workspace }}/flyte

.github/workflows/unit-tests.yml

@@ -10,7 +10,7 @@ on:
         required: true
         type: string
     secrets:
-      FLYTE_BOT_PAT:
+      CODECOV_TOKEN:
         required: true
 jobs:
   tests:
@@ -27,15 +27,11 @@ jobs:
         with:
           go-version: ${{ inputs.go-version }}
       - name: Unit Tests
-        env:
-          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
         run: make install && make test_unit_codecov
-      # TODO(monorepo): Figure out how to do code coverage aggregation
       - name: Push CodeCov
-        uses: codecov/[email protected]
-        env:
-          GITHUB_TOKEN: ${{ secrets.FLYTE_BOT_PAT }}
+        uses: codecov/codecov-action@v4
         with:
           file: coverage.txt
-          flags: unittests
+          flags: unittests-${{ inputs.component }}
           fail_ci_if_error: false
+          token: ${{ secrets.CODECOV_TOKEN }}

.gitignore

@@ -2,6 +2,7 @@
 _build/
 _bin/
 build/
+.tmp_build/
 .vscode/
 *.swp
 *.swo

.readthedocs.yml

@@ -10,6 +10,7 @@ build:
     python: "mambaforge-22.9"
   jobs:
     post_install:
+      - cat monodocs-environment.lock.yaml
       - conda-lock install --name $READTHEDOCS_VERSION monodocs-environment.lock.yaml
       - pip install ./flyteidl
       - conda info

Makefile

@@ -11,6 +11,7 @@ GIT_HASH := $(shell git rev-parse --short HEAD)
 TIMESTAMP := $(shell date '+%Y-%m-%d')
 PACKAGE ?=github.com/flyteorg/flytestdlib
 LD_FLAGS="-s -w -X $(PACKAGE)/version.Version=$(GIT_VERSION) -X $(PACKAGE)/version.Build=$(GIT_HASH) -X $(PACKAGE)/version.BuildTime=$(TIMESTAMP)"
+TMP_BUILD_DIR := .tmp_build
 
 .PHONY: cmd/single/dist
 cmd/single/dist: export FLYTECONSOLE_VERSION ?= latest
@@ -43,7 +44,7 @@ release_automation:
 	$(MAKE) -C docker/sandbox-bundled manifests
 
 .PHONY: deploy_sandbox
-deploy_sandbox: 
+deploy_sandbox:
 	bash script/deploy.sh
 
 .PHONY: install-piptools
@@ -82,10 +83,30 @@ helm_install: ## Install helm charts
 helm_upgrade: ## Upgrade helm charts
 	helm upgrade flyte --debug ./charts/flyte -f ./charts/flyte/values.yaml --create-namespace --namespace=flyte
 
+# Used in CI
 .PHONY: docs
 docs:
 	make -C docs clean html SPHINXOPTS=-W
 
+$(TMP_BUILD_DIR):
+	mkdir $@
+
+$(TMP_BUILD_DIR)/conda-lock-image: docs/Dockerfile.conda-lock | $(TMP_BUILD_DIR)
+	docker buildx build --load --platform=linux/amd64 --build-arg USER_UID=$$(id -u) --build-arg USER_GID=$$(id -g) -t flyte-conda-lock:latest -f docs/Dockerfile.conda-lock .
+	touch $(TMP_BUILD_DIR)/conda-lock-image
+
+monodocs-environment.lock.yaml: monodocs-environment.yaml $(TMP_BUILD_DIR)/conda-lock-image
+	docker run --platform=linux/amd64 --rm --pull never -v ./:/flyte flyte-conda-lock:latest lock --file monodocs-environment.yaml --lockfile monodocs-environment.lock.yaml
+
+$(TMP_BUILD_DIR)/dev-docs-image: docs/Dockerfile.docs monodocs-environment.lock.yaml | $(TMP_BUILD_DIR)
+	docker buildx build --load --platform=linux/amd64 --build-arg USER_UID=$$(id -u) --build-arg USER_GID=$$(id -g) -t flyte-dev-docs:latest -f docs/Dockerfile.docs .
+	touch $(TMP_BUILD_DIR)/dev-docs-image
+
+# Build docs in docker container for local development
+.PHONY: dev-docs
+dev-docs: $(TMP_BUILD_DIR)/dev-docs-image
+	bash script/local_build_docs.sh
+
 .PHONY: help
 help: SHELL := /bin/sh
 help: ## List available commands and their usage
@@ -121,5 +142,5 @@ lint-helm-charts:
 	act pull_request -W .github/workflows/validate-helm-charts.yaml --container-architecture linux/amd64 -e charts/event.json
 
 .PHONY: clean
-clean: ## Remove the HTML files related to the Flyteconsole.
-	rm -rf cmd/single/dist
+clean: ## Remove the HTML files related to the Flyteconsole and Makefile
+	rm -rf cmd/single/dist .tmp_build

boilerplate/flyte/golang_test_targets/Makefile

@@ -54,7 +54,6 @@ test_unit_visual:
 .PHONY: test_unit_codecov
 test_unit_codecov:
 	go test ./... -race -coverprofile=coverage.txt -covermode=atomic
-	curl -s https://codecov.io/bash > codecov_bash.sh && bash codecov_bash.sh
 
 .PHONY: go-tidy
 go-tidy:

charts/flyte-binary/README.md

@@ -42,7 +42,7 @@ Chart for basic single Flyte executable deployment
 | configuration.auth.oidc.clientId | string | `""` |  |
 | configuration.auth.oidc.clientSecret | string | `""` |  |
 | configuration.co-pilot.image.repository | string | `"cr.flyte.org/flyteorg/flytecopilot"` |  |
-| configuration.co-pilot.image.tag | string | `"v1.11.1-b1"` |  |
+| configuration.co-pilot.image.tag | string | `"v1.12.0-b0"` |  |
 | configuration.database.dbname | string | `"flyte"` |  |
 | configuration.database.host | string | `"127.0.0.1"` |  |
 | configuration.database.options | string | `"sslmode=disable"` |  |

charts/flyte-binary/values.yaml

@@ -159,7 +159,7 @@ configuration:
       # repository CoPilot sidecar image repository
       repository: cr.flyte.org/flyteorg/flytecopilot  # FLYTECOPILOT_IMAGE
       # tag CoPilot sidecar image tag
-      tag: v1.11.1-b1  # FLYTECOPILOT_TAG
+      tag: v1.12.0-b0  # FLYTECOPILOT_TAG
   # agentService Flyte Agent configuration
   agentService:
     defaultAgent:

charts/flyte-core/README.md

@@ -95,8 +95,8 @@ helm install gateway bitnami/contour -n flyte
 | configmap.clusters.clusterConfigs | list | `[]` |  |
 | configmap.clusters.labelClusterMap | object | `{}` |  |
 | configmap.console | object | `{"BASE_URL":"/console","CONFIG_DIR":"/etc/flyte/config"}` | Configuration for Flyte console UI |
-| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
-| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
+| configmap.copilot | object | `{"plugins":{"k8s":{"co-pilot":{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.12.0-b0","name":"flyte-copilot-","start-timeout":"30s"}}}}` | Copilot configuration |
+| configmap.copilot.plugins.k8s.co-pilot | object | `{"image":"cr.flyte.org/flyteorg/flytecopilot:v1.12.0-b0","name":"flyte-copilot-","start-timeout":"30s"}` | Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig) |
 | configmap.core | object | `{"manager":{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"},"propeller":{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"},"webhook":{"certDir":"/etc/webhook/certs","serviceName":"flyte-pod-webhook"}}` | Core propeller configuration |
 | configmap.core.manager | object | `{"pod-application":"flytepropeller","pod-template-container-name":"flytepropeller","pod-template-name":"flytepropeller-template"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/manager/config#Config). |
 | configmap.core.propeller | object | `{"downstream-eval-duration":"30s","enable-admin-launcher":true,"leader-election":{"enabled":true,"lease-duration":"15s","lock-config-map":{"name":"propeller-leader","namespace":"flyte"},"renew-deadline":"10s","retry-period":"2s"},"limit-namespace":"all","max-workflow-retries":30,"metadata-prefix":"metadata/propeller","metrics-prefix":"flyte","prof-port":10254,"queue":{"batch-size":-1,"batching-interval":"2s","queue":{"base-delay":"5s","capacity":1000,"max-delay":"120s","rate":100,"type":"maxof"},"sub-queue":{"capacity":100,"rate":10,"type":"bucket"},"type":"batch"},"rawoutput-prefix":"s3://my-s3-bucket/","workers":4,"workflow-reeval-duration":"30s"}` | follows the structure specified [here](https://pkg.go.dev/github.com/flyteorg/flytepropeller/pkg/controller/config). |
@@ -130,7 +130,7 @@ helm install gateway bitnami/contour -n flyte
 | datacatalog.extraArgs | object | `{}` | Appends extra command line arguments to the main command |
 | datacatalog.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | datacatalog.image.repository | string | `"cr.flyte.org/flyteorg/datacatalog"` | Docker image for Datacatalog deployment |
-| datacatalog.image.tag | string | `"v1.11.1-b1"` | Docker image tag |
+| datacatalog.image.tag | string | `"v1.12.0-b0"` | Docker image tag |
 | datacatalog.nodeSelector | object | `{}` | nodeSelector for Datacatalog deployment |
 | datacatalog.podAnnotations | object | `{}` | Annotations for Datacatalog pods |
 | datacatalog.podEnv | object | `{}` | Additional Datacatalog container environment variables |
@@ -166,7 +166,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteadmin.extraArgs | object | `{}` | Appends extra command line arguments to the serve command |
 | flyteadmin.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flyteadmin.image.repository | string | `"cr.flyte.org/flyteorg/flyteadmin"` | Docker image for Flyteadmin deployment |
-| flyteadmin.image.tag | string | `"v1.11.1-b1"` |  |
+| flyteadmin.image.tag | string | `"v1.12.0-b0"` |  |
 | flyteadmin.initialProjects | list | `["flytesnacks","flytetester","flyteexamples"]` | Initial projects to create |
 | flyteadmin.nodeSelector | object | `{}` | nodeSelector for Flyteadmin deployment |
 | flyteadmin.podAnnotations | object | `{}` | Annotations for Flyteadmin pods |
@@ -204,7 +204,7 @@ helm install gateway bitnami/contour -n flyte
 | flyteconsole.ga.tracking_id | string | `"G-0QW4DJWJ20"` |  |
 | flyteconsole.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flyteconsole.image.repository | string | `"cr.flyte.org/flyteorg/flyteconsole"` | Docker image for Flyteconsole deployment |
-| flyteconsole.image.tag | string | `"v1.10.3"` |  |
+| flyteconsole.image.tag | string | `"v1.12.2"` |  |
 | flyteconsole.imagePullSecrets | list | `[]` | ImagePullSecrets to assign to the Flyteconsole deployment |
 | flyteconsole.nodeSelector | object | `{}` | nodeSelector for Flyteconsole deployment |
 | flyteconsole.podAnnotations | object | `{}` | Annotations for Flyteconsole pods |
@@ -228,7 +228,7 @@ helm install gateway bitnami/contour -n flyte
 | flytepropeller.extraArgs | object | `{}` | Appends extra command line arguments to the main command |
 | flytepropeller.image.pullPolicy | string | `"IfNotPresent"` |  |
 | flytepropeller.image.repository | string | `"cr.flyte.org/flyteorg/flytepropeller"` | Docker image for Flytepropeller deployment |
-| flytepropeller.image.tag | string | `"v1.11.1-b1"` |  |
+| flytepropeller.image.tag | string | `"v1.12.0-b0"` |  |
 | flytepropeller.manager | bool | `false` |  |
 | flytepropeller.nodeSelector | object | `{}` | nodeSelector for Flytepropeller deployment |
 | flytepropeller.podAnnotations | object | `{}` | Annotations for Flytepropeller pods |
@@ -259,7 +259,7 @@ helm install gateway bitnami/contour -n flyte
 | flytescheduler.configPath | string | `"/etc/flyte/config/*.yaml"` | Default regex string for searching configuration files |
 | flytescheduler.image.pullPolicy | string | `"IfNotPresent"` | Docker image pull policy |
 | flytescheduler.image.repository | string | `"cr.flyte.org/flyteorg/flytescheduler"` | Docker image for Flytescheduler deployment |
-| flytescheduler.image.tag | string | `"v1.11.1-b1"` | Docker image tag |
+| flytescheduler.image.tag | string | `"v1.12.0-b0"` | Docker image tag |
 | flytescheduler.nodeSelector | object | `{}` | nodeSelector for Flytescheduler deployment |
 | flytescheduler.podAnnotations | object | `{}` | Annotations for Flytescheduler pods |
 | flytescheduler.podEnv | object | `{}` | Additional Flytescheduler container environment variables |

charts/flyte-core/values.yaml

@@ -16,7 +16,7 @@ flyteadmin:
   image:
     # -- Docker image for Flyteadmin deployment
     repository: cr.flyte.org/flyteorg/flyteadmin  # FLYTEADMIN_IMAGE
-    tag: v1.11.1-b1  # FLYTEADMIN_TAG
+    tag: v1.12.0-b0  # FLYTEADMIN_TAG
     pullPolicy: IfNotPresent
   # -- Additional flyteadmin container environment variables
   #
@@ -144,7 +144,7 @@ flytescheduler:
     # -- Docker image for Flytescheduler deployment
     repository: cr.flyte.org/flyteorg/flytescheduler  # FLYTESCHEDULER_IMAGE
     # -- Docker image tag
-    tag: v1.11.1-b1  # FLYTESCHEDULER_TAG
+    tag: v1.12.0-b0  # FLYTESCHEDULER_TAG
     # -- Docker image pull policy
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytescheduler deployment
@@ -210,7 +210,7 @@ datacatalog:
     # -- Docker image for Datacatalog deployment
     repository: cr.flyte.org/flyteorg/datacatalog  # DATACATALOG_IMAGE
     # -- Docker image tag
-    tag: v1.11.1-b1  # DATACATALOG_TAG
+    tag: v1.12.0-b0  # DATACATALOG_TAG
     # -- Docker image pull policy
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Datacatalog deployment
@@ -300,7 +300,7 @@ flytepropeller:
   image:
     # -- Docker image for Flytepropeller deployment
     repository: cr.flyte.org/flyteorg/flytepropeller  # FLYTEPROPELLER_IMAGE
-    tag: v1.11.1-b1  # FLYTEPROPELLER_TAG
+    tag: v1.12.0-b0  # FLYTEPROPELLER_TAG
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flytepropeller deployment
   resources:
@@ -385,7 +385,7 @@ flyteconsole:
   image:
     # -- Docker image for Flyteconsole deployment
     repository: cr.flyte.org/flyteorg/flyteconsole  # FLYTECONSOLE_IMAGE
-    tag: v1.10.3  # FLYTECONSOLE_TAG
+    tag: v1.12.2  # FLYTECONSOLE_TAG
     pullPolicy: IfNotPresent
   # -- Default resources requests and limits for Flyteconsole deployment
   resources:
@@ -743,7 +743,7 @@ configmap:
         # -- Structure documented [here](https://pkg.go.dev/github.com/lyft/[email protected]/go/tasks/pluginmachinery/flytek8s/config#FlyteCoPilotConfig)
         co-pilot:
           name: flyte-copilot-
-          image: cr.flyte.org/flyteorg/flytecopilot:v1.11.1-b1  # FLYTECOPILOT_IMAGE
+          image: cr.flyte.org/flyteorg/flytecopilot:v1.12.0-b0  # FLYTECOPILOT_IMAGE
           start-timeout: 30s
 
   # -- Core propeller configuration