Merge branch 'flyteorg:master' into master

.github/actions/clear-action-cache/action.yml

@@ -0,0 +1,11 @@
+name: 'Clear action cache'
+description: 'As suggested by GitHub to prevent low disk space: https://github.com/actions/runner-images/issues/2840#issuecomment-790492173'
+runs:
+  using: 'composite'
+  steps:
+    - shell: bash
+      run: |
+        rm -rf /usr/share/dotnet
+        rm -rf /opt/ghc
+        rm -rf "/usr/local/share/boost"
+        rm -rf "$AGENT_TOOLSDIRECTORY"

.github/workflows/checks.yml

@@ -10,7 +10,7 @@ on:
     branches:
       - master
 env:
-  GO_VERSION: "1.21"
+  GO_VERSION: "1.22"
   PRIORITIES: "P0"
 jobs:
   unpack-envvars:

.github/workflows/create_release.yml

@@ -28,7 +28,6 @@ jobs:
               "datacatalog",
               "flyteadmin",
               "flytecopilot",
-              "flyteidl",
               "flyteplugins",
               "flytepropeller",
               "flytestdlib",

.github/workflows/flytectl-release.yml

@@ -1,13 +1,34 @@
 name: Flytectl release
 
 on:
-  push:
-    tags:
-      - flytectl/v*.*.*
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "version. Do *not* use the `flytectl/` prefix, e.g. `flytectl/v1.2.3`, instead use only `v1.2.3` (including the `v`)"
+        required: true
 
 jobs:
+  push-flytectl-tag:
+    name: Push git tag containing the `flyteidl/` prefix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.FLYTE_BOT_PAT }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/flytectl/${{ github.event.inputs.version }}`,
+              sha: context.sha
+            })
   release:
     name: Goreleaser
+    needs:
+      - push-flytectl-tag
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.github/workflows/flyteidl-checks.yml

@@ -10,7 +10,7 @@ on:
     branches:
       - master
 env:
-  GO_VERSION: "1.21"
+  GO_VERSION: "1.22"
 jobs:
   unpack-envvars:
     runs-on: ubuntu-latest

.github/workflows/flyteidl-release.yml

@@ -1,18 +1,41 @@
-name: Upload flyteidl to PyPI and npm
+name: Release flyteidl
 
 on:
-  push:
-    tags:
-      - flyteidl/v*.*.*
+  workflow_dispatch:
+    inputs:
+      version:
+        description: "version. Do *not* use the `flyteidl/` prefix, e.g. `flyteidl/v1.2.3`, instead use only `v1.2.3` (including the `v`)"
+        required: true
 
 jobs:
+  push-flyteidl-tag:
+    name: Push git tag containing the `flyteidl/` prefix
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
+      - uses: actions/github-script@v6
+        with:
+          github-token: ${{ secrets.FLYTE_BOT_PAT }}
+          script: |
+            github.rest.git.createRef({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              ref: `refs/tags/flyteidl/${{ github.event.inputs.version }}`,
+              sha: context.sha
+            })
   deploy-to-pypi:
+    needs:
+      - push-flyteidl-tag
     runs-on: ubuntu-latest
     defaults:
       run:
         working-directory: flyteidl
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: '0'
       - name: Set up Python
         uses: actions/setup-python@v1
         with:
@@ -29,6 +52,8 @@ jobs:
           python -m build
           twine upload dist/*
   deploy-to-npm:
+    needs:
+      - push-flyteidl-tag
     runs-on: ubuntu-latest
     defaults:
       run:
@@ -41,8 +66,8 @@ jobs:
           registry-url: "https://registry.npmjs.org"
       - name: Set version in npm package
         run: |
-          # from refs/tags/v1.2.3 get 1.2.3
-          VERSION=$(echo $GITHUB_REF | sed 's#.*/v##')
+          # v1.2.3 get 1.2.3
+          VERSION=$(echo ${{ inputs.version }} | sed 's#.*v##')
           VERSION=$VERSION make update_npmversion
         shell: bash
       - run: |

.github/workflows/generate_flyte_manifest.yml

@@ -15,9 +15,9 @@ jobs:
       - uses: actions/checkout@v4
         with:
           fetch-depth: "0"
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.22"
       - name: Update references
         env:
           VERSION: ${{ github.event.inputs.next-version }}

.github/workflows/go_generate.yml

@@ -21,13 +21,13 @@ jobs:
         working-directory: ${{ inputs.component }}
     steps:
       - uses: actions/checkout@v4
-      - uses: arduino/setup-protoc@v1
+      - uses: arduino/setup-protoc@v3
         with:
           repo-token: ${{ secrets.FLYTE_BOT_PAT }}
       - uses: bufbuild/buf-setup-action@v1
         with:
           github_token: ${{ secrets.FLYTE_BOT_PAT }}
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ inputs.go-version }}
       - name: Go generate and diff

.github/workflows/integration.yml

@@ -37,7 +37,7 @@ jobs:
         with:
           version: "v0.11.1"
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ inputs.go-version }}
       - name: Integration

.github/workflows/lint.yml

@@ -20,7 +20,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ inputs.go-version }}
       - name: Lint

.github/workflows/single-binary.yml

@@ -17,9 +17,9 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.22"
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -175,7 +175,7 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install uv
-          uv pip install --system flytekit flytekitplugins-deck-standard flytekitplugins-envd "numpy<2.0.0"
+          uv pip install --system flytekit flytekitplugins-deck-standard flytekitplugins-envd "numpy<2.0.0" pyarrow
           uv pip freeze
       - name: Checkout flytesnacks
         uses: actions/checkout@v4

.github/workflows/tests.yml

@@ -16,9 +16,9 @@ jobs:
       - name: Fetch the code
         uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.22"
       - name: Compile
         run: make compile
       - name: Run tests
@@ -30,12 +30,14 @@ jobs:
       - name: Fetch flyte code
         uses: actions/checkout@v4
         with:
-          path: "${{ github.workspace }}/flyte"
+          path: flyte
+      - name: 'Clear action cache'
+        uses: ./flyte/.github/actions/clear-action-cache
       - name: Fetch flytekit code
         uses: actions/checkout@v4
         with:
           repository: flyteorg/flytekit
-          path: "${{ github.workspace }}/flytekit"
+          path: flytekit
       - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
@@ -75,8 +77,8 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-go@v4
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.21"
+          go-version: "1.22"
       - name: Helm and diff
         run: DELTA_CHECK=true make helm

.github/workflows/unit-tests.yml

@@ -23,7 +23,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Set up Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: ${{ inputs.go-version }}
       - name: Unit Tests

CHANGELOG/CHANGELOG-v0.14.0.md

@@ -1,7 +1,7 @@
 # Flyte v0.14.0
 
 ## Platform
-- Update to the Flyte Compiler, to supported nested branches and more. Better
+- Update to the Flyte Compiler, to support nested branches and more. Better
   regression tests
 - support for iam roles and k8s serviceaccounts
 - customizable pod specs for pod tasks (add labels and annotations)

CHANGELOG/CHANGELOG-v0.17.0.md

@@ -8,7 +8,7 @@
 
 ## Flytekit
 1. Great Expectations Integration ([docs](https://docs.flyte.org/en/latest/flytesnacks/examples/greatexpectations_plugin/index.html)).
-1. Access to durable blob stores (AWS/GCS/etc) are now pluggable.
+1. Access to durable blob stores (AWS/GCS/etc) is now pluggable.
 1. Local task execution has been updated to also trigger the type engine.
 1. Tasks that have `cache=True` should now be cached when running locally as well ([docs](https://docs.flyte.org/en/latest/user_guide/development_lifecycle/caching.html#how-does-local-caching-work)).
 

CHANGELOG/CHANGELOG-v0.19.4.md

@@ -17,7 +17,7 @@
 
 ## SDK
 - Script mode: register and run workflows all in one command using a pre-defined base image
-- Flyte remote GA: register workflows and interact with Flyte execution artifacts programmmatically
+- Flyte remote GA: register workflows and interact with Flyte execution artifacts programmatically
 - Configuration overhaul: use the same config across flytekit and flytectl
 - Fast register without having AWS or other cloud credentials on your laptop, all you need is Flyte access
 

CHANGELOG/CHANGELOG-v0.2.0.md

@@ -16,7 +16,7 @@
 - RawOutputDirectories created in FlytePropeller
 - Improve visibility and observability
 - User/System error differentiation
-- Optional interruptible tasks (lets use spot instances, reduce cost)
+- Optional interruptible tasks (let's use spot instances, to reduce cost)
 - Caps on queue time for workflows
 - Multi cluster improvements
 - Visibility into execution cluster for the execution

CHANGELOG/CHANGELOG-v1.0.0.md

@@ -21,7 +21,7 @@ The pyflyte run command has been slightly updated to not need a `:` when selecti
     $ pyflyte run --remote example.py wf --n 500 --mean 42 --sigma 2
     ```
 * Script mode: register and run workflows all in one command using a pre-defined base image
-* Flyte remote GA: register workflows and interact with Flyte execution artifacts programmmatically
+* Flyte remote GA: register workflows and interact with Flyte execution artifacts programmatically
 * Configuration overhaul: use the same config across flytekit and flytectl
 * Fast register without having AWS/GCP or other cloud credentials on your laptop, all you need is Flyte access
 

CHANGELOG/CHANGELOG-v1.0.2-b1.md

@@ -5,7 +5,7 @@
 1. [Bugfix](https://github.com/flyteorg/flyte/issues/2444) With GRPC v1.46.0 non-ascii chars are not permitted in grpc metadata
 1. [Housekeeping](https://github.com/flyteorg/flyte/issues/1698) Configure grpc_health_prob in admin
 1. [Feature](https://github.com/flyteorg/flyte/issues/2329) In Flytectl use Launchplan with latest version for scheduled workflows
-1. [Bugfix](https://github.com/flyteorg/flyte/issues/2262) Pods started before InjectFinalizer is disabled are never deleted
+1. [Bugfix](https://github.com/flyteorg/flyte/issues/2262) Pods started before InjectFinalizer was disabled are never deleted
 1. [Housekeeping](https://github.com/flyteorg/flyte/issues/2504) Checksum grpc_health_probe
 1. [Feature](https://github.com/flyteorg/flyte/issues/2284) Allow to choose Spot Instances at workflow start time
 1. [Feature](https://github.com/flyteorg/flyte/pull/2439) Use the same pod annotation formatting in syncresources cronjob

CHANGELOG/CHANGELOG-v1.1.0.md

@@ -17,7 +17,7 @@ Support for [Optional types](https://github.com/flyteorg/flyte/issues/2426). Wit
 
 ### Bug Fixes
 * [Propeller](https://github.com/flyteorg/flyte/issues/2298) calling finalize rather than abort
-* [Propeller](https://github.com/flyteorg/flyte/issues/2404) correctly identify error when requesting a launch plan that does not exist.
+* [Propeller](https://github.com/flyteorg/flyte/issues/2404) correctly identifies an error when requesting a launch plan that does not exist.
 * Better handle [execution CRDs](https://github.com/flyteorg/flyte/issues/2275) that don't exist in Admin.
 * [Fix panic](https://github.com/flyteorg/flyte/issues/2597) when creating additional label options.
 * Check [validity](https://github.com/flyteorg/flyte/issues/2601) of notifications.

CHANGELOG/CHANGELOG-v1.13.0.md

@@ -0,0 +1,55 @@
+# Flyte 1.13.0 Release Notes
+
+Flyte 1.13.0 brings a host of new features, optimizations, and fixes, enhancing the platform's functionality and user experience. This release also welcomes several new contributors to the Flyte community. Below are the highlights of this release.
+
+## 🚀 New Features & Improvements
+
+1. **Features**
+   - [Watch agent metadata service](https://github.com/flyteorg/flyte/pull/5017)
+   - [Allow controlling in which task phases log links are shown](https://github.com/flyteorg/flyte/pull/4726)
+   - [Add executionClusterLabel](https://github.com/flyteorg/flyte/pull/5394)
+
+2. **Helm Charts & Manifests**
+   - [Grafana dashboard updates](https://github.com/flyteorg/flyte/pull/5255) for better monitoring and observability.
+   - [Core helm chart updates for propeller configuration of agent service](https://github.com/flyteorg/flyte/pull/5402).
+   - [Helm chart updates related to Prometheus, Webhook HPA, and Flyteconsole probes](https://github.com/flyteorg/flyte/pull/5508).
+
+3. **Documentation Improvements**
+   - Added examples and guides, including [a guide for setting up the OpenAI batch agent backend](https://github.com/flyteorg/flyte/pull/5291) and [updated Optimization Performance docs](https://github.com/flyteorg/flyte/pull/5278).
+   - [Fix doc link to testing agent on local cluster](https://github.com/flyteorg/flyte/pull/5398) and [replace Azure AD OIDC URL](https://github.com/flyteorg/flyte/pull/4075).
+   - [Clarify networking configurations between data plane propeller and control plane data catalog in multi-cluster deployments](https://github.com/flyteorg/flyte/pull/5345).
+
+4. **Performance & Bug Fixes**
+   - [Fix cache and token management issues](https://github.com/flyteorg/flyte/pull/5388), including addressing auto-refresh cache race conditions and improving token refresh mechanisms.
+   - [Fix SQL insert operations](https://github.com/flyteorg/flyte/pull/5482) to handle NULL values appropriately.
+   - [Keep EnvFrom from pod template](https://github.com/flyteorg/flyte/pull/5423)
+   - [Miscellaneous fixes](https://github.com/flyteorg/flyte/pull/5416) for pod templates, Kubeflow webhook errors, and logging configurations.
+
+5. **Flytectl**
+   - [Add prefetch functionality for paginator](https://github.com/flyteorg/flyte/pull/5310)
+   - [Fix `upgrade` and `version` commands](https://github.com/flyteorg/flyte/pull/5470)
+
+
+## 🔧 Housekeeping & Deprecations
+
+1. **Miscellaneous**
+   - [Add OTLP and sampling to otelutils](https://github.com/flyteorg/flyte/pull/5504)
+   - [Refactor distributed job using common ReplicaSpec](https://github.com/flyteorg/flyte/pull/5355)
+   - [Fix typos using codespell CI job](https://github.com/flyteorg/flyte/pull/5418).
+   - [Remove obsolete Flyte config files](https://github.com/flyteorg/flyte/pull/5495).
+   - [Upgraded various dependencies](https://github.com/flyteorg/flyte/pull/5313), including Golang, Docker, and others.
+   - [Move to upstream mockery](https://github.com/flyteorg/flyte/pull/4937), which simplifies the migration to mockery-v2 
+   - [Key-value execution tags](https://github.com/flyteorg/flyte/pull/5453)
+
+## 🆕 New Contributors
+
+- **@zychen5186** for [adding prefetch functionality for paginator](https://github.com/flyteorg/flyte/pull/5310).
+- **@EraYaN** for [replacing Azure AD OIDC URL](https://github.com/flyteorg/flyte/pull/4075).
+- **@Dlougach** for [making BaseURL insensitive to trailing slashes for metadata endpoint redirect](https://github.com/flyteorg/flyte/pull/5458).
+- **@flixr** for [honoring redoc.enabled=false in charts](https://github.com/flyteorg/flyte/pull/5452).
+- **@trevormcguire** for [including group in apiVersion in plugin_collector](https://github.com/flyteorg/flyte/pull/5457).
+- **@va6996** for [inheriting execution cluster label from source execution](https://github.com/flyteorg/flyte/pull/5431).
+- **@mhotan** for [Helm chart updates related to Prometheus, Webhook HPA, and Flyteconsole probes](https://github.com/flyteorg/flyte/pull/5508).
+- **@eltociear** for [updating token_source.go](https://github.com/flyteorg/flyte/pull/5396).
+
+**[Full Changelog](https://github.com/flyteorg/flyte/compare/v1.12.0...v1.13.0)**