merge master

Signed-off-by: Paul Dittamo <[email protected]>

.gitattributes

@@ -1 +1,5 @@
 docs/**/*html linguist-generated=true
+flyteidl/gen/** linguist-generated=true
+flyteidl/protos/**/*.rst linguist-generated=true
+flyteidl/clients/go/assets/admin.swagger.json linguist-generated=true
+

.github/workflows/generate_flyte_manifest.yml

@@ -23,7 +23,6 @@ jobs:
           VERSION: ${{ github.event.inputs.next-version }}
         run: |
           make release_automation
-          make kustomize
           make helm
       - name: Create Pull Request
         id: cpr
@@ -39,17 +38,12 @@ jobs:
           title: 'Update Flyte components'
           body: |
             Updated flyte deployment
-            - Updated GCP Flyte kustomize generated manifest file
-            - Updated EKS Flyte kustomize generated manifest file
-            - Updated Sandbox Flyte  kustomize generated manifest file
-            - Updated TEST Flyte kustomize generated manifest file
             - Updated GCP Flyte helm generated manifest file
             - Updated EKS Flyte helm generated manifest file
             - Updated Sandbox Flyte helm generated manifest file
             - Updated TEST Flyte helm generated manifest file
             - Auto-generated by [flyte-bot]
           labels: |
-            kustomize
             helm
           team-reviewers: |
             flyte-maintainers

.github/workflows/sandbox.yml

@@ -62,60 +62,3 @@ jobs:
           tags: ${{ steps.dind-names.outputs.tags }}
           build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}"
           file: docker/sandbox/Dockerfile
-
-  trigger-sandbox-lite-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: "0"
-      - name: Setup Golang caches
-        uses: actions/cache@v3
-        with:
-          path: |
-            /root/.cache/go-build
-            /root/go/pkg/mod
-          key: ${{ runner.os }}-golang-${{ hashFiles('go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-golang-
-      - name: Set flyte version to release
-        id: set_version
-        run: |
-          if [ ${{ github.event_name}} = "release" ]; then
-            echo ::set-output name=flyte_version::$(echo ${{ github.event.release.tag_name }})
-          else
-            echo ::set-output name=flyte_version::latest
-          fi
-      - name: Prepare DIND Image Names
-        id: dind-names
-        uses: docker/metadata-action@v3
-        with:
-          # list of Docker images to use as base name for tags
-          images: |
-            ghcr.io/${{ github.repository_owner }}/flyte-sandbox-lite
-          tags: |
-            ${{ steps.set_version.outputs.flyte_version }}
-            type=sha,format=long
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      - name: Set up Docker Buildx
-        id: buildx
-        uses: docker/setup-buildx-action@v1
-      - name: Login to GitHub Container Registry
-        if: ${{ github.event_name == 'release' }}
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: "${{ secrets.FLYTE_BOT_USERNAME }}"
-          password: "${{ secrets.FLYTE_BOT_PAT }}"
-      - name: Build and push DIND Image
-        uses: docker/build-push-action@v2
-        with:
-          context: .
-          platforms: linux/arm64, linux/amd64
-          push: ${{ github.event_name == 'release' }}
-          target: dind
-          tags: ${{ steps.dind-names.outputs.tags }}
-          build-args: "FLYTE_VERSION=${{ steps.set_version.outputs.flyte_version }}"
-          file: Dockerfile.sandbox-lite

.github/workflows/tests.yml

@@ -27,20 +27,35 @@ jobs:
   docs:
     runs-on: ubuntu-latest
     steps:
-      - name: Fetch the code
+      - name: Fetch flyte code
+        uses: actions/checkout@v4
+        with:
+          path: "${{ github.workspace }}/flyte"
+      - name: Fetch flytekit code
         uses: actions/checkout@v4
+        with:
+          repository: flyteorg/flytekit
+          path: "${{ github.workspace }}/flytekit"
       - uses: conda-incubator/setup-miniconda@v3
         with:
           auto-update-conda: true
           python-version: 3.9
       - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flyte
         run: |
           conda install -c conda-forge conda-lock
           conda-lock install -n monodocs-env monodocs-environment.lock.yaml
       - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flyte
         run: |
           conda activate monodocs-env
+          export SETUPTOOLS_SCM_PRETEND_VERSION="2.0.0"
           pip install -e ./flyteidl
+      - shell: bash -el {0}
+        working-directory: ${{ github.workspace }}/flytekit
+        run: |
+          conda activate monodocs-env
+          pip install -e .
           conda info
           conda list
           conda config --show-sources
@@ -49,21 +64,12 @@ jobs:
       - name: Setup Graphviz
         uses: ts-graphviz/setup-graphviz@v1
       - name: Build the documentation
+        working-directory: ${{ github.workspace }}/flyte
         shell: bash -el {0}
         run: |
           conda activate monodocs-env
           make docs
 
-  generate_kustomize:
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v2
-      - uses: actions/setup-go@v2
-        with:
-          go-version: "1.21"
-      - name: Kustomize and diff
-        run: DELTA_CHECK=true make kustomize
-
   generate_helm:
     runs-on: ubuntu-latest
     steps:

.gitignore

@@ -20,8 +20,6 @@ __pycache__/
 /charts/flyteagent/Chart.lock
 /charts/flyte-deps/Chart.lock
 /charts/flyte-core/Chart.lock
-/docker/sandbox-lite/storage/charts
-/docker/sandbox-lite/storage/Chart.lock
 **/*.bak-
 .python-version
 dist/

CHANGELOG/CHANGELOG-v1.11.0-b0.md

@@ -0,0 +1,3 @@
+# Flyte v1.11.0-b0
+
+Beta release to test new idl

CHANGELOG/CHANGELOG-v1.11.0-b1.md

@@ -0,0 +1,3 @@
+# Flyte v1.11.0-b1
+
+Second beta release for 1.11.0.

CHANGELOG/CHANGELOG-v1.11.0.md

@@ -0,0 +1,32 @@
+# Flyte v1.11.0 Release Notes
+
+We're excited to announce the release of Flyte v1.11.0! This version brings a host of improvements, bug fixes, and new features designed to enhance your experience with Flyte. From operational enhancements to documentation updates, this release aims to make Flyte more robust, user-friendly, and feature-rich.
+
+## Highlights
+
+- **Agents hit General Availability (GA):** Agents, now in General Availability, are long-running, stateless services that facilitate asynchronous job launches on platforms like Databricks or Snowflake and enable external service calls. They are versatile, supporting implementations in any language through a protobuf interface, enhancing Flyte's flexibility and operational efficiency.
+- **Improved Caching:** Support for loading cached sublists with multiple data types has been introduced, eliminating issues related to cache retrieval across varied data formats.
+- **Tracing and Observability:** The introduction of opentelemetry BlobstoreClientTracer in flyteadmin enhances observability, allowing for better monitoring and troubleshooting.
+- **Security Enhancements:** Added securityContext configuration to Flyte-core charts, strengthening the security posture of Flyte deployments.
+- **Documentation Overhaul:** Continuous improvements and updates have been made to the documentation, fixing broken links and updating content for better clarity and usability.
+- **Operational Improvements:** This release introduces enhancements such as adding a service account for V1 Ray Jobs, caching console assets in a single binary, and conditional mounting of secrets to improve the operational efficiency of Flyte. Additionally, we are removing `kustomize` from our deployment process to simplify the configuration and management of Flyte instances, making it easier for users to maintain and streamline their deployment workflows.
+
+
+## Bug Fixes
+
+- **Fixed Literal in Launchplan:** Added fixed_literal to the launchplan template, addressing issues with hardcoded values in workflows.
+- **Corrected Metadata and Resources:** Fixes have been applied to correct IsParent metadata in ArrayNode eventing and to address invalid "resources" scope issues in deployment configurations.
+- **Enhanced Stability and Performance:** Numerous bug fixes have been implemented to address stability and performance issues, including fixes for data catalog errors, yaml comment errors in pod template examples, and more.
+
+## Documentation and Guides
+
+- **Comprehensive Guides:** New guides and documentation updates have been added, including a ChatGPT Agent Setup guide and an Airflow migration guide. Improvements in documentation for developing agents have been integrated into the broader enhancements for this release.
+- **Updated Troubleshooting and Configuration Docs:** New troubleshooting guides for spark task execution and updates to deployment configuration documents enhance the knowledge base for Flyte users.
+
+## Contributors
+
+We extend our deepest gratitude to all the contributors who made this release possible. Special shoutouts to @neilisaur, @lowc1012, @MortalHappiness, @novahow, and @pryce-turner for making their first contributions!
+
+**For a full list of changes, enhancements, and bug fixes, visit our [changelog](https://github.com/flyteorg/flyte/compare/v1.10.7...v1.11.0).**
+
+Thank you for your continued support of Flyte. We look forward to hearing your feedback on this release!